diff --git a/libevent/x86_64/lib/libevent.a b/libevent/x86_64/lib/libevent.a index 27ec407a..1e0b777b 100644 Binary files a/libevent/x86_64/lib/libevent.a and b/libevent/x86_64/lib/libevent.a differ diff --git a/libevent/x86_64/lib/libevent_core.a b/libevent/x86_64/lib/libevent_core.a index 4e2b0b31..daa05a27 100644 Binary files a/libevent/x86_64/lib/libevent_core.a and b/libevent/x86_64/lib/libevent_core.a differ diff --git a/opencc/arm64-v8a/include/opencc/Common.hpp b/opencc/arm64-v8a/include/opencc/Common.hpp new file mode 100644 index 00000000..067966b5 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Common.hpp @@ -0,0 +1,82 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Microsoft Visual C++ specific +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820) +#endif + +#include +#include +#include +#include + +#include "Export.hpp" +#include "Optional.hpp" +#include "opencc_config.h" + +// Forward decalarations and alias +namespace opencc { +class Config; +class Conversion; +class ConversionChain; +class Converter; +class Dict; +class DictEntry; +class DictGroup; +class Lexicon; +class MarisaDict; +class MultiValueDictEntry; +class NoValueDictEntry; +class Segmentation; +class Segments; +class SerializableDict; +class SingleValueDictEntry; +class TextDict; +typedef std::shared_ptr ConversionPtr; +typedef std::shared_ptr ConversionChainPtr; +typedef std::shared_ptr ConverterPtr; +typedef std::shared_ptr DictPtr; +typedef std::shared_ptr DictGroupPtr; +typedef std::shared_ptr LexiconPtr; +typedef std::shared_ptr MarisaDictPtr; +typedef std::shared_ptr SegmentationPtr; +typedef std::shared_ptr SegmentsPtr; +typedef std::shared_ptr SerializableDictPtr; +typedef std::shared_ptr TextDictPtr; + +#ifdef OPENCC_ENABLE_DARTS +class BinaryDict; +class DartsDict; +typedef std::shared_ptr BinaryDictPtr; +typedef std::shared_ptr DartsDictPtr; +#endif + +} // namespace opencc + +#ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = ""; +#else // ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/"; +#endif // ifndef PKGDATADIR + +#ifndef VERSION +#define VERSION "1.0.*" +#endif // ifndef VERSION diff --git a/opencc/arm64-v8a/include/opencc/Config.hpp b/opencc/arm64-v8a/include/opencc/Config.hpp new file mode 100644 index 00000000..7a904cee --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Config.hpp @@ -0,0 +1,42 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Configuration loader + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Config { +public: + Config(); + + virtual ~Config(); + + ConverterPtr NewFromString(const std::string& json, + const std::string& configDirectory); + + ConverterPtr NewFromFile(const std::string& fileName); + +private: + void* internal; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Conversion.hpp b/opencc/arm64-v8a/include/opencc/Conversion.hpp new file mode 100644 index 00000000..cf73a254 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Conversion.hpp @@ -0,0 +1,47 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Conversion interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Conversion { +public: + Conversion(DictPtr _dict) : dict(_dict) {} + + // Convert single phrase + std::string Convert(const std::string& phrase) const; + + // Convert single phrase + std::string Convert(const char* phrase) const; + + // Convert segmented text + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/ConversionChain.hpp b/opencc/arm64-v8a/include/opencc/ConversionChain.hpp new file mode 100644 index 00000000..28853c67 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/ConversionChain.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Conversion.hpp" + +namespace opencc { +/** + * Chain of conversions + * Consists of a list of conversions. Converts input in sequence. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT ConversionChain { +public: + ConversionChain(const std::list _conversions); + + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const std::list GetConversions() const { return conversions; } + +private: + const std::list conversions; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Converter.hpp b/opencc/arm64-v8a/include/opencc/Converter.hpp new file mode 100644 index 00000000..9f1f3f94 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Converter.hpp @@ -0,0 +1,51 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Controller of segmentation and conversion + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Converter { +public: + Converter(const std::string& _name, SegmentationPtr _segmentation, + ConversionChainPtr _conversionChain) + : name(_name), segmentation(_segmentation), + conversionChain(_conversionChain) {} + + std::string Convert(const std::string& text) const; + + size_t Convert(const char* input, char* output) const; + + const SegmentationPtr GetSegmentation() const { return segmentation; } + + const ConversionChainPtr GetConversionChain() const { + return conversionChain; + } + +private: + const std::string name; + const SegmentationPtr segmentation; + const ConversionChainPtr conversionChain; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Dict.hpp b/opencc/arm64-v8a/include/opencc/Dict.hpp new file mode 100644 index 00000000..1c810346 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Dict.hpp @@ -0,0 +1,92 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Abstract class of dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Dict { +public: + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + virtual Optional Match(const char* word, + size_t len) const = 0; + + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + Optional Match(const std::string& word) const { + return Match(word.c_str(), word.length()); + } + + /** + * Matches the longest matched prefix of a word. + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * the longest prefix of "banana" matched is "bana". + */ + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const char* word) const { + return MatchPrefix(word, KeyMaxLength()); + } + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const std::string& word) const { + return MatchPrefix(word.c_str(), word.length()); + } + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * all the matched prefixes of "banana" are "bana", "ban", "ba", "b". + */ + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + */ + std::vector + MatchAllPrefixes(const std::string& word) const { + return MatchAllPrefixes(word.c_str(), word.length()); + } + + /** + * Returns the length of the longest key in the dictionary. + */ + virtual size_t KeyMaxLength() const = 0; + + /** + * Returns all entries in the dictionary. + */ + virtual LexiconPtr GetLexicon() const = 0; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/DictConverter.hpp b/opencc/arm64-v8a/include/opencc/DictConverter.hpp new file mode 100644 index 00000000..f911c4fe --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/DictConverter.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2017 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Converts a dictionary from a format to another. + * @ingroup opencc_cpp_api + */ +OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName, + const std::string& outputFileName, + const std::string& formatFrom, + const std::string& formatTo); +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/DictEntry.hpp b/opencc/arm64-v8a/include/opencc/DictEntry.hpp new file mode 100644 index 00000000..7b2babda --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/DictEntry.hpp @@ -0,0 +1,173 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segments.hpp" +#include "UTF8Util.hpp" + +namespace opencc { +/** + * Key-values pair entry + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictEntry { +public: + virtual ~DictEntry() {} + + virtual std::string Key() const = 0; + + virtual std::vector Values() const = 0; + + virtual std::string GetDefault() const = 0; + + virtual size_t NumValues() const = 0; + + virtual std::string ToString() const = 0; + + size_t KeyLength() const { return Key().length(); } + + bool operator<(const DictEntry& that) const { return Key() < that.Key(); } + + bool operator==(const DictEntry& that) const { return Key() == that.Key(); } + + static bool UPtrLessThan(const std::unique_ptr& a, + const std::unique_ptr& b) { + return *a < *b; + } +}; + +class OPENCC_EXPORT NoValueDictEntry : public DictEntry { +public: + NoValueDictEntry(const std::string& _key) : key(_key) {} + + virtual ~NoValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::vector Values() const { + return std::vector(); + } + + virtual std::string GetDefault() const { return key; } + + virtual size_t NumValues() const { return 0; } + + virtual std::string ToString() const { return key; } + +private: + std::string key; +}; + +class OPENCC_EXPORT SingleValueDictEntry : public DictEntry { +public: + virtual std::string Value() const = 0; + + virtual std::vector Values() const { + return std::vector{Value()}; + } + + virtual std::string GetDefault() const { return Value(); } + + virtual size_t NumValues() const { return 1; } + + virtual std::string ToString() const { + return std::string(Key()) + "\t" + Value(); + } +}; + +class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry { +public: + StrSingleValueDictEntry(const std::string& _key, const std::string& _value) + : key(_key), value(_value) {} + + virtual ~StrSingleValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::string Value() const { return value; } + +private: + std::string key; + std::string value; +}; + +class OPENCC_EXPORT MultiValueDictEntry : public DictEntry { +public: + virtual std::string GetDefault() const { + if (NumValues() > 0) { + return Values().at(0); + } else { + return Key(); + } + } + + virtual std::string ToString() const; +}; + +class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry { +public: + StrMultiValueDictEntry(const std::string& _key, + const std::vector& _values) + : key(_key), values(_values) {} + + virtual ~StrMultiValueDictEntry() {} + + virtual std::string Key() const { return key; } + + size_t NumValues() const { return values.size(); } + + std::vector Values() const { return values; } + +private: + std::string key; + std::vector values; +}; + +class OPENCC_EXPORT DictEntryFactory { +public: + static DictEntry* New(const std::string& key) { + return new NoValueDictEntry(key); + } + + static DictEntry* New(const std::string& key, const std::string& value) { + return new StrSingleValueDictEntry(key, value); + } + + static DictEntry* New(const std::string& key, + const std::vector& values) { + if (values.size() == 0) { + return New(key); + } else if (values.size() == 1) { + return New(key, values.front()); + } + return new StrMultiValueDictEntry(key, values); + } + + static DictEntry* New(const DictEntry* entry) { + if (entry->NumValues() == 0) { + return new NoValueDictEntry(entry->Key()); + } else if (entry->NumValues() == 1) { + return new StrSingleValueDictEntry(entry->Key(), entry->Values().front()); + } else { + return new StrMultiValueDictEntry(entry->Key(), entry->Values()); + } + } +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/DictGroup.hpp b/opencc/arm64-v8a/include/opencc/DictGroup.hpp new file mode 100644 index 00000000..fd51d913 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/DictGroup.hpp @@ -0,0 +1,57 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Dict.hpp" + +namespace opencc { +/** + * Group of dictionaries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictGroup : public Dict { +public: + DictGroup(const std::list& dicts); + + static DictGroupPtr NewFromDict(const Dict& dict); + + virtual ~DictGroup(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + const std::list GetDicts() const { return dicts; } + +private: + const size_t keyMaxLength; + const std::list dicts; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Exception.hpp b/opencc/arm64-v8a/include/opencc/Exception.hpp new file mode 100644 index 00000000..7875537f --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Exception.hpp @@ -0,0 +1,88 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Export.hpp" + +#if defined(_MSC_VER) && _MSC_VER < 1900 +// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not +// supported +#define noexcept +#endif // ifdef _MSC_VER + +namespace opencc { + +class OPENCC_EXPORT Exception { +public: + Exception() {} + + virtual ~Exception() throw() {} + + Exception(const std::string& _message) : message(_message) {} + + virtual const char* what() const noexcept { return message.c_str(); } + +protected: + std::string message; +}; + +class OPENCC_EXPORT FileNotFound : public Exception { +public: + FileNotFound(const std::string& fileName) + : Exception(fileName + " not found or not accessible.") {} +}; + +class OPENCC_EXPORT FileNotWritable : public Exception { +public: + FileNotWritable(const std::string& fileName) + : Exception(fileName + " not writable.") {} +}; + +class OPENCC_EXPORT InvalidFormat : public Exception { +public: + InvalidFormat(const std::string& message) + : Exception("Invalid format: " + message) {} +}; + +class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat { +public: + InvalidTextDictionary(const std::string& _message, size_t lineNum) + : InvalidFormat("") { + std::ostringstream buffer; + buffer << "Invalid text dictionary at line " << lineNum << ": " << _message; + message = buffer.str(); + } +}; + +class OPENCC_EXPORT InvalidUTF8 : public Exception { +public: + InvalidUTF8(const std::string& _message) + : Exception("Invalid UTF8: " + _message) {} +}; + +class OPENCC_EXPORT ShouldNotBeHere : public Exception { +public: + ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {} +}; + +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Export.hpp b/opencc/arm64-v8a/include/opencc/Export.hpp new file mode 100644 index 00000000..0015e880 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Export.hpp @@ -0,0 +1,40 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#define OPENCC_EXPORT +#define OPENCC_NO_EXPORT +#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#ifndef OPENCC_EXPORT +#ifdef libopencc_EXPORTS + +/* We are building this library */ +#define OPENCC_EXPORT __declspec(dllexport) +#else // ifdef libopencc_EXPORTS + +/* We are using this library */ +#define OPENCC_EXPORT __declspec(dllimport) +#endif // ifdef libopencc_EXPORTS +#endif // ifndef OPENCC_EXPORT + +#ifndef OPENCC_NO_EXPORT +#define OPENCC_NO_EXPORT +#endif // ifndef OPENCC_NO_EXPORT +#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) diff --git a/opencc/arm64-v8a/include/opencc/Lexicon.hpp b/opencc/arm64-v8a/include/opencc/Lexicon.hpp new file mode 100644 index 00000000..68891287 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Lexicon.hpp @@ -0,0 +1,68 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Storage of all entries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Lexicon { +public: + Lexicon() {} + Lexicon(std::vector> entries_) + : entries(std::move(entries_)) {} + Lexicon(const Lexicon&) = delete; + Lexicon& operator=(const Lexicon&) = delete; + + // Lexicon will take the ownership of the entry. + void Add(DictEntry* entry) { entries.emplace_back(entry); } + + void Add(std::unique_ptr entry) { + entries.push_back(std::move(entry)); + } + + void Sort(); + + // Returns true if the lexicon is sorted by key. + bool IsSorted(); + + // Returns true if every key unique (after sorted). + // When dupkey is set, it is set to the duplicate key. + bool IsUnique(std::string* dupkey = nullptr); + + const DictEntry* At(size_t index) const { return entries.at(index).get(); } + + size_t Length() const { return entries.size(); } + + std::vector>::const_iterator begin() const { + return entries.begin(); + } + + std::vector>::const_iterator end() const { + return entries.end(); + } + +private: + std::vector> entries; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/MarisaDict.hpp b/opencc/arm64-v8a/include/opencc/MarisaDict.hpp new file mode 100644 index 00000000..6917c23a --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/MarisaDict.hpp @@ -0,0 +1,63 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Darts dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict { +public: + virtual ~MarisaDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a MarisaDict from another dictionary. + */ + static MarisaDictPtr NewFromDict(const Dict& thatDict); + + static MarisaDictPtr NewFromFile(FILE* fp); + +private: + MarisaDict(); + + size_t maxLength; + LexiconPtr lexicon; + + class MarisaInternal; + std::unique_ptr internal; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/MaxMatchSegmentation.hpp b/opencc/arm64-v8a/include/opencc/MaxMatchSegmentation.hpp new file mode 100644 index 00000000..1ecc227e --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/MaxMatchSegmentation.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictGroup.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Implementation of maximal match segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation { +public: + MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {} + + virtual ~MaxMatchSegmentation() {} + + virtual SegmentsPtr Segment(const std::string& text) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Optional.hpp b/opencc/arm64-v8a/include/opencc/Optional.hpp new file mode 100644 index 00000000..8a510f43 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Optional.hpp @@ -0,0 +1,76 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace opencc { +/** + * A class that wraps type T into a nullable type. + * @ingroup opencc_cpp_api + */ +template class Optional { +public: + /** + * The constructor of Optional. + */ + Optional(T actual) : isNull(false), data(actual) {} + + /** + * Returns true if the instance is null. + */ + bool IsNull() const { return isNull; } + + /** + * Returns the containing data of the instance. + */ + const T& Get() const { return data; } + + /** + * Constructs a null instance. + */ + static Optional Null() { return Optional(); } + +private: + Optional() : isNull(true) {} + + bool isNull; + T data; +}; + +/** + * Specialization of Optional for pointers. + * + * Reduce a bool. + */ +template class Optional { +private: + Optional() : data(nullptr) {} + + typedef T* TPtr; + TPtr data; + +public: + Optional(TPtr actual) : data(actual) {} + + bool IsNull() const { return data == nullptr; } + + const TPtr& Get() const { return data; } + + static Optional Null() { return Optional(); } +}; +} // namespace opencc \ No newline at end of file diff --git a/opencc/arm64-v8a/include/opencc/PhraseExtract.hpp b/opencc/arm64-v8a/include/opencc/PhraseExtract.hpp new file mode 100644 index 00000000..75a165de --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/PhraseExtract.hpp @@ -0,0 +1,195 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Common.hpp" +#include "UTF8StringSlice.hpp" + +namespace opencc { + +class OPENCC_EXPORT PhraseExtract { +public: + typedef UTF8StringSlice::LengthType LengthType; + + typedef UTF8StringSliceBase UTF8StringSlice8Bit; + + PhraseExtract(); + + virtual ~PhraseExtract(); + + void Extract(const std::string& text) { + SetFullText(text); + ExtractSuffixes(); + CalculateFrequency(); + CalculateSuffixEntropy(); + ReleaseSuffixes(); + ExtractPrefixes(); + CalculatePrefixEntropy(); + ReleasePrefixes(); + ExtractWordCandidates(); + CalculateCohesions(); + SelectWords(); + } + + void SetFullText(const std::string& fullText) { + utf8FullText = UTF8StringSlice(fullText.c_str()); + } + + void SetFullText(const char* fullText) { + utf8FullText = UTF8StringSlice(fullText); + } + + void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; } + + void SetWordMinLength(const LengthType _wordMinLength) { + wordMinLength = _wordMinLength; + } + + void SetWordMaxLength(const LengthType _wordMaxLength) { + wordMaxLength = _wordMaxLength; + } + + void SetPrefixSetLength(const LengthType _prefixSetLength) { + prefixSetLength = _prefixSetLength; + } + + void SetSuffixSetLength(const LengthType _suffixSetLength) { + suffixSetLength = _suffixSetLength; + } + + // PreCalculationFilter is called after frequencies statistics. + void SetPreCalculationFilter( + const std::function& filter) { + preCalculationFilter = filter; + } + + void SetPostCalculationFilter( + const std::function& filter) { + postCalculationFilter = filter; + } + + void ReleaseSuffixes() { std::vector().swap(suffixes); } + + void ReleasePrefixes() { std::vector().swap(prefixes); } + + const std::vector& Words() const { return words; } + + const std::vector& WordCandidates() const { + return wordCandidates; + } + + struct Signals { + size_t frequency; + double cohesion; + double suffixEntropy; + double prefixEntropy; + }; + + const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const; + + double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double Entropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + size_t Frequency(const UTF8StringSlice8Bit& word) const; + + double Probability(const UTF8StringSlice8Bit& word) const; + + double LogProbability(const UTF8StringSlice8Bit& word) const; + + void Reset(); + + void ExtractSuffixes(); + + void ExtractPrefixes(); + + void ExtractWordCandidates(); + + void CalculateFrequency(); + + void CalculateCohesions(); + + void CalculateSuffixEntropy(); + + void CalculatePrefixEntropy(); + + void SelectWords(); + + static bool + DefaultPreCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + + static bool + DefaultPostCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + +private: + class DictType; + + // Pointwise Mutual Information + double PMI(const UTF8StringSlice8Bit& wordCandidate, + const UTF8StringSlice8Bit& part1, + const UTF8StringSlice8Bit& part2) const; + + double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double CalculateEntropy( + const std::unordered_map& choices) const; + + LengthType wordMinLength; + LengthType wordMaxLength; + LengthType prefixSetLength; + LengthType suffixSetLength; + std::function + preCalculationFilter; + std::function + postCalculationFilter; + + bool prefixesExtracted; + bool suffixesExtracted; + bool frequenciesCalculated; + bool wordCandidatesExtracted; + bool cohesionsCalculated; + bool prefixEntropiesCalculated; + bool suffixEntropiesCalculated; + bool wordsSelected; + + UTF8StringSlice utf8FullText; + size_t totalOccurrence; + double logTotalOccurrence; + std::vector prefixes; + std::vector suffixes; + std::vector wordCandidates; + std::vector words; + DictType* signals; + + friend class PhraseExtractTest; +}; + +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Segmentation.hpp b/opencc/arm64-v8a/include/opencc/Segmentation.hpp new file mode 100644 index 00000000..c260ea6d --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Segmentation.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Abstract segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segmentation { +public: + virtual SegmentsPtr Segment(const std::string& text) const = 0; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/Segments.hpp b/opencc/arm64-v8a/include/opencc/Segments.hpp new file mode 100644 index 00000000..21c39813 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/Segments.hpp @@ -0,0 +1,114 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" + +namespace opencc { +/** + * Segmented text + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segments { +public: + Segments() {} + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + void AddSegment(const char* unmanagedString) { + indexes.push_back(std::make_pair(unmanaged.size(), false)); + unmanaged.push_back(unmanagedString); + } + + void AddSegment(const std::string& str) { + indexes.push_back(std::make_pair(managed.size(), true)); + managed.push_back(str); + } + + class iterator : public std::iterator { + public: + iterator(const Segments* const _segments, size_t _cursor) + : segments(_segments), cursor(_cursor) {} + + iterator& operator++() { + cursor++; + return *this; + } + + bool operator==(const iterator& that) const { + return cursor == that.cursor && segments == that.segments; + } + + bool operator!=(const iterator& that) const { + return !this->operator==(that); + } + + const char* operator*() const { return segments->At(cursor); } + + private: + const Segments* const segments; + size_t cursor; + }; + + const char* At(size_t cursor) const { + const auto& index = indexes[cursor]; + if (index.second) { + return managed[index.first].c_str(); + } else { + return unmanaged[index.first]; + } + } + + size_t Length() const { return indexes.size(); } + + iterator begin() const { return iterator(this, 0); } + + iterator end() const { return iterator(this, indexes.size()); } + + std::string ToString() const { + // TODO implement a nested structure to reduce concatenation, + // like a purely functional differential list + std::ostringstream buffer; + for (const char* segment : *this) { + buffer << segment; + } + return buffer.str(); + } + +private: + Segments(const Segments&) {} + + std::vector unmanaged; + std::vector managed; + // index, managed + std::vector> indexes; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/SerializableDict.hpp b/opencc/arm64-v8a/include/opencc/SerializableDict.hpp new file mode 100644 index 00000000..17cea890 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/SerializableDict.hpp @@ -0,0 +1,77 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Dict.hpp" + +namespace opencc { +/** + * Serializable dictionary interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializableDict { +public: + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(FILE* fp) const = 0; + + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(const std::string& fileName) const { + FILE* fp = fopen(fileName.c_str(), "wb"); + if (fp == NULL) { + throw FileNotWritable(fileName); + } + SerializeToFile(fp); + fclose(fp); + } + + template + static bool TryLoadFromFile(const std::string& fileName, + std::shared_ptr* dict) { + FILE* fp = +#ifdef _MSC_VER + // well, the 'GetPlatformString' shall return a 'wstring' + _wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb") +#else + fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb") +#endif // _MSC_VER + ; + + if (fp == NULL) { + return false; + } + std::shared_ptr loadedDict = DICT::NewFromFile(fp); + fclose(fp); + *dict = loadedDict; + return true; + } + + template + static std::shared_ptr NewFromFile(const std::string& fileName) { + std::shared_ptr dict; + if (!TryLoadFromFile(fileName, &dict)) { + throw FileNotFound(fileName); + } + return dict; + } +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/SerializedValues.hpp b/opencc/arm64-v8a/include/opencc/SerializedValues.hpp new file mode 100644 index 00000000..c17210ba --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/SerializedValues.hpp @@ -0,0 +1,50 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Binary format for dictionary values serialization. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializedValues : public SerializableDict { +public: + SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {} + + virtual ~SerializedValues() {} + + virtual void SerializeToFile(FILE* fp) const; + + static std::shared_ptr NewFromFile(FILE* fp); + + const LexiconPtr& GetLexicon() const { return lexicon; } + + size_t KeyMaxLength() const; + +private: + LexiconPtr lexicon; + + void ConstructBuffer(std::string* valueBuffer, + std::vector* valueBytes, + uint32_t* valueTotalLength) const; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/SimpleConverter.hpp b/opencc/arm64-v8a/include/opencc/SimpleConverter.hpp new file mode 100644 index 00000000..56932b76 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/SimpleConverter.hpp @@ -0,0 +1,93 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Export.hpp" +#include + +#ifndef __OPENCC_SIMPLECONVERTER_HPP_ +#define __OPENCC_SIMPLECONVERTER_HPP_ + +/** + * @defgroup opencc_simple_api OpenCC C++ Simple API + * + * Simple API in C++ language + */ + +namespace opencc { +/** + * A high level converter + * This interface does not require C++11 to compile. + * @ingroup opencc_simple_api + */ +class OPENCC_EXPORT SimpleConverter { +public: + /** + * Constructor of SimpleConverter + * @param configFileName File name of configuration. + */ + SimpleConverter(const std::string& configFileName); + + ~SimpleConverter(); + + /** + * Converts a text + * @param input Text to be converted. + */ + std::string Convert(const std::string& input) const; + + /** + * Converts a text + * @param input A C-Style std::string (terminated by '\0') to be converted. + */ + std::string Convert(const char* input) const; + + /** + * Converts a text + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + */ + std::string Convert(const char* input, size_t length) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string (terminated by '\0') to be converted. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, char* output) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, size_t length, char* output) const; + +private: + const void* internalData; +}; + +} // namespace opencc + +#endif diff --git a/opencc/arm64-v8a/include/opencc/TextDict.hpp b/opencc/arm64-v8a/include/opencc/TextDict.hpp new file mode 100644 index 00000000..f1cb67d9 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/TextDict.hpp @@ -0,0 +1,60 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Text dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT TextDict : public Dict, public SerializableDict { +public: + /** + * Constructor of TextDict. + * _lexicon must be sorted. + */ + TextDict(const LexiconPtr& _lexicon); + + virtual ~TextDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a TextDict from another dictionary. + */ + static TextDictPtr NewFromDict(const Dict& dict); + + static TextDictPtr NewFromFile(FILE* fp); + + static TextDictPtr NewFromSortedFile(FILE* fp); + +private: + const size_t maxLength; + const LexiconPtr lexicon; +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/UTF8StringSlice.hpp b/opencc/arm64-v8a/include/opencc/UTF8StringSlice.hpp new file mode 100644 index 00000000..0c9dfdc6 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/UTF8StringSlice.hpp @@ -0,0 +1,246 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "Common.hpp" +#include "UTF8Util.hpp" + +namespace opencc { + +namespace internal { + +inline size_t FNVHash(const char* text, const size_t byteLength, + const size_t FNV_prime, const size_t FNV_offset_basis) { + size_t hash = FNV_offset_basis; + for (const char* pstr = text; pstr < text + byteLength; pstr++) { + hash ^= *pstr; + hash *= FNV_prime; + } + return hash; +} + +template size_t FNVHash(const char* text, const size_t byteLength); + +template <> +inline size_t FNVHash<4>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 16777619UL, 2166136261UL); +} + +#if SIZE_MAX == 0xffffffffffffffff +template <> +inline size_t FNVHash<8>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL); +} +#endif + +} // namespace internal + +template class UTF8StringSliceBase { +public: + typedef LENGTH_TYPE LengthType; + + UTF8StringSliceBase(const char* _str) + : str(_str), utf8Length(static_cast(UTF8Util::Length(_str))), + byteLength(static_cast(strlen(_str))) {} + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length) + : str(_str), utf8Length(_utf8Length) { + CalculateByteLength(); + } + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length, + const LengthType _byteLength) + : str(_str), utf8Length(_utf8Length), byteLength(_byteLength) { + CalculateByteLength(); + } + + LengthType UTF8Length() const { return utf8Length; } + + LengthType ByteLength() const { return byteLength; } + + UTF8StringSliceBase Left(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + return UTF8StringSliceBase(str, numberOfCharacters); + } + } + + UTF8StringSliceBase Right(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + const char* pstr = str + byteLength; + for (size_t i = 0; i < numberOfCharacters; i++) { + pstr = UTF8Util::PrevChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + UTF8StringSliceBase SubString(const LengthType offset, + const LengthType numberOfCharacters) const { + if (offset == 0) { + return Left(numberOfCharacters); + } else { + const char* pstr = str; + for (size_t i = 0; i < offset; i++) { + pstr = UTF8Util::NextChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + std::string ToString() const { return std::string(str, str + byteLength); } + + const char* CString() const { return str; } + + LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const { + if (str == that.str) { + return (std::min)(utf8Length, that.utf8Length); + } else { + const char* pstr1 = str; + const char* pstr2 = that.str; + for (size_t length = 0; length < utf8Length && length < that.utf8Length; + length++) { + size_t charLen1 = UTF8Util::NextCharLength(pstr1); + size_t charLen2 = UTF8Util::NextCharLength(pstr2); + if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) { + return length; + } + pstr1 += charLen1; + pstr2 += charLen2; + } + return 0; + } + } + + void MoveRight() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::NextCharLength(str); + str += charLen; + utf8Length--; + byteLength -= charLen; + } + } + + void MoveLeft() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::PrevCharLength(str + byteLength); + utf8Length--; + byteLength -= charLen; + } + } + + int ReverseCompare(const UTF8StringSliceBase& that) const { + const char* pstr1 = str + byteLength; + const char* pstr2 = that.str + that.byteLength; + const size_t length = (std::min)(utf8Length, that.utf8Length); + for (size_t i = 0; i < length; i++) { + const size_t charLen1 = UTF8Util::PrevCharLength(pstr1); + const size_t charLen2 = UTF8Util::PrevCharLength(pstr2); + pstr1 -= charLen1; + pstr2 -= charLen2; + const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2)); + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } else if (charLen1 < charLen2) { + return -1; + } else if (charLen1 > charLen2) { + return 1; + } + } + if (utf8Length < that.utf8Length) { + return -1; + } else if (utf8Length > that.utf8Length) { + return 1; + } else { + return 0; + } + } + + LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const { + return static_cast( + ToString().find(pattern.str, 0, pattern.byteLength)); + } + + bool operator<(const UTF8StringSliceBase& that) const { + return Compare(that) < 0; + } + + bool operator>(const UTF8StringSliceBase& that) const { + return Compare(that) > 0; + } + + bool operator==(const UTF8StringSliceBase& that) const { + return (str == that.str && utf8Length == that.utf8Length) || + Compare(that) == 0; + } + + bool operator!=(const UTF8StringSliceBase& that) const { + return !this->operator==(that); + } + + class Hasher { + public: + size_t operator()(const UTF8StringSliceBase& text) const { + return internal::FNVHash(text.CString(), + text.ByteLength()); + } + }; + +private: + inline int Compare(const UTF8StringSliceBase& that) const { + int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength)); + if (cmp == 0) { + if (utf8Length < that.utf8Length) { + cmp = -1; + } else if (utf8Length > that.utf8Length) { + cmp = 1; + } else { + cmp = 0; + } + } + return cmp; + } + + void CalculateByteLength() { + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + pstr = UTF8Util::NextChar(pstr); + } + byteLength = static_cast(pstr - str); + } + + const char* str; + LengthType utf8Length; + LengthType byteLength; +}; + +typedef UTF8StringSliceBase UTF8StringSlice; + +template +std::ostream& operator<<(::std::ostream& os, + const UTF8StringSliceBase& str) { + return os << str.ToString(); +} + +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/UTF8Util.hpp b/opencc/arm64-v8a/include/opencc/UTF8Util.hpp new file mode 100644 index 00000000..9d3e39b8 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/UTF8Util.hpp @@ -0,0 +1,290 @@ +/* + * Open Chinese Convert + * + * Copyright 2013 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef _MSC_VER +#define NOMINMAX +#include +#undef NOMINMAX +#endif // _MSC_VER + +#include + +#include "Common.hpp" +#include "Exception.hpp" + +namespace opencc { +/** + * UTF8 std::string utilities + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT UTF8Util { +public: + /** + * Detect UTF8 BOM and skip it. + */ + static void SkipUtf8Bom(FILE* fp); + + /** + * Returns the length in byte for the next UTF8 character. + * On error returns 0. + */ + static size_t NextCharLengthNoException(const char* str) { + char ch = *str; + if ((ch & 0xF0) == 0xE0) { + return 3; + } else if ((ch & 0x80) == 0x00) { + return 1; + } else if ((ch & 0xE0) == 0xC0) { + return 2; + } else if ((ch & 0xF8) == 0xF0) { + return 4; + } else if ((ch & 0xFC) == 0xF8) { + return 5; + } else if ((ch & 0xFE) == 0xFC) { + return 6; + } + return 0; + } + + /** + * Returns the length in byte for the next UTF8 character. + */ + static size_t NextCharLength(const char* str) { + size_t length = NextCharLengthNoException(str); + if (length == 0) { + throw InvalidUTF8(str); + } + return length; + } + + /** + * Returns the length in byte for the previous UTF8 character. + */ + static size_t PrevCharLength(const char* str) { + { + const size_t length = NextCharLengthNoException(str - 3); + if (length == 3) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 1); + if (length == 1) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 2); + if (length == 2) { + return length; + } + } + for (size_t i = 4; i <= 6; i++) { + const size_t length = NextCharLengthNoException(str - i); + if (length == i) { + return length; + } + } + throw InvalidUTF8(str); + } + + /** + * Returns the char* pointer over the next UTF8 character. + */ + static const char* NextChar(const char* str) { + return str + NextCharLength(str); + } + + /** + * Move the char* pointer before the previous UTF8 character. + */ + static const char* PrevChar(const char* str) { + return str - PrevCharLength(str); + } + + /** + * Returns the UTF8 length of a valid UTF8 std::string. + */ + static size_t Length(const char* str) { + size_t length = 0; + while (*str != '\0') { + str = NextChar(str); + length++; + } + return length; + } + + /** + * Finds a character in the same line. + * @param str The text to be searched in. + * @param ch The character to find. + * @return The pointer that points to the found chacter in str or EOL/EOF. + */ + static const char* FindNextInline(const char* str, const char ch) { + while (!IsLineEndingOrFileEnding(*str) && *str != ch) { + str = NextChar(str); + } + return str; + } + + /** + * Returns true if the character is a line ending or end of file. + */ + static bool IsLineEndingOrFileEnding(const char ch) { + return ch == '\0' || ch == '\n' || ch == '\r'; + } + + /** + * Copies a substring with given length to a new string. + */ + static std::string FromSubstr(const char* str, size_t length) { + std::string newStr; + newStr.resize(length); + strncpy(const_cast(newStr.c_str()), str, length); + return newStr; + } + + /** + * Returns true if the given std::string is longer or as long as the given + * length. + */ + static bool NotShorterThan(const char* str, size_t byteLength) { + while (byteLength > 0) { + if (*str == '\0') { + return false; + } + byteLength--; + str++; + } + return true; + } + + /** + * Truncates a std::string with a maximal length in byte. + * No UTF8 character will be broken. + */ + static std::string TruncateUTF8(const char* str, size_t maxByteLength) { + std::string wordTrunc; + if (NotShorterThan(str, maxByteLength)) { + size_t len = 0; + const char* pStr = str; + for (;;) { + const size_t charLength = NextCharLength(pStr); + if (len + charLength > maxByteLength) { + break; + } + pStr += charLength; + len += charLength; + } + wordTrunc = FromSubstr(str, len); + } else { + wordTrunc = str; + } + return wordTrunc; + } + + /** + * Replaces all patterns in a std::string in place. + */ + static void ReplaceAll(std::string& str, const char* from, const char* to) { + std::string::size_type pos = 0; + std::string::size_type fromLen = strlen(from); + std::string::size_type toLen = strlen(to); + while ((pos = str.find(from, pos)) != std::string::npos) { + str.replace(pos, fromLen, to); + pos += toLen; + } + } + + /** + * Joins a std::string vector in to a std::string with a separator. + */ + static std::string Join(const std::vector& strings, + const std::string& separator) { + std::ostringstream buffer; + bool first = true; + for (const auto& str : strings) { + if (!first) { + buffer << separator; + } + buffer << str; + first = false; + } + return buffer.str(); + } + + /** + * Joins a std::string vector in to a std::string. + */ + static std::string Join(const std::vector& strings) { + std::ostringstream buffer; + for (const auto& str : strings) { + buffer << str; + } + return buffer.str(); + } + + static void GetByteMap(const char* str, const size_t utf8Length, + std::vector* byteMap) { + if (byteMap->size() < utf8Length) { + byteMap->resize(utf8Length); + } + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + (*byteMap)[i] = pstr - str; + pstr = NextChar(pstr); + } + } + +#ifdef _MSC_VER + static std::wstring GetPlatformString(const std::string& str) { + return U8ToU16(str); + } +#else + static std::string GetPlatformString(const std::string& str) { return str; } +#endif // _MSC_VER + +#ifdef _MSC_VER + static std::string U16ToU8(const std::wstring& wstr) { + std::string ret; + int length = static_cast(wstr.length()); + int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0, + NULL, NULL); + if (convcnt > 0) { + ret.resize(convcnt); + WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt, + NULL, NULL); + } + return ret; + } + + static std::wstring U8ToU16(const std::string& str) { + std::wstring ret; + int length = static_cast(str.length()); + int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0); + if (convcnt > 0) { + ret.resize(convcnt); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt); + } + return ret; + } +#endif // _MSC_VER +}; +} // namespace opencc diff --git a/opencc/arm64-v8a/include/opencc/opencc.h b/opencc/arm64-v8a/include/opencc/opencc.h new file mode 100644 index 00000000..bc3a9e97 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/opencc.h @@ -0,0 +1,161 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __OPENCC_H_ +#define __OPENCC_H_ + +#ifdef __cplusplus + +#include "Export.hpp" +#include "SimpleConverter.hpp" +#include + +extern "C" { +#else +#include +#endif + +#ifndef OPENCC_EXPORT +#define OPENCC_EXPORT +#endif + +/** + * @defgroup opencc_c_api OpenCC C API + * + * API in C language + */ + +/** + * Filename of default Simplified to Traditional configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json" + +/** + * Filename of default Traditional to Simplified configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json" + +/** + * Type of opencc descriptor + * + * @ingroup opencc_c_api + */ +typedef void* opencc_t; + +/** + * Makes an instance of opencc + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open(const char* configFileName); +#ifdef _MSC_VER +/** + * Makes an instance of opencc (wide char / Unicode) + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName); +#endif /* _MSC_VER */ + +/** + * Destroys an instance of opencc + * + * @param opencc The description pointer. + * @return 0 on success or non-zero number on failure. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT int opencc_close(opencc_t opencc); + +/** + * Converts UTF-8 std::string + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * @param output The buffer to store converted text. You MUST make sure this + * buffer has sufficient space. + * + * @return The length of converted std::string or (size_t)-1 on error. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc, + const char* input, + size_t length, char* output); + +/** + * Converts UTF-8 std::string + * This function returns an allocated C-Style std::string, which stores + * the converted std::string. + * You MUST call opencc_convert_utf8_free() to release allocated memory. + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * + * @return The newly allocated UTF-8 std::string that stores text + * converted, or NULL on error. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input, + size_t length); + +/** + * Releases allocated buffer by opencc_convert_utf8 + * + * @param str Pointer to the allocated std::string buffer by + * opencc_convert_utf8. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT void opencc_convert_utf8_free(char* str); + +/** + * Returns the last error message + * + * Note that this function is the only one which is NOT thread-safe. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT const char* opencc_error(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +/** + * @defgroup opencc_cpp_api OpenCC C++ Comprehensive API + * + * Comprehensive API in C++ language + */ + +#endif diff --git a/opencc/arm64-v8a/include/opencc/opencc_config.h b/opencc/arm64-v8a/include/opencc/opencc_config.h new file mode 100644 index 00000000..8dad4489 --- /dev/null +++ b/opencc/arm64-v8a/include/opencc/opencc_config.h @@ -0,0 +1,21 @@ +/* + * Open Chinese Convert + * + * Copyright 2021 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* #undef OPENCC_ENABLE_DARTS */ diff --git a/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfig.cmake b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfig.cmake new file mode 100644 index 00000000..a360c2a4 --- /dev/null +++ b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfig.cmake @@ -0,0 +1,31 @@ + +####### Expanded from @PACKAGE_INIT@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run #### +####### The input file was OpenCCConfig.cmake.in ######## + +get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) + +macro(set_and_check _var _file) + set(${_var} "${_file}") + if(NOT EXISTS "${_file}") + message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") + endif() +endmacro() + +macro(check_required_components _NAME) + foreach(comp ${${_NAME}_FIND_COMPONENTS}) + if(NOT ${_NAME}_${comp}_FOUND) + if(${_NAME}_FIND_REQUIRED_${comp}) + set(${_NAME}_FOUND FALSE) + endif() + endif() + endforeach() +endmacro() + +#################################################################################### + +include(${CMAKE_CURRENT_LIST_DIR}/OpenCCTargets.cmake) + +set_and_check(OPENCC_INCLUDE_DIR ${PACKAGE_PREFIX_DIR}/include/opencc) + +check_required_components(OpenCC) diff --git a/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfigVersion.cmake b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfigVersion.cmake new file mode 100644 index 00000000..7fa638f2 --- /dev/null +++ b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCConfigVersion.cmake @@ -0,0 +1,70 @@ +# This is a basic version file for the Config-mode of find_package(). +# It is used by write_basic_package_version_file() as input file for configure_file() +# to create a version-file which can be installed along a config.cmake file. +# +# The created file sets PACKAGE_VERSION_EXACT if the current version string and +# the requested version string are exactly the same and it sets +# PACKAGE_VERSION_COMPATIBLE if the current version is >= requested version, +# but only if the requested major version is the same as the current one. +# The variable CVF_VERSION must be set before calling configure_file(). + + +set(PACKAGE_VERSION "1.1.6") + +if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + + if("1.1.6" MATCHES "^([0-9]+)\\.") + set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}") + if(NOT CVF_VERSION_MAJOR VERSION_EQUAL 0) + string(REGEX REPLACE "^0+" "" CVF_VERSION_MAJOR "${CVF_VERSION_MAJOR}") + endif() + else() + set(CVF_VERSION_MAJOR "1.1.6") + endif() + + if(PACKAGE_FIND_VERSION_RANGE) + # both endpoints of the range must have the expected major version + math (EXPR CVF_VERSION_MAJOR_NEXT "${CVF_VERSION_MAJOR} + 1") + if (NOT PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + OR ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX_MAJOR STREQUAL CVF_VERSION_MAJOR) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX VERSION_LESS_EQUAL CVF_VERSION_MAJOR_NEXT))) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + elseif(PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + AND ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND PACKAGE_VERSION VERSION_LESS_EQUAL PACKAGE_FIND_VERSION_MAX) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MAX))) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + else() + if(PACKAGE_FIND_VERSION_MAJOR STREQUAL CVF_VERSION_MAJOR) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + + if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + endif() + endif() +endif() + + +# if the installed project requested no architecture check, don't perform the check +if("FALSE") + return() +endif() + +# if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it: +if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "" OR "8" STREQUAL "") + return() +endif() + +# check that the installed version has the same 32/64bit-ness as the one which is currently searching: +if(NOT CMAKE_SIZEOF_VOID_P STREQUAL "8") + math(EXPR installedBits "8 * 8") + set(PACKAGE_VERSION "${PACKAGE_VERSION} (${installedBits}bit)") + set(PACKAGE_VERSION_UNSUITABLE TRUE) +endif() diff --git a/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets-release.cmake b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets-release.cmake new file mode 100644 index 00000000..199b87f5 --- /dev/null +++ b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets-release.cmake @@ -0,0 +1,29 @@ +#---------------------------------------------------------------- +# Generated CMake target import file for configuration "Release". +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Import target "OpenCC::OpenCC" for configuration "Release" +set_property(TARGET OpenCC::OpenCC APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::OpenCC PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libopencc.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::OpenCC ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::OpenCC "${_IMPORT_PREFIX}/lib/libopencc.a" ) + +# Import target "OpenCC::marisa" for configuration "Release" +set_property(TARGET OpenCC::marisa APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::marisa PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libmarisa.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::marisa ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::marisa "${_IMPORT_PREFIX}/lib/libmarisa.a" ) + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) diff --git a/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets.cmake b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets.cmake new file mode 100644 index 00000000..db461d58 --- /dev/null +++ b/opencc/arm64-v8a/lib/cmake/opencc/OpenCCTargets.cmake @@ -0,0 +1,102 @@ +# Generated by CMake + +if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS 2.6) + message(FATAL_ERROR "CMake >= 2.6.0 required") +endif() +cmake_policy(PUSH) +cmake_policy(VERSION 2.6...3.20) +#---------------------------------------------------------------- +# Generated CMake target import file. +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Protect against multiple inclusion, which would fail when already imported targets are added once more. +set(_targetsDefined) +set(_targetsNotDefined) +set(_expectedTargets) +foreach(_expectedTarget OpenCC::OpenCC OpenCC::marisa) + list(APPEND _expectedTargets ${_expectedTarget}) + if(NOT TARGET ${_expectedTarget}) + list(APPEND _targetsNotDefined ${_expectedTarget}) + endif() + if(TARGET ${_expectedTarget}) + list(APPEND _targetsDefined ${_expectedTarget}) + endif() +endforeach() +if("${_targetsDefined}" STREQUAL "${_expectedTargets}") + unset(_targetsDefined) + unset(_targetsNotDefined) + unset(_expectedTargets) + set(CMAKE_IMPORT_FILE_VERSION) + cmake_policy(POP) + return() +endif() +if(NOT "${_targetsDefined}" STREQUAL "") + message(FATAL_ERROR "Some (but not all) targets in this export set were already defined.\nTargets Defined: ${_targetsDefined}\nTargets not yet defined: ${_targetsNotDefined}\n") +endif() +unset(_targetsDefined) +unset(_targetsNotDefined) +unset(_expectedTargets) + + +# Compute the installation prefix relative to this file. +get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +if(_IMPORT_PREFIX STREQUAL "/") + set(_IMPORT_PREFIX "") +endif() + +# Create imported target OpenCC::OpenCC +add_library(OpenCC::OpenCC STATIC IMPORTED) + +set_target_properties(OpenCC::OpenCC PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include/opencc" + INTERFACE_LINK_LIBRARIES "OpenCC::marisa" +) + +# Create imported target OpenCC::marisa +add_library(OpenCC::marisa STATIC IMPORTED) + +if(CMAKE_VERSION VERSION_LESS 2.8.12) + message(FATAL_ERROR "This file relies on consumers using CMake 2.8.12 or greater.") +endif() + +# Load information for each installed configuration. +get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +file(GLOB CONFIG_FILES "${_DIR}/OpenCCTargets-*.cmake") +foreach(f ${CONFIG_FILES}) + include(${f}) +endforeach() + +# Cleanup temporary variables. +set(_IMPORT_PREFIX) + +# Loop over all imported files and verify that they actually exist +foreach(target ${_IMPORT_CHECK_TARGETS} ) + foreach(file ${_IMPORT_CHECK_FILES_FOR_${target}} ) + if(NOT EXISTS "${file}" ) + message(FATAL_ERROR "The imported target \"${target}\" references the file + \"${file}\" +but this file does not exist. Possible reasons include: +* The file was deleted, renamed, or moved to another location. +* An install or uninstall procedure did not complete successfully. +* The installation package was faulty and contained + \"${CMAKE_CURRENT_LIST_FILE}\" +but not all the files it references. +") + endif() + endforeach() + unset(_IMPORT_CHECK_FILES_FOR_${target}) +endforeach() +unset(_IMPORT_CHECK_TARGETS) + +# This file does not depend on other imported targets which have +# been exported from the same project but in a separate export set. + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) +cmake_policy(POP) diff --git a/opencc/arm64-v8a/lib/libmarisa.a b/opencc/arm64-v8a/lib/libmarisa.a new file mode 100644 index 00000000..b0055eb9 Binary files /dev/null and b/opencc/arm64-v8a/lib/libmarisa.a differ diff --git a/opencc/arm64-v8a/lib/libopencc.a b/opencc/arm64-v8a/lib/libopencc.a new file mode 100644 index 00000000..5c46dd91 Binary files /dev/null and b/opencc/arm64-v8a/lib/libopencc.a differ diff --git a/opencc/arm64-v8a/share/opencc b/opencc/arm64-v8a/share/opencc new file mode 120000 index 00000000..e67b4559 --- /dev/null +++ b/opencc/arm64-v8a/share/opencc @@ -0,0 +1 @@ +../../data \ No newline at end of file diff --git a/opencc/armeabi-v7a/include/opencc/Common.hpp b/opencc/armeabi-v7a/include/opencc/Common.hpp new file mode 100644 index 00000000..067966b5 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Common.hpp @@ -0,0 +1,82 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Microsoft Visual C++ specific +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820) +#endif + +#include +#include +#include +#include + +#include "Export.hpp" +#include "Optional.hpp" +#include "opencc_config.h" + +// Forward decalarations and alias +namespace opencc { +class Config; +class Conversion; +class ConversionChain; +class Converter; +class Dict; +class DictEntry; +class DictGroup; +class Lexicon; +class MarisaDict; +class MultiValueDictEntry; +class NoValueDictEntry; +class Segmentation; +class Segments; +class SerializableDict; +class SingleValueDictEntry; +class TextDict; +typedef std::shared_ptr ConversionPtr; +typedef std::shared_ptr ConversionChainPtr; +typedef std::shared_ptr ConverterPtr; +typedef std::shared_ptr DictPtr; +typedef std::shared_ptr DictGroupPtr; +typedef std::shared_ptr LexiconPtr; +typedef std::shared_ptr MarisaDictPtr; +typedef std::shared_ptr SegmentationPtr; +typedef std::shared_ptr SegmentsPtr; +typedef std::shared_ptr SerializableDictPtr; +typedef std::shared_ptr TextDictPtr; + +#ifdef OPENCC_ENABLE_DARTS +class BinaryDict; +class DartsDict; +typedef std::shared_ptr BinaryDictPtr; +typedef std::shared_ptr DartsDictPtr; +#endif + +} // namespace opencc + +#ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = ""; +#else // ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/"; +#endif // ifndef PKGDATADIR + +#ifndef VERSION +#define VERSION "1.0.*" +#endif // ifndef VERSION diff --git a/opencc/armeabi-v7a/include/opencc/Config.hpp b/opencc/armeabi-v7a/include/opencc/Config.hpp new file mode 100644 index 00000000..7a904cee --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Config.hpp @@ -0,0 +1,42 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Configuration loader + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Config { +public: + Config(); + + virtual ~Config(); + + ConverterPtr NewFromString(const std::string& json, + const std::string& configDirectory); + + ConverterPtr NewFromFile(const std::string& fileName); + +private: + void* internal; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Conversion.hpp b/opencc/armeabi-v7a/include/opencc/Conversion.hpp new file mode 100644 index 00000000..cf73a254 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Conversion.hpp @@ -0,0 +1,47 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Conversion interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Conversion { +public: + Conversion(DictPtr _dict) : dict(_dict) {} + + // Convert single phrase + std::string Convert(const std::string& phrase) const; + + // Convert single phrase + std::string Convert(const char* phrase) const; + + // Convert segmented text + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/ConversionChain.hpp b/opencc/armeabi-v7a/include/opencc/ConversionChain.hpp new file mode 100644 index 00000000..28853c67 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/ConversionChain.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Conversion.hpp" + +namespace opencc { +/** + * Chain of conversions + * Consists of a list of conversions. Converts input in sequence. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT ConversionChain { +public: + ConversionChain(const std::list _conversions); + + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const std::list GetConversions() const { return conversions; } + +private: + const std::list conversions; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Converter.hpp b/opencc/armeabi-v7a/include/opencc/Converter.hpp new file mode 100644 index 00000000..9f1f3f94 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Converter.hpp @@ -0,0 +1,51 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Controller of segmentation and conversion + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Converter { +public: + Converter(const std::string& _name, SegmentationPtr _segmentation, + ConversionChainPtr _conversionChain) + : name(_name), segmentation(_segmentation), + conversionChain(_conversionChain) {} + + std::string Convert(const std::string& text) const; + + size_t Convert(const char* input, char* output) const; + + const SegmentationPtr GetSegmentation() const { return segmentation; } + + const ConversionChainPtr GetConversionChain() const { + return conversionChain; + } + +private: + const std::string name; + const SegmentationPtr segmentation; + const ConversionChainPtr conversionChain; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Dict.hpp b/opencc/armeabi-v7a/include/opencc/Dict.hpp new file mode 100644 index 00000000..1c810346 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Dict.hpp @@ -0,0 +1,92 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Abstract class of dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Dict { +public: + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + virtual Optional Match(const char* word, + size_t len) const = 0; + + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + Optional Match(const std::string& word) const { + return Match(word.c_str(), word.length()); + } + + /** + * Matches the longest matched prefix of a word. + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * the longest prefix of "banana" matched is "bana". + */ + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const char* word) const { + return MatchPrefix(word, KeyMaxLength()); + } + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const std::string& word) const { + return MatchPrefix(word.c_str(), word.length()); + } + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * all the matched prefixes of "banana" are "bana", "ban", "ba", "b". + */ + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + */ + std::vector + MatchAllPrefixes(const std::string& word) const { + return MatchAllPrefixes(word.c_str(), word.length()); + } + + /** + * Returns the length of the longest key in the dictionary. + */ + virtual size_t KeyMaxLength() const = 0; + + /** + * Returns all entries in the dictionary. + */ + virtual LexiconPtr GetLexicon() const = 0; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/DictConverter.hpp b/opencc/armeabi-v7a/include/opencc/DictConverter.hpp new file mode 100644 index 00000000..f911c4fe --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/DictConverter.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2017 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Converts a dictionary from a format to another. + * @ingroup opencc_cpp_api + */ +OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName, + const std::string& outputFileName, + const std::string& formatFrom, + const std::string& formatTo); +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/DictEntry.hpp b/opencc/armeabi-v7a/include/opencc/DictEntry.hpp new file mode 100644 index 00000000..7b2babda --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/DictEntry.hpp @@ -0,0 +1,173 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segments.hpp" +#include "UTF8Util.hpp" + +namespace opencc { +/** + * Key-values pair entry + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictEntry { +public: + virtual ~DictEntry() {} + + virtual std::string Key() const = 0; + + virtual std::vector Values() const = 0; + + virtual std::string GetDefault() const = 0; + + virtual size_t NumValues() const = 0; + + virtual std::string ToString() const = 0; + + size_t KeyLength() const { return Key().length(); } + + bool operator<(const DictEntry& that) const { return Key() < that.Key(); } + + bool operator==(const DictEntry& that) const { return Key() == that.Key(); } + + static bool UPtrLessThan(const std::unique_ptr& a, + const std::unique_ptr& b) { + return *a < *b; + } +}; + +class OPENCC_EXPORT NoValueDictEntry : public DictEntry { +public: + NoValueDictEntry(const std::string& _key) : key(_key) {} + + virtual ~NoValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::vector Values() const { + return std::vector(); + } + + virtual std::string GetDefault() const { return key; } + + virtual size_t NumValues() const { return 0; } + + virtual std::string ToString() const { return key; } + +private: + std::string key; +}; + +class OPENCC_EXPORT SingleValueDictEntry : public DictEntry { +public: + virtual std::string Value() const = 0; + + virtual std::vector Values() const { + return std::vector{Value()}; + } + + virtual std::string GetDefault() const { return Value(); } + + virtual size_t NumValues() const { return 1; } + + virtual std::string ToString() const { + return std::string(Key()) + "\t" + Value(); + } +}; + +class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry { +public: + StrSingleValueDictEntry(const std::string& _key, const std::string& _value) + : key(_key), value(_value) {} + + virtual ~StrSingleValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::string Value() const { return value; } + +private: + std::string key; + std::string value; +}; + +class OPENCC_EXPORT MultiValueDictEntry : public DictEntry { +public: + virtual std::string GetDefault() const { + if (NumValues() > 0) { + return Values().at(0); + } else { + return Key(); + } + } + + virtual std::string ToString() const; +}; + +class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry { +public: + StrMultiValueDictEntry(const std::string& _key, + const std::vector& _values) + : key(_key), values(_values) {} + + virtual ~StrMultiValueDictEntry() {} + + virtual std::string Key() const { return key; } + + size_t NumValues() const { return values.size(); } + + std::vector Values() const { return values; } + +private: + std::string key; + std::vector values; +}; + +class OPENCC_EXPORT DictEntryFactory { +public: + static DictEntry* New(const std::string& key) { + return new NoValueDictEntry(key); + } + + static DictEntry* New(const std::string& key, const std::string& value) { + return new StrSingleValueDictEntry(key, value); + } + + static DictEntry* New(const std::string& key, + const std::vector& values) { + if (values.size() == 0) { + return New(key); + } else if (values.size() == 1) { + return New(key, values.front()); + } + return new StrMultiValueDictEntry(key, values); + } + + static DictEntry* New(const DictEntry* entry) { + if (entry->NumValues() == 0) { + return new NoValueDictEntry(entry->Key()); + } else if (entry->NumValues() == 1) { + return new StrSingleValueDictEntry(entry->Key(), entry->Values().front()); + } else { + return new StrMultiValueDictEntry(entry->Key(), entry->Values()); + } + } +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/DictGroup.hpp b/opencc/armeabi-v7a/include/opencc/DictGroup.hpp new file mode 100644 index 00000000..fd51d913 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/DictGroup.hpp @@ -0,0 +1,57 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Dict.hpp" + +namespace opencc { +/** + * Group of dictionaries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictGroup : public Dict { +public: + DictGroup(const std::list& dicts); + + static DictGroupPtr NewFromDict(const Dict& dict); + + virtual ~DictGroup(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + const std::list GetDicts() const { return dicts; } + +private: + const size_t keyMaxLength; + const std::list dicts; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Exception.hpp b/opencc/armeabi-v7a/include/opencc/Exception.hpp new file mode 100644 index 00000000..7875537f --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Exception.hpp @@ -0,0 +1,88 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Export.hpp" + +#if defined(_MSC_VER) && _MSC_VER < 1900 +// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not +// supported +#define noexcept +#endif // ifdef _MSC_VER + +namespace opencc { + +class OPENCC_EXPORT Exception { +public: + Exception() {} + + virtual ~Exception() throw() {} + + Exception(const std::string& _message) : message(_message) {} + + virtual const char* what() const noexcept { return message.c_str(); } + +protected: + std::string message; +}; + +class OPENCC_EXPORT FileNotFound : public Exception { +public: + FileNotFound(const std::string& fileName) + : Exception(fileName + " not found or not accessible.") {} +}; + +class OPENCC_EXPORT FileNotWritable : public Exception { +public: + FileNotWritable(const std::string& fileName) + : Exception(fileName + " not writable.") {} +}; + +class OPENCC_EXPORT InvalidFormat : public Exception { +public: + InvalidFormat(const std::string& message) + : Exception("Invalid format: " + message) {} +}; + +class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat { +public: + InvalidTextDictionary(const std::string& _message, size_t lineNum) + : InvalidFormat("") { + std::ostringstream buffer; + buffer << "Invalid text dictionary at line " << lineNum << ": " << _message; + message = buffer.str(); + } +}; + +class OPENCC_EXPORT InvalidUTF8 : public Exception { +public: + InvalidUTF8(const std::string& _message) + : Exception("Invalid UTF8: " + _message) {} +}; + +class OPENCC_EXPORT ShouldNotBeHere : public Exception { +public: + ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {} +}; + +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Export.hpp b/opencc/armeabi-v7a/include/opencc/Export.hpp new file mode 100644 index 00000000..0015e880 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Export.hpp @@ -0,0 +1,40 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#define OPENCC_EXPORT +#define OPENCC_NO_EXPORT +#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#ifndef OPENCC_EXPORT +#ifdef libopencc_EXPORTS + +/* We are building this library */ +#define OPENCC_EXPORT __declspec(dllexport) +#else // ifdef libopencc_EXPORTS + +/* We are using this library */ +#define OPENCC_EXPORT __declspec(dllimport) +#endif // ifdef libopencc_EXPORTS +#endif // ifndef OPENCC_EXPORT + +#ifndef OPENCC_NO_EXPORT +#define OPENCC_NO_EXPORT +#endif // ifndef OPENCC_NO_EXPORT +#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) diff --git a/opencc/armeabi-v7a/include/opencc/Lexicon.hpp b/opencc/armeabi-v7a/include/opencc/Lexicon.hpp new file mode 100644 index 00000000..68891287 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Lexicon.hpp @@ -0,0 +1,68 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Storage of all entries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Lexicon { +public: + Lexicon() {} + Lexicon(std::vector> entries_) + : entries(std::move(entries_)) {} + Lexicon(const Lexicon&) = delete; + Lexicon& operator=(const Lexicon&) = delete; + + // Lexicon will take the ownership of the entry. + void Add(DictEntry* entry) { entries.emplace_back(entry); } + + void Add(std::unique_ptr entry) { + entries.push_back(std::move(entry)); + } + + void Sort(); + + // Returns true if the lexicon is sorted by key. + bool IsSorted(); + + // Returns true if every key unique (after sorted). + // When dupkey is set, it is set to the duplicate key. + bool IsUnique(std::string* dupkey = nullptr); + + const DictEntry* At(size_t index) const { return entries.at(index).get(); } + + size_t Length() const { return entries.size(); } + + std::vector>::const_iterator begin() const { + return entries.begin(); + } + + std::vector>::const_iterator end() const { + return entries.end(); + } + +private: + std::vector> entries; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/MarisaDict.hpp b/opencc/armeabi-v7a/include/opencc/MarisaDict.hpp new file mode 100644 index 00000000..6917c23a --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/MarisaDict.hpp @@ -0,0 +1,63 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Darts dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict { +public: + virtual ~MarisaDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a MarisaDict from another dictionary. + */ + static MarisaDictPtr NewFromDict(const Dict& thatDict); + + static MarisaDictPtr NewFromFile(FILE* fp); + +private: + MarisaDict(); + + size_t maxLength; + LexiconPtr lexicon; + + class MarisaInternal; + std::unique_ptr internal; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/MaxMatchSegmentation.hpp b/opencc/armeabi-v7a/include/opencc/MaxMatchSegmentation.hpp new file mode 100644 index 00000000..1ecc227e --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/MaxMatchSegmentation.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictGroup.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Implementation of maximal match segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation { +public: + MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {} + + virtual ~MaxMatchSegmentation() {} + + virtual SegmentsPtr Segment(const std::string& text) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Optional.hpp b/opencc/armeabi-v7a/include/opencc/Optional.hpp new file mode 100644 index 00000000..8a510f43 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Optional.hpp @@ -0,0 +1,76 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace opencc { +/** + * A class that wraps type T into a nullable type. + * @ingroup opencc_cpp_api + */ +template class Optional { +public: + /** + * The constructor of Optional. + */ + Optional(T actual) : isNull(false), data(actual) {} + + /** + * Returns true if the instance is null. + */ + bool IsNull() const { return isNull; } + + /** + * Returns the containing data of the instance. + */ + const T& Get() const { return data; } + + /** + * Constructs a null instance. + */ + static Optional Null() { return Optional(); } + +private: + Optional() : isNull(true) {} + + bool isNull; + T data; +}; + +/** + * Specialization of Optional for pointers. + * + * Reduce a bool. + */ +template class Optional { +private: + Optional() : data(nullptr) {} + + typedef T* TPtr; + TPtr data; + +public: + Optional(TPtr actual) : data(actual) {} + + bool IsNull() const { return data == nullptr; } + + const TPtr& Get() const { return data; } + + static Optional Null() { return Optional(); } +}; +} // namespace opencc \ No newline at end of file diff --git a/opencc/armeabi-v7a/include/opencc/PhraseExtract.hpp b/opencc/armeabi-v7a/include/opencc/PhraseExtract.hpp new file mode 100644 index 00000000..75a165de --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/PhraseExtract.hpp @@ -0,0 +1,195 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Common.hpp" +#include "UTF8StringSlice.hpp" + +namespace opencc { + +class OPENCC_EXPORT PhraseExtract { +public: + typedef UTF8StringSlice::LengthType LengthType; + + typedef UTF8StringSliceBase UTF8StringSlice8Bit; + + PhraseExtract(); + + virtual ~PhraseExtract(); + + void Extract(const std::string& text) { + SetFullText(text); + ExtractSuffixes(); + CalculateFrequency(); + CalculateSuffixEntropy(); + ReleaseSuffixes(); + ExtractPrefixes(); + CalculatePrefixEntropy(); + ReleasePrefixes(); + ExtractWordCandidates(); + CalculateCohesions(); + SelectWords(); + } + + void SetFullText(const std::string& fullText) { + utf8FullText = UTF8StringSlice(fullText.c_str()); + } + + void SetFullText(const char* fullText) { + utf8FullText = UTF8StringSlice(fullText); + } + + void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; } + + void SetWordMinLength(const LengthType _wordMinLength) { + wordMinLength = _wordMinLength; + } + + void SetWordMaxLength(const LengthType _wordMaxLength) { + wordMaxLength = _wordMaxLength; + } + + void SetPrefixSetLength(const LengthType _prefixSetLength) { + prefixSetLength = _prefixSetLength; + } + + void SetSuffixSetLength(const LengthType _suffixSetLength) { + suffixSetLength = _suffixSetLength; + } + + // PreCalculationFilter is called after frequencies statistics. + void SetPreCalculationFilter( + const std::function& filter) { + preCalculationFilter = filter; + } + + void SetPostCalculationFilter( + const std::function& filter) { + postCalculationFilter = filter; + } + + void ReleaseSuffixes() { std::vector().swap(suffixes); } + + void ReleasePrefixes() { std::vector().swap(prefixes); } + + const std::vector& Words() const { return words; } + + const std::vector& WordCandidates() const { + return wordCandidates; + } + + struct Signals { + size_t frequency; + double cohesion; + double suffixEntropy; + double prefixEntropy; + }; + + const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const; + + double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double Entropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + size_t Frequency(const UTF8StringSlice8Bit& word) const; + + double Probability(const UTF8StringSlice8Bit& word) const; + + double LogProbability(const UTF8StringSlice8Bit& word) const; + + void Reset(); + + void ExtractSuffixes(); + + void ExtractPrefixes(); + + void ExtractWordCandidates(); + + void CalculateFrequency(); + + void CalculateCohesions(); + + void CalculateSuffixEntropy(); + + void CalculatePrefixEntropy(); + + void SelectWords(); + + static bool + DefaultPreCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + + static bool + DefaultPostCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + +private: + class DictType; + + // Pointwise Mutual Information + double PMI(const UTF8StringSlice8Bit& wordCandidate, + const UTF8StringSlice8Bit& part1, + const UTF8StringSlice8Bit& part2) const; + + double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double CalculateEntropy( + const std::unordered_map& choices) const; + + LengthType wordMinLength; + LengthType wordMaxLength; + LengthType prefixSetLength; + LengthType suffixSetLength; + std::function + preCalculationFilter; + std::function + postCalculationFilter; + + bool prefixesExtracted; + bool suffixesExtracted; + bool frequenciesCalculated; + bool wordCandidatesExtracted; + bool cohesionsCalculated; + bool prefixEntropiesCalculated; + bool suffixEntropiesCalculated; + bool wordsSelected; + + UTF8StringSlice utf8FullText; + size_t totalOccurrence; + double logTotalOccurrence; + std::vector prefixes; + std::vector suffixes; + std::vector wordCandidates; + std::vector words; + DictType* signals; + + friend class PhraseExtractTest; +}; + +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Segmentation.hpp b/opencc/armeabi-v7a/include/opencc/Segmentation.hpp new file mode 100644 index 00000000..c260ea6d --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Segmentation.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Abstract segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segmentation { +public: + virtual SegmentsPtr Segment(const std::string& text) const = 0; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/Segments.hpp b/opencc/armeabi-v7a/include/opencc/Segments.hpp new file mode 100644 index 00000000..21c39813 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/Segments.hpp @@ -0,0 +1,114 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" + +namespace opencc { +/** + * Segmented text + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segments { +public: + Segments() {} + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + void AddSegment(const char* unmanagedString) { + indexes.push_back(std::make_pair(unmanaged.size(), false)); + unmanaged.push_back(unmanagedString); + } + + void AddSegment(const std::string& str) { + indexes.push_back(std::make_pair(managed.size(), true)); + managed.push_back(str); + } + + class iterator : public std::iterator { + public: + iterator(const Segments* const _segments, size_t _cursor) + : segments(_segments), cursor(_cursor) {} + + iterator& operator++() { + cursor++; + return *this; + } + + bool operator==(const iterator& that) const { + return cursor == that.cursor && segments == that.segments; + } + + bool operator!=(const iterator& that) const { + return !this->operator==(that); + } + + const char* operator*() const { return segments->At(cursor); } + + private: + const Segments* const segments; + size_t cursor; + }; + + const char* At(size_t cursor) const { + const auto& index = indexes[cursor]; + if (index.second) { + return managed[index.first].c_str(); + } else { + return unmanaged[index.first]; + } + } + + size_t Length() const { return indexes.size(); } + + iterator begin() const { return iterator(this, 0); } + + iterator end() const { return iterator(this, indexes.size()); } + + std::string ToString() const { + // TODO implement a nested structure to reduce concatenation, + // like a purely functional differential list + std::ostringstream buffer; + for (const char* segment : *this) { + buffer << segment; + } + return buffer.str(); + } + +private: + Segments(const Segments&) {} + + std::vector unmanaged; + std::vector managed; + // index, managed + std::vector> indexes; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/SerializableDict.hpp b/opencc/armeabi-v7a/include/opencc/SerializableDict.hpp new file mode 100644 index 00000000..17cea890 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/SerializableDict.hpp @@ -0,0 +1,77 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Dict.hpp" + +namespace opencc { +/** + * Serializable dictionary interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializableDict { +public: + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(FILE* fp) const = 0; + + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(const std::string& fileName) const { + FILE* fp = fopen(fileName.c_str(), "wb"); + if (fp == NULL) { + throw FileNotWritable(fileName); + } + SerializeToFile(fp); + fclose(fp); + } + + template + static bool TryLoadFromFile(const std::string& fileName, + std::shared_ptr* dict) { + FILE* fp = +#ifdef _MSC_VER + // well, the 'GetPlatformString' shall return a 'wstring' + _wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb") +#else + fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb") +#endif // _MSC_VER + ; + + if (fp == NULL) { + return false; + } + std::shared_ptr loadedDict = DICT::NewFromFile(fp); + fclose(fp); + *dict = loadedDict; + return true; + } + + template + static std::shared_ptr NewFromFile(const std::string& fileName) { + std::shared_ptr dict; + if (!TryLoadFromFile(fileName, &dict)) { + throw FileNotFound(fileName); + } + return dict; + } +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/SerializedValues.hpp b/opencc/armeabi-v7a/include/opencc/SerializedValues.hpp new file mode 100644 index 00000000..c17210ba --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/SerializedValues.hpp @@ -0,0 +1,50 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Binary format for dictionary values serialization. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializedValues : public SerializableDict { +public: + SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {} + + virtual ~SerializedValues() {} + + virtual void SerializeToFile(FILE* fp) const; + + static std::shared_ptr NewFromFile(FILE* fp); + + const LexiconPtr& GetLexicon() const { return lexicon; } + + size_t KeyMaxLength() const; + +private: + LexiconPtr lexicon; + + void ConstructBuffer(std::string* valueBuffer, + std::vector* valueBytes, + uint32_t* valueTotalLength) const; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/SimpleConverter.hpp b/opencc/armeabi-v7a/include/opencc/SimpleConverter.hpp new file mode 100644 index 00000000..56932b76 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/SimpleConverter.hpp @@ -0,0 +1,93 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Export.hpp" +#include + +#ifndef __OPENCC_SIMPLECONVERTER_HPP_ +#define __OPENCC_SIMPLECONVERTER_HPP_ + +/** + * @defgroup opencc_simple_api OpenCC C++ Simple API + * + * Simple API in C++ language + */ + +namespace opencc { +/** + * A high level converter + * This interface does not require C++11 to compile. + * @ingroup opencc_simple_api + */ +class OPENCC_EXPORT SimpleConverter { +public: + /** + * Constructor of SimpleConverter + * @param configFileName File name of configuration. + */ + SimpleConverter(const std::string& configFileName); + + ~SimpleConverter(); + + /** + * Converts a text + * @param input Text to be converted. + */ + std::string Convert(const std::string& input) const; + + /** + * Converts a text + * @param input A C-Style std::string (terminated by '\0') to be converted. + */ + std::string Convert(const char* input) const; + + /** + * Converts a text + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + */ + std::string Convert(const char* input, size_t length) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string (terminated by '\0') to be converted. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, char* output) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, size_t length, char* output) const; + +private: + const void* internalData; +}; + +} // namespace opencc + +#endif diff --git a/opencc/armeabi-v7a/include/opencc/TextDict.hpp b/opencc/armeabi-v7a/include/opencc/TextDict.hpp new file mode 100644 index 00000000..f1cb67d9 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/TextDict.hpp @@ -0,0 +1,60 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Text dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT TextDict : public Dict, public SerializableDict { +public: + /** + * Constructor of TextDict. + * _lexicon must be sorted. + */ + TextDict(const LexiconPtr& _lexicon); + + virtual ~TextDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a TextDict from another dictionary. + */ + static TextDictPtr NewFromDict(const Dict& dict); + + static TextDictPtr NewFromFile(FILE* fp); + + static TextDictPtr NewFromSortedFile(FILE* fp); + +private: + const size_t maxLength; + const LexiconPtr lexicon; +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/UTF8StringSlice.hpp b/opencc/armeabi-v7a/include/opencc/UTF8StringSlice.hpp new file mode 100644 index 00000000..0c9dfdc6 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/UTF8StringSlice.hpp @@ -0,0 +1,246 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "Common.hpp" +#include "UTF8Util.hpp" + +namespace opencc { + +namespace internal { + +inline size_t FNVHash(const char* text, const size_t byteLength, + const size_t FNV_prime, const size_t FNV_offset_basis) { + size_t hash = FNV_offset_basis; + for (const char* pstr = text; pstr < text + byteLength; pstr++) { + hash ^= *pstr; + hash *= FNV_prime; + } + return hash; +} + +template size_t FNVHash(const char* text, const size_t byteLength); + +template <> +inline size_t FNVHash<4>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 16777619UL, 2166136261UL); +} + +#if SIZE_MAX == 0xffffffffffffffff +template <> +inline size_t FNVHash<8>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL); +} +#endif + +} // namespace internal + +template class UTF8StringSliceBase { +public: + typedef LENGTH_TYPE LengthType; + + UTF8StringSliceBase(const char* _str) + : str(_str), utf8Length(static_cast(UTF8Util::Length(_str))), + byteLength(static_cast(strlen(_str))) {} + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length) + : str(_str), utf8Length(_utf8Length) { + CalculateByteLength(); + } + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length, + const LengthType _byteLength) + : str(_str), utf8Length(_utf8Length), byteLength(_byteLength) { + CalculateByteLength(); + } + + LengthType UTF8Length() const { return utf8Length; } + + LengthType ByteLength() const { return byteLength; } + + UTF8StringSliceBase Left(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + return UTF8StringSliceBase(str, numberOfCharacters); + } + } + + UTF8StringSliceBase Right(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + const char* pstr = str + byteLength; + for (size_t i = 0; i < numberOfCharacters; i++) { + pstr = UTF8Util::PrevChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + UTF8StringSliceBase SubString(const LengthType offset, + const LengthType numberOfCharacters) const { + if (offset == 0) { + return Left(numberOfCharacters); + } else { + const char* pstr = str; + for (size_t i = 0; i < offset; i++) { + pstr = UTF8Util::NextChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + std::string ToString() const { return std::string(str, str + byteLength); } + + const char* CString() const { return str; } + + LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const { + if (str == that.str) { + return (std::min)(utf8Length, that.utf8Length); + } else { + const char* pstr1 = str; + const char* pstr2 = that.str; + for (size_t length = 0; length < utf8Length && length < that.utf8Length; + length++) { + size_t charLen1 = UTF8Util::NextCharLength(pstr1); + size_t charLen2 = UTF8Util::NextCharLength(pstr2); + if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) { + return length; + } + pstr1 += charLen1; + pstr2 += charLen2; + } + return 0; + } + } + + void MoveRight() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::NextCharLength(str); + str += charLen; + utf8Length--; + byteLength -= charLen; + } + } + + void MoveLeft() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::PrevCharLength(str + byteLength); + utf8Length--; + byteLength -= charLen; + } + } + + int ReverseCompare(const UTF8StringSliceBase& that) const { + const char* pstr1 = str + byteLength; + const char* pstr2 = that.str + that.byteLength; + const size_t length = (std::min)(utf8Length, that.utf8Length); + for (size_t i = 0; i < length; i++) { + const size_t charLen1 = UTF8Util::PrevCharLength(pstr1); + const size_t charLen2 = UTF8Util::PrevCharLength(pstr2); + pstr1 -= charLen1; + pstr2 -= charLen2; + const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2)); + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } else if (charLen1 < charLen2) { + return -1; + } else if (charLen1 > charLen2) { + return 1; + } + } + if (utf8Length < that.utf8Length) { + return -1; + } else if (utf8Length > that.utf8Length) { + return 1; + } else { + return 0; + } + } + + LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const { + return static_cast( + ToString().find(pattern.str, 0, pattern.byteLength)); + } + + bool operator<(const UTF8StringSliceBase& that) const { + return Compare(that) < 0; + } + + bool operator>(const UTF8StringSliceBase& that) const { + return Compare(that) > 0; + } + + bool operator==(const UTF8StringSliceBase& that) const { + return (str == that.str && utf8Length == that.utf8Length) || + Compare(that) == 0; + } + + bool operator!=(const UTF8StringSliceBase& that) const { + return !this->operator==(that); + } + + class Hasher { + public: + size_t operator()(const UTF8StringSliceBase& text) const { + return internal::FNVHash(text.CString(), + text.ByteLength()); + } + }; + +private: + inline int Compare(const UTF8StringSliceBase& that) const { + int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength)); + if (cmp == 0) { + if (utf8Length < that.utf8Length) { + cmp = -1; + } else if (utf8Length > that.utf8Length) { + cmp = 1; + } else { + cmp = 0; + } + } + return cmp; + } + + void CalculateByteLength() { + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + pstr = UTF8Util::NextChar(pstr); + } + byteLength = static_cast(pstr - str); + } + + const char* str; + LengthType utf8Length; + LengthType byteLength; +}; + +typedef UTF8StringSliceBase UTF8StringSlice; + +template +std::ostream& operator<<(::std::ostream& os, + const UTF8StringSliceBase& str) { + return os << str.ToString(); +} + +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/UTF8Util.hpp b/opencc/armeabi-v7a/include/opencc/UTF8Util.hpp new file mode 100644 index 00000000..9d3e39b8 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/UTF8Util.hpp @@ -0,0 +1,290 @@ +/* + * Open Chinese Convert + * + * Copyright 2013 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef _MSC_VER +#define NOMINMAX +#include +#undef NOMINMAX +#endif // _MSC_VER + +#include + +#include "Common.hpp" +#include "Exception.hpp" + +namespace opencc { +/** + * UTF8 std::string utilities + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT UTF8Util { +public: + /** + * Detect UTF8 BOM and skip it. + */ + static void SkipUtf8Bom(FILE* fp); + + /** + * Returns the length in byte for the next UTF8 character. + * On error returns 0. + */ + static size_t NextCharLengthNoException(const char* str) { + char ch = *str; + if ((ch & 0xF0) == 0xE0) { + return 3; + } else if ((ch & 0x80) == 0x00) { + return 1; + } else if ((ch & 0xE0) == 0xC0) { + return 2; + } else if ((ch & 0xF8) == 0xF0) { + return 4; + } else if ((ch & 0xFC) == 0xF8) { + return 5; + } else if ((ch & 0xFE) == 0xFC) { + return 6; + } + return 0; + } + + /** + * Returns the length in byte for the next UTF8 character. + */ + static size_t NextCharLength(const char* str) { + size_t length = NextCharLengthNoException(str); + if (length == 0) { + throw InvalidUTF8(str); + } + return length; + } + + /** + * Returns the length in byte for the previous UTF8 character. + */ + static size_t PrevCharLength(const char* str) { + { + const size_t length = NextCharLengthNoException(str - 3); + if (length == 3) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 1); + if (length == 1) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 2); + if (length == 2) { + return length; + } + } + for (size_t i = 4; i <= 6; i++) { + const size_t length = NextCharLengthNoException(str - i); + if (length == i) { + return length; + } + } + throw InvalidUTF8(str); + } + + /** + * Returns the char* pointer over the next UTF8 character. + */ + static const char* NextChar(const char* str) { + return str + NextCharLength(str); + } + + /** + * Move the char* pointer before the previous UTF8 character. + */ + static const char* PrevChar(const char* str) { + return str - PrevCharLength(str); + } + + /** + * Returns the UTF8 length of a valid UTF8 std::string. + */ + static size_t Length(const char* str) { + size_t length = 0; + while (*str != '\0') { + str = NextChar(str); + length++; + } + return length; + } + + /** + * Finds a character in the same line. + * @param str The text to be searched in. + * @param ch The character to find. + * @return The pointer that points to the found chacter in str or EOL/EOF. + */ + static const char* FindNextInline(const char* str, const char ch) { + while (!IsLineEndingOrFileEnding(*str) && *str != ch) { + str = NextChar(str); + } + return str; + } + + /** + * Returns true if the character is a line ending or end of file. + */ + static bool IsLineEndingOrFileEnding(const char ch) { + return ch == '\0' || ch == '\n' || ch == '\r'; + } + + /** + * Copies a substring with given length to a new string. + */ + static std::string FromSubstr(const char* str, size_t length) { + std::string newStr; + newStr.resize(length); + strncpy(const_cast(newStr.c_str()), str, length); + return newStr; + } + + /** + * Returns true if the given std::string is longer or as long as the given + * length. + */ + static bool NotShorterThan(const char* str, size_t byteLength) { + while (byteLength > 0) { + if (*str == '\0') { + return false; + } + byteLength--; + str++; + } + return true; + } + + /** + * Truncates a std::string with a maximal length in byte. + * No UTF8 character will be broken. + */ + static std::string TruncateUTF8(const char* str, size_t maxByteLength) { + std::string wordTrunc; + if (NotShorterThan(str, maxByteLength)) { + size_t len = 0; + const char* pStr = str; + for (;;) { + const size_t charLength = NextCharLength(pStr); + if (len + charLength > maxByteLength) { + break; + } + pStr += charLength; + len += charLength; + } + wordTrunc = FromSubstr(str, len); + } else { + wordTrunc = str; + } + return wordTrunc; + } + + /** + * Replaces all patterns in a std::string in place. + */ + static void ReplaceAll(std::string& str, const char* from, const char* to) { + std::string::size_type pos = 0; + std::string::size_type fromLen = strlen(from); + std::string::size_type toLen = strlen(to); + while ((pos = str.find(from, pos)) != std::string::npos) { + str.replace(pos, fromLen, to); + pos += toLen; + } + } + + /** + * Joins a std::string vector in to a std::string with a separator. + */ + static std::string Join(const std::vector& strings, + const std::string& separator) { + std::ostringstream buffer; + bool first = true; + for (const auto& str : strings) { + if (!first) { + buffer << separator; + } + buffer << str; + first = false; + } + return buffer.str(); + } + + /** + * Joins a std::string vector in to a std::string. + */ + static std::string Join(const std::vector& strings) { + std::ostringstream buffer; + for (const auto& str : strings) { + buffer << str; + } + return buffer.str(); + } + + static void GetByteMap(const char* str, const size_t utf8Length, + std::vector* byteMap) { + if (byteMap->size() < utf8Length) { + byteMap->resize(utf8Length); + } + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + (*byteMap)[i] = pstr - str; + pstr = NextChar(pstr); + } + } + +#ifdef _MSC_VER + static std::wstring GetPlatformString(const std::string& str) { + return U8ToU16(str); + } +#else + static std::string GetPlatformString(const std::string& str) { return str; } +#endif // _MSC_VER + +#ifdef _MSC_VER + static std::string U16ToU8(const std::wstring& wstr) { + std::string ret; + int length = static_cast(wstr.length()); + int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0, + NULL, NULL); + if (convcnt > 0) { + ret.resize(convcnt); + WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt, + NULL, NULL); + } + return ret; + } + + static std::wstring U8ToU16(const std::string& str) { + std::wstring ret; + int length = static_cast(str.length()); + int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0); + if (convcnt > 0) { + ret.resize(convcnt); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt); + } + return ret; + } +#endif // _MSC_VER +}; +} // namespace opencc diff --git a/opencc/armeabi-v7a/include/opencc/opencc.h b/opencc/armeabi-v7a/include/opencc/opencc.h new file mode 100644 index 00000000..bc3a9e97 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/opencc.h @@ -0,0 +1,161 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __OPENCC_H_ +#define __OPENCC_H_ + +#ifdef __cplusplus + +#include "Export.hpp" +#include "SimpleConverter.hpp" +#include + +extern "C" { +#else +#include +#endif + +#ifndef OPENCC_EXPORT +#define OPENCC_EXPORT +#endif + +/** + * @defgroup opencc_c_api OpenCC C API + * + * API in C language + */ + +/** + * Filename of default Simplified to Traditional configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json" + +/** + * Filename of default Traditional to Simplified configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json" + +/** + * Type of opencc descriptor + * + * @ingroup opencc_c_api + */ +typedef void* opencc_t; + +/** + * Makes an instance of opencc + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open(const char* configFileName); +#ifdef _MSC_VER +/** + * Makes an instance of opencc (wide char / Unicode) + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName); +#endif /* _MSC_VER */ + +/** + * Destroys an instance of opencc + * + * @param opencc The description pointer. + * @return 0 on success or non-zero number on failure. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT int opencc_close(opencc_t opencc); + +/** + * Converts UTF-8 std::string + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * @param output The buffer to store converted text. You MUST make sure this + * buffer has sufficient space. + * + * @return The length of converted std::string or (size_t)-1 on error. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc, + const char* input, + size_t length, char* output); + +/** + * Converts UTF-8 std::string + * This function returns an allocated C-Style std::string, which stores + * the converted std::string. + * You MUST call opencc_convert_utf8_free() to release allocated memory. + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * + * @return The newly allocated UTF-8 std::string that stores text + * converted, or NULL on error. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input, + size_t length); + +/** + * Releases allocated buffer by opencc_convert_utf8 + * + * @param str Pointer to the allocated std::string buffer by + * opencc_convert_utf8. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT void opencc_convert_utf8_free(char* str); + +/** + * Returns the last error message + * + * Note that this function is the only one which is NOT thread-safe. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT const char* opencc_error(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +/** + * @defgroup opencc_cpp_api OpenCC C++ Comprehensive API + * + * Comprehensive API in C++ language + */ + +#endif diff --git a/opencc/armeabi-v7a/include/opencc/opencc_config.h b/opencc/armeabi-v7a/include/opencc/opencc_config.h new file mode 100644 index 00000000..8dad4489 --- /dev/null +++ b/opencc/armeabi-v7a/include/opencc/opencc_config.h @@ -0,0 +1,21 @@ +/* + * Open Chinese Convert + * + * Copyright 2021 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* #undef OPENCC_ENABLE_DARTS */ diff --git a/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfig.cmake b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfig.cmake new file mode 100644 index 00000000..a360c2a4 --- /dev/null +++ b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfig.cmake @@ -0,0 +1,31 @@ + +####### Expanded from @PACKAGE_INIT@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run #### +####### The input file was OpenCCConfig.cmake.in ######## + +get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) + +macro(set_and_check _var _file) + set(${_var} "${_file}") + if(NOT EXISTS "${_file}") + message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") + endif() +endmacro() + +macro(check_required_components _NAME) + foreach(comp ${${_NAME}_FIND_COMPONENTS}) + if(NOT ${_NAME}_${comp}_FOUND) + if(${_NAME}_FIND_REQUIRED_${comp}) + set(${_NAME}_FOUND FALSE) + endif() + endif() + endforeach() +endmacro() + +#################################################################################### + +include(${CMAKE_CURRENT_LIST_DIR}/OpenCCTargets.cmake) + +set_and_check(OPENCC_INCLUDE_DIR ${PACKAGE_PREFIX_DIR}/include/opencc) + +check_required_components(OpenCC) diff --git a/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfigVersion.cmake b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfigVersion.cmake new file mode 100644 index 00000000..5b32bd03 --- /dev/null +++ b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCConfigVersion.cmake @@ -0,0 +1,70 @@ +# This is a basic version file for the Config-mode of find_package(). +# It is used by write_basic_package_version_file() as input file for configure_file() +# to create a version-file which can be installed along a config.cmake file. +# +# The created file sets PACKAGE_VERSION_EXACT if the current version string and +# the requested version string are exactly the same and it sets +# PACKAGE_VERSION_COMPATIBLE if the current version is >= requested version, +# but only if the requested major version is the same as the current one. +# The variable CVF_VERSION must be set before calling configure_file(). + + +set(PACKAGE_VERSION "1.1.6") + +if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + + if("1.1.6" MATCHES "^([0-9]+)\\.") + set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}") + if(NOT CVF_VERSION_MAJOR VERSION_EQUAL 0) + string(REGEX REPLACE "^0+" "" CVF_VERSION_MAJOR "${CVF_VERSION_MAJOR}") + endif() + else() + set(CVF_VERSION_MAJOR "1.1.6") + endif() + + if(PACKAGE_FIND_VERSION_RANGE) + # both endpoints of the range must have the expected major version + math (EXPR CVF_VERSION_MAJOR_NEXT "${CVF_VERSION_MAJOR} + 1") + if (NOT PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + OR ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX_MAJOR STREQUAL CVF_VERSION_MAJOR) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX VERSION_LESS_EQUAL CVF_VERSION_MAJOR_NEXT))) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + elseif(PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + AND ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND PACKAGE_VERSION VERSION_LESS_EQUAL PACKAGE_FIND_VERSION_MAX) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MAX))) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + else() + if(PACKAGE_FIND_VERSION_MAJOR STREQUAL CVF_VERSION_MAJOR) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + + if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + endif() + endif() +endif() + + +# if the installed project requested no architecture check, don't perform the check +if("FALSE") + return() +endif() + +# if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it: +if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "" OR "4" STREQUAL "") + return() +endif() + +# check that the installed version has the same 32/64bit-ness as the one which is currently searching: +if(NOT CMAKE_SIZEOF_VOID_P STREQUAL "4") + math(EXPR installedBits "4 * 8") + set(PACKAGE_VERSION "${PACKAGE_VERSION} (${installedBits}bit)") + set(PACKAGE_VERSION_UNSUITABLE TRUE) +endif() diff --git a/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets-release.cmake b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets-release.cmake new file mode 100644 index 00000000..199b87f5 --- /dev/null +++ b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets-release.cmake @@ -0,0 +1,29 @@ +#---------------------------------------------------------------- +# Generated CMake target import file for configuration "Release". +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Import target "OpenCC::OpenCC" for configuration "Release" +set_property(TARGET OpenCC::OpenCC APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::OpenCC PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libopencc.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::OpenCC ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::OpenCC "${_IMPORT_PREFIX}/lib/libopencc.a" ) + +# Import target "OpenCC::marisa" for configuration "Release" +set_property(TARGET OpenCC::marisa APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::marisa PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libmarisa.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::marisa ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::marisa "${_IMPORT_PREFIX}/lib/libmarisa.a" ) + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) diff --git a/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets.cmake b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets.cmake new file mode 100644 index 00000000..db461d58 --- /dev/null +++ b/opencc/armeabi-v7a/lib/cmake/opencc/OpenCCTargets.cmake @@ -0,0 +1,102 @@ +# Generated by CMake + +if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS 2.6) + message(FATAL_ERROR "CMake >= 2.6.0 required") +endif() +cmake_policy(PUSH) +cmake_policy(VERSION 2.6...3.20) +#---------------------------------------------------------------- +# Generated CMake target import file. +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Protect against multiple inclusion, which would fail when already imported targets are added once more. +set(_targetsDefined) +set(_targetsNotDefined) +set(_expectedTargets) +foreach(_expectedTarget OpenCC::OpenCC OpenCC::marisa) + list(APPEND _expectedTargets ${_expectedTarget}) + if(NOT TARGET ${_expectedTarget}) + list(APPEND _targetsNotDefined ${_expectedTarget}) + endif() + if(TARGET ${_expectedTarget}) + list(APPEND _targetsDefined ${_expectedTarget}) + endif() +endforeach() +if("${_targetsDefined}" STREQUAL "${_expectedTargets}") + unset(_targetsDefined) + unset(_targetsNotDefined) + unset(_expectedTargets) + set(CMAKE_IMPORT_FILE_VERSION) + cmake_policy(POP) + return() +endif() +if(NOT "${_targetsDefined}" STREQUAL "") + message(FATAL_ERROR "Some (but not all) targets in this export set were already defined.\nTargets Defined: ${_targetsDefined}\nTargets not yet defined: ${_targetsNotDefined}\n") +endif() +unset(_targetsDefined) +unset(_targetsNotDefined) +unset(_expectedTargets) + + +# Compute the installation prefix relative to this file. +get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +if(_IMPORT_PREFIX STREQUAL "/") + set(_IMPORT_PREFIX "") +endif() + +# Create imported target OpenCC::OpenCC +add_library(OpenCC::OpenCC STATIC IMPORTED) + +set_target_properties(OpenCC::OpenCC PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include/opencc" + INTERFACE_LINK_LIBRARIES "OpenCC::marisa" +) + +# Create imported target OpenCC::marisa +add_library(OpenCC::marisa STATIC IMPORTED) + +if(CMAKE_VERSION VERSION_LESS 2.8.12) + message(FATAL_ERROR "This file relies on consumers using CMake 2.8.12 or greater.") +endif() + +# Load information for each installed configuration. +get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +file(GLOB CONFIG_FILES "${_DIR}/OpenCCTargets-*.cmake") +foreach(f ${CONFIG_FILES}) + include(${f}) +endforeach() + +# Cleanup temporary variables. +set(_IMPORT_PREFIX) + +# Loop over all imported files and verify that they actually exist +foreach(target ${_IMPORT_CHECK_TARGETS} ) + foreach(file ${_IMPORT_CHECK_FILES_FOR_${target}} ) + if(NOT EXISTS "${file}" ) + message(FATAL_ERROR "The imported target \"${target}\" references the file + \"${file}\" +but this file does not exist. Possible reasons include: +* The file was deleted, renamed, or moved to another location. +* An install or uninstall procedure did not complete successfully. +* The installation package was faulty and contained + \"${CMAKE_CURRENT_LIST_FILE}\" +but not all the files it references. +") + endif() + endforeach() + unset(_IMPORT_CHECK_FILES_FOR_${target}) +endforeach() +unset(_IMPORT_CHECK_TARGETS) + +# This file does not depend on other imported targets which have +# been exported from the same project but in a separate export set. + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) +cmake_policy(POP) diff --git a/opencc/armeabi-v7a/lib/libmarisa.a b/opencc/armeabi-v7a/lib/libmarisa.a new file mode 100644 index 00000000..8b1725e4 Binary files /dev/null and b/opencc/armeabi-v7a/lib/libmarisa.a differ diff --git a/opencc/armeabi-v7a/lib/libopencc.a b/opencc/armeabi-v7a/lib/libopencc.a new file mode 100644 index 00000000..dc7af3a1 Binary files /dev/null and b/opencc/armeabi-v7a/lib/libopencc.a differ diff --git a/opencc/armeabi-v7a/share/opencc b/opencc/armeabi-v7a/share/opencc new file mode 120000 index 00000000..e67b4559 --- /dev/null +++ b/opencc/armeabi-v7a/share/opencc @@ -0,0 +1 @@ +../../data \ No newline at end of file diff --git a/opencc/data/HKVariants.ocd2 b/opencc/data/HKVariants.ocd2 new file mode 100644 index 00000000..fa2edde1 Binary files /dev/null and b/opencc/data/HKVariants.ocd2 differ diff --git a/opencc/data/HKVariantsRev.ocd2 b/opencc/data/HKVariantsRev.ocd2 new file mode 100644 index 00000000..abc97a94 Binary files /dev/null and b/opencc/data/HKVariantsRev.ocd2 differ diff --git a/opencc/data/HKVariantsRevPhrases.ocd2 b/opencc/data/HKVariantsRevPhrases.ocd2 new file mode 100644 index 00000000..84822772 Binary files /dev/null and b/opencc/data/HKVariantsRevPhrases.ocd2 differ diff --git a/opencc/data/JPShinjitaiCharacters.ocd2 b/opencc/data/JPShinjitaiCharacters.ocd2 new file mode 100644 index 00000000..d2780147 Binary files /dev/null and b/opencc/data/JPShinjitaiCharacters.ocd2 differ diff --git a/opencc/data/JPShinjitaiPhrases.ocd2 b/opencc/data/JPShinjitaiPhrases.ocd2 new file mode 100644 index 00000000..e4c823f1 Binary files /dev/null and b/opencc/data/JPShinjitaiPhrases.ocd2 differ diff --git a/opencc/data/JPVariants.ocd2 b/opencc/data/JPVariants.ocd2 new file mode 100644 index 00000000..18587b01 Binary files /dev/null and b/opencc/data/JPVariants.ocd2 differ diff --git a/opencc/data/JPVariantsRev.ocd2 b/opencc/data/JPVariantsRev.ocd2 new file mode 100644 index 00000000..8abee318 Binary files /dev/null and b/opencc/data/JPVariantsRev.ocd2 differ diff --git a/opencc/data/STCharacters.ocd2 b/opencc/data/STCharacters.ocd2 new file mode 100644 index 00000000..74f7fbe2 Binary files /dev/null and b/opencc/data/STCharacters.ocd2 differ diff --git a/opencc/data/STPhrases.ocd2 b/opencc/data/STPhrases.ocd2 new file mode 100644 index 00000000..13cd9be0 Binary files /dev/null and b/opencc/data/STPhrases.ocd2 differ diff --git a/opencc/data/TSCharacters.ocd2 b/opencc/data/TSCharacters.ocd2 new file mode 100644 index 00000000..3c0d60a9 Binary files /dev/null and b/opencc/data/TSCharacters.ocd2 differ diff --git a/opencc/data/TSPhrases.ocd2 b/opencc/data/TSPhrases.ocd2 new file mode 100644 index 00000000..dc8ef5d7 Binary files /dev/null and b/opencc/data/TSPhrases.ocd2 differ diff --git a/opencc/data/TWPhrases.ocd2 b/opencc/data/TWPhrases.ocd2 new file mode 100644 index 00000000..9107d1ab Binary files /dev/null and b/opencc/data/TWPhrases.ocd2 differ diff --git a/opencc/data/TWPhrasesRev.ocd2 b/opencc/data/TWPhrasesRev.ocd2 new file mode 100644 index 00000000..acc8662a Binary files /dev/null and b/opencc/data/TWPhrasesRev.ocd2 differ diff --git a/opencc/data/TWVariants.ocd2 b/opencc/data/TWVariants.ocd2 new file mode 100644 index 00000000..8d743955 Binary files /dev/null and b/opencc/data/TWVariants.ocd2 differ diff --git a/opencc/data/TWVariantsRev.ocd2 b/opencc/data/TWVariantsRev.ocd2 new file mode 100644 index 00000000..2546811c Binary files /dev/null and b/opencc/data/TWVariantsRev.ocd2 differ diff --git a/opencc/data/TWVariantsRevPhrases.ocd2 b/opencc/data/TWVariantsRevPhrases.ocd2 new file mode 100644 index 00000000..95f26afc Binary files /dev/null and b/opencc/data/TWVariantsRevPhrases.ocd2 differ diff --git a/opencc/data/hk2s.json b/opencc/data/hk2s.json new file mode 100644 index 00000000..cf0e9b97 --- /dev/null +++ b/opencc/data/hk2s.json @@ -0,0 +1,33 @@ +{ + "name": "Traditional Chinese (Hong Kong variant) to Simplified Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TSPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "HKVariantsRevPhrases.ocd2" + }, { + "type": "ocd2", + "file": "HKVariantsRev.ocd2" + }] + } + }, { + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TSPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TSCharacters.ocd2" + }] + } + }] +} diff --git a/opencc/data/hk2t.json b/opencc/data/hk2t.json new file mode 100644 index 00000000..0d47b917 --- /dev/null +++ b/opencc/data/hk2t.json @@ -0,0 +1,22 @@ +{ + "name": "Traditional Chinese (Hong Kong variant) to Traditional Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "HKVariantsRevPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "HKVariantsRevPhrases.ocd2" + }, { + "type": "ocd2", + "file": "HKVariantsRev.ocd2" + }] + } + }] +} diff --git a/opencc/data/jp2t.json b/opencc/data/jp2t.json new file mode 100644 index 00000000..025d8919 --- /dev/null +++ b/opencc/data/jp2t.json @@ -0,0 +1,25 @@ +{ + "name": "New Japanese Kanji (Shinjitai) to Traditional Chinese Characters (Kyūjitai)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "JPShinjitaiPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "JPShinjitaiPhrases.ocd2" + }, { + "type": "ocd2", + "file": "JPShinjitaiCharacters.ocd2" + }, { + "type": "ocd2", + "file": "JPVariantsRev.ocd2" + }] + } + }] +} diff --git a/opencc/data/s2hk.json b/opencc/data/s2hk.json new file mode 100644 index 00000000..fcaa017e --- /dev/null +++ b/opencc/data/s2hk.json @@ -0,0 +1,27 @@ +{ + "name": "Simplified Chinese to Traditional Chinese (Hong Kong variant)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "STPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "STPhrases.ocd2" + }, { + "type": "ocd2", + "file": "STCharacters.ocd2" + }] + } + }, { + "dict": { + "type": "ocd2", + "file": "HKVariants.ocd2" + } + }] +} diff --git a/opencc/data/s2t.json b/opencc/data/s2t.json new file mode 100644 index 00000000..87516acb --- /dev/null +++ b/opencc/data/s2t.json @@ -0,0 +1,22 @@ +{ + "name": "Simplified Chinese to Traditional Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "STPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "STPhrases.ocd2" + }, { + "type": "ocd2", + "file": "STCharacters.ocd2" + }] + } + }] +} diff --git a/opencc/data/s2tw.json b/opencc/data/s2tw.json new file mode 100644 index 00000000..2a3d7656 --- /dev/null +++ b/opencc/data/s2tw.json @@ -0,0 +1,27 @@ +{ + "name": "Simplified Chinese to Traditional Chinese (Taiwan standard)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "STPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "STPhrases.ocd2" + }, { + "type": "ocd2", + "file": "STCharacters.ocd2" + }] + } + }, { + "dict": { + "type": "ocd2", + "file": "TWVariants.ocd2" + } + }] +} diff --git a/opencc/data/s2twp.json b/opencc/data/s2twp.json new file mode 100644 index 00000000..2f36e935 --- /dev/null +++ b/opencc/data/s2twp.json @@ -0,0 +1,32 @@ +{ + "name": "Simplified Chinese to Traditional Chinese (Taiwan standard, with phrases)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "STPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "STPhrases.ocd2" + }, { + "type": "ocd2", + "file": "STCharacters.ocd2" + }] + } + }, { + "dict": { + "type": "ocd2", + "file": "TWPhrases.ocd2" + } + }, { + "dict": { + "type": "ocd2", + "file": "TWVariants.ocd2" + } + }] +} diff --git a/opencc/data/t2hk.json b/opencc/data/t2hk.json new file mode 100644 index 00000000..519d4a3f --- /dev/null +++ b/opencc/data/t2hk.json @@ -0,0 +1,16 @@ +{ + "name": "Traditional Chinese to Traditional Chinese (Hong Kong variant)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "HKVariants.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "ocd2", + "file": "HKVariants.ocd2" + } + }] +} diff --git a/opencc/data/t2jp.json b/opencc/data/t2jp.json new file mode 100644 index 00000000..7a43217f --- /dev/null +++ b/opencc/data/t2jp.json @@ -0,0 +1,16 @@ +{ + "name": "Traditional Chinese Characters (Kyūjitai) to New Japanese Kanji (Shinjitai)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "JPVariants.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "ocd2", + "file": "JPVariants.ocd2" + } + }] +} diff --git a/opencc/data/t2s.json b/opencc/data/t2s.json new file mode 100644 index 00000000..06cf5f58 --- /dev/null +++ b/opencc/data/t2s.json @@ -0,0 +1,22 @@ +{ + "name": "Traditional Chinese to Simplified Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TSPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TSPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TSCharacters.ocd2" + }] + } + }] +} diff --git a/opencc/data/t2tw.json b/opencc/data/t2tw.json new file mode 100644 index 00000000..0394f600 --- /dev/null +++ b/opencc/data/t2tw.json @@ -0,0 +1,16 @@ +{ + "name": "Traditional Chinese to Traditional Chinese (Taiwan standard)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TWVariants.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "ocd2", + "file": "TWVariants.ocd2" + } + }] +} diff --git a/opencc/data/tw2s.json b/opencc/data/tw2s.json new file mode 100644 index 00000000..4f554393 --- /dev/null +++ b/opencc/data/tw2s.json @@ -0,0 +1,33 @@ +{ + "name": "Traditional Chinese (Taiwan standard) to Simplified Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TSPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TWVariantsRevPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TWVariantsRev.ocd2" + }] + } + }, { + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TSPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TSCharacters.ocd2" + }] + } + }] +} diff --git a/opencc/data/tw2sp.json b/opencc/data/tw2sp.json new file mode 100644 index 00000000..64eb9d97 --- /dev/null +++ b/opencc/data/tw2sp.json @@ -0,0 +1,36 @@ +{ + "name": "Traditional Chinese (Taiwan standard) to Simplified Chinese (with phrases)", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TSPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TWPhrasesRev.ocd2" + }, { + "type": "ocd2", + "file": "TWVariantsRevPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TWVariantsRev.ocd2" + }] + } + }, { + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TSPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TSCharacters.ocd2" + }] + } + }] +} diff --git a/opencc/data/tw2t.json b/opencc/data/tw2t.json new file mode 100644 index 00000000..ad5295b6 --- /dev/null +++ b/opencc/data/tw2t.json @@ -0,0 +1,22 @@ +{ + "name": "Traditional Chinese (Taiwan standard) to Traditional Chinese", + "segmentation": { + "type": "mmseg", + "dict": { + "type": "ocd2", + "file": "TWVariantsRevPhrases.ocd2" + } + }, + "conversion_chain": [{ + "dict": { + "type": "group", + "dicts": [{ + "type": "ocd2", + "file": "TWVariantsRevPhrases.ocd2" + }, { + "type": "ocd2", + "file": "TWVariantsRev.ocd2" + }] + } + }] +} diff --git a/opencc/x86/include/opencc/Common.hpp b/opencc/x86/include/opencc/Common.hpp new file mode 100644 index 00000000..067966b5 --- /dev/null +++ b/opencc/x86/include/opencc/Common.hpp @@ -0,0 +1,82 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Microsoft Visual C++ specific +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820) +#endif + +#include +#include +#include +#include + +#include "Export.hpp" +#include "Optional.hpp" +#include "opencc_config.h" + +// Forward decalarations and alias +namespace opencc { +class Config; +class Conversion; +class ConversionChain; +class Converter; +class Dict; +class DictEntry; +class DictGroup; +class Lexicon; +class MarisaDict; +class MultiValueDictEntry; +class NoValueDictEntry; +class Segmentation; +class Segments; +class SerializableDict; +class SingleValueDictEntry; +class TextDict; +typedef std::shared_ptr ConversionPtr; +typedef std::shared_ptr ConversionChainPtr; +typedef std::shared_ptr ConverterPtr; +typedef std::shared_ptr DictPtr; +typedef std::shared_ptr DictGroupPtr; +typedef std::shared_ptr LexiconPtr; +typedef std::shared_ptr MarisaDictPtr; +typedef std::shared_ptr SegmentationPtr; +typedef std::shared_ptr SegmentsPtr; +typedef std::shared_ptr SerializableDictPtr; +typedef std::shared_ptr TextDictPtr; + +#ifdef OPENCC_ENABLE_DARTS +class BinaryDict; +class DartsDict; +typedef std::shared_ptr BinaryDictPtr; +typedef std::shared_ptr DartsDictPtr; +#endif + +} // namespace opencc + +#ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = ""; +#else // ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/"; +#endif // ifndef PKGDATADIR + +#ifndef VERSION +#define VERSION "1.0.*" +#endif // ifndef VERSION diff --git a/opencc/x86/include/opencc/Config.hpp b/opencc/x86/include/opencc/Config.hpp new file mode 100644 index 00000000..7a904cee --- /dev/null +++ b/opencc/x86/include/opencc/Config.hpp @@ -0,0 +1,42 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Configuration loader + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Config { +public: + Config(); + + virtual ~Config(); + + ConverterPtr NewFromString(const std::string& json, + const std::string& configDirectory); + + ConverterPtr NewFromFile(const std::string& fileName); + +private: + void* internal; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Conversion.hpp b/opencc/x86/include/opencc/Conversion.hpp new file mode 100644 index 00000000..cf73a254 --- /dev/null +++ b/opencc/x86/include/opencc/Conversion.hpp @@ -0,0 +1,47 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Conversion interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Conversion { +public: + Conversion(DictPtr _dict) : dict(_dict) {} + + // Convert single phrase + std::string Convert(const std::string& phrase) const; + + // Convert single phrase + std::string Convert(const char* phrase) const; + + // Convert segmented text + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/ConversionChain.hpp b/opencc/x86/include/opencc/ConversionChain.hpp new file mode 100644 index 00000000..28853c67 --- /dev/null +++ b/opencc/x86/include/opencc/ConversionChain.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Conversion.hpp" + +namespace opencc { +/** + * Chain of conversions + * Consists of a list of conversions. Converts input in sequence. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT ConversionChain { +public: + ConversionChain(const std::list _conversions); + + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const std::list GetConversions() const { return conversions; } + +private: + const std::list conversions; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Converter.hpp b/opencc/x86/include/opencc/Converter.hpp new file mode 100644 index 00000000..9f1f3f94 --- /dev/null +++ b/opencc/x86/include/opencc/Converter.hpp @@ -0,0 +1,51 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Controller of segmentation and conversion + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Converter { +public: + Converter(const std::string& _name, SegmentationPtr _segmentation, + ConversionChainPtr _conversionChain) + : name(_name), segmentation(_segmentation), + conversionChain(_conversionChain) {} + + std::string Convert(const std::string& text) const; + + size_t Convert(const char* input, char* output) const; + + const SegmentationPtr GetSegmentation() const { return segmentation; } + + const ConversionChainPtr GetConversionChain() const { + return conversionChain; + } + +private: + const std::string name; + const SegmentationPtr segmentation; + const ConversionChainPtr conversionChain; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Dict.hpp b/opencc/x86/include/opencc/Dict.hpp new file mode 100644 index 00000000..1c810346 --- /dev/null +++ b/opencc/x86/include/opencc/Dict.hpp @@ -0,0 +1,92 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Abstract class of dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Dict { +public: + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + virtual Optional Match(const char* word, + size_t len) const = 0; + + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + Optional Match(const std::string& word) const { + return Match(word.c_str(), word.length()); + } + + /** + * Matches the longest matched prefix of a word. + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * the longest prefix of "banana" matched is "bana". + */ + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const char* word) const { + return MatchPrefix(word, KeyMaxLength()); + } + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const std::string& word) const { + return MatchPrefix(word.c_str(), word.length()); + } + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * all the matched prefixes of "banana" are "bana", "ban", "ba", "b". + */ + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + */ + std::vector + MatchAllPrefixes(const std::string& word) const { + return MatchAllPrefixes(word.c_str(), word.length()); + } + + /** + * Returns the length of the longest key in the dictionary. + */ + virtual size_t KeyMaxLength() const = 0; + + /** + * Returns all entries in the dictionary. + */ + virtual LexiconPtr GetLexicon() const = 0; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/DictConverter.hpp b/opencc/x86/include/opencc/DictConverter.hpp new file mode 100644 index 00000000..f911c4fe --- /dev/null +++ b/opencc/x86/include/opencc/DictConverter.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2017 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Converts a dictionary from a format to another. + * @ingroup opencc_cpp_api + */ +OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName, + const std::string& outputFileName, + const std::string& formatFrom, + const std::string& formatTo); +} // namespace opencc diff --git a/opencc/x86/include/opencc/DictEntry.hpp b/opencc/x86/include/opencc/DictEntry.hpp new file mode 100644 index 00000000..7b2babda --- /dev/null +++ b/opencc/x86/include/opencc/DictEntry.hpp @@ -0,0 +1,173 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segments.hpp" +#include "UTF8Util.hpp" + +namespace opencc { +/** + * Key-values pair entry + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictEntry { +public: + virtual ~DictEntry() {} + + virtual std::string Key() const = 0; + + virtual std::vector Values() const = 0; + + virtual std::string GetDefault() const = 0; + + virtual size_t NumValues() const = 0; + + virtual std::string ToString() const = 0; + + size_t KeyLength() const { return Key().length(); } + + bool operator<(const DictEntry& that) const { return Key() < that.Key(); } + + bool operator==(const DictEntry& that) const { return Key() == that.Key(); } + + static bool UPtrLessThan(const std::unique_ptr& a, + const std::unique_ptr& b) { + return *a < *b; + } +}; + +class OPENCC_EXPORT NoValueDictEntry : public DictEntry { +public: + NoValueDictEntry(const std::string& _key) : key(_key) {} + + virtual ~NoValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::vector Values() const { + return std::vector(); + } + + virtual std::string GetDefault() const { return key; } + + virtual size_t NumValues() const { return 0; } + + virtual std::string ToString() const { return key; } + +private: + std::string key; +}; + +class OPENCC_EXPORT SingleValueDictEntry : public DictEntry { +public: + virtual std::string Value() const = 0; + + virtual std::vector Values() const { + return std::vector{Value()}; + } + + virtual std::string GetDefault() const { return Value(); } + + virtual size_t NumValues() const { return 1; } + + virtual std::string ToString() const { + return std::string(Key()) + "\t" + Value(); + } +}; + +class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry { +public: + StrSingleValueDictEntry(const std::string& _key, const std::string& _value) + : key(_key), value(_value) {} + + virtual ~StrSingleValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::string Value() const { return value; } + +private: + std::string key; + std::string value; +}; + +class OPENCC_EXPORT MultiValueDictEntry : public DictEntry { +public: + virtual std::string GetDefault() const { + if (NumValues() > 0) { + return Values().at(0); + } else { + return Key(); + } + } + + virtual std::string ToString() const; +}; + +class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry { +public: + StrMultiValueDictEntry(const std::string& _key, + const std::vector& _values) + : key(_key), values(_values) {} + + virtual ~StrMultiValueDictEntry() {} + + virtual std::string Key() const { return key; } + + size_t NumValues() const { return values.size(); } + + std::vector Values() const { return values; } + +private: + std::string key; + std::vector values; +}; + +class OPENCC_EXPORT DictEntryFactory { +public: + static DictEntry* New(const std::string& key) { + return new NoValueDictEntry(key); + } + + static DictEntry* New(const std::string& key, const std::string& value) { + return new StrSingleValueDictEntry(key, value); + } + + static DictEntry* New(const std::string& key, + const std::vector& values) { + if (values.size() == 0) { + return New(key); + } else if (values.size() == 1) { + return New(key, values.front()); + } + return new StrMultiValueDictEntry(key, values); + } + + static DictEntry* New(const DictEntry* entry) { + if (entry->NumValues() == 0) { + return new NoValueDictEntry(entry->Key()); + } else if (entry->NumValues() == 1) { + return new StrSingleValueDictEntry(entry->Key(), entry->Values().front()); + } else { + return new StrMultiValueDictEntry(entry->Key(), entry->Values()); + } + } +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/DictGroup.hpp b/opencc/x86/include/opencc/DictGroup.hpp new file mode 100644 index 00000000..fd51d913 --- /dev/null +++ b/opencc/x86/include/opencc/DictGroup.hpp @@ -0,0 +1,57 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Dict.hpp" + +namespace opencc { +/** + * Group of dictionaries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictGroup : public Dict { +public: + DictGroup(const std::list& dicts); + + static DictGroupPtr NewFromDict(const Dict& dict); + + virtual ~DictGroup(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + const std::list GetDicts() const { return dicts; } + +private: + const size_t keyMaxLength; + const std::list dicts; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Exception.hpp b/opencc/x86/include/opencc/Exception.hpp new file mode 100644 index 00000000..7875537f --- /dev/null +++ b/opencc/x86/include/opencc/Exception.hpp @@ -0,0 +1,88 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Export.hpp" + +#if defined(_MSC_VER) && _MSC_VER < 1900 +// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not +// supported +#define noexcept +#endif // ifdef _MSC_VER + +namespace opencc { + +class OPENCC_EXPORT Exception { +public: + Exception() {} + + virtual ~Exception() throw() {} + + Exception(const std::string& _message) : message(_message) {} + + virtual const char* what() const noexcept { return message.c_str(); } + +protected: + std::string message; +}; + +class OPENCC_EXPORT FileNotFound : public Exception { +public: + FileNotFound(const std::string& fileName) + : Exception(fileName + " not found or not accessible.") {} +}; + +class OPENCC_EXPORT FileNotWritable : public Exception { +public: + FileNotWritable(const std::string& fileName) + : Exception(fileName + " not writable.") {} +}; + +class OPENCC_EXPORT InvalidFormat : public Exception { +public: + InvalidFormat(const std::string& message) + : Exception("Invalid format: " + message) {} +}; + +class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat { +public: + InvalidTextDictionary(const std::string& _message, size_t lineNum) + : InvalidFormat("") { + std::ostringstream buffer; + buffer << "Invalid text dictionary at line " << lineNum << ": " << _message; + message = buffer.str(); + } +}; + +class OPENCC_EXPORT InvalidUTF8 : public Exception { +public: + InvalidUTF8(const std::string& _message) + : Exception("Invalid UTF8: " + _message) {} +}; + +class OPENCC_EXPORT ShouldNotBeHere : public Exception { +public: + ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {} +}; + +} // namespace opencc diff --git a/opencc/x86/include/opencc/Export.hpp b/opencc/x86/include/opencc/Export.hpp new file mode 100644 index 00000000..0015e880 --- /dev/null +++ b/opencc/x86/include/opencc/Export.hpp @@ -0,0 +1,40 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#define OPENCC_EXPORT +#define OPENCC_NO_EXPORT +#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#ifndef OPENCC_EXPORT +#ifdef libopencc_EXPORTS + +/* We are building this library */ +#define OPENCC_EXPORT __declspec(dllexport) +#else // ifdef libopencc_EXPORTS + +/* We are using this library */ +#define OPENCC_EXPORT __declspec(dllimport) +#endif // ifdef libopencc_EXPORTS +#endif // ifndef OPENCC_EXPORT + +#ifndef OPENCC_NO_EXPORT +#define OPENCC_NO_EXPORT +#endif // ifndef OPENCC_NO_EXPORT +#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) diff --git a/opencc/x86/include/opencc/Lexicon.hpp b/opencc/x86/include/opencc/Lexicon.hpp new file mode 100644 index 00000000..68891287 --- /dev/null +++ b/opencc/x86/include/opencc/Lexicon.hpp @@ -0,0 +1,68 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Storage of all entries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Lexicon { +public: + Lexicon() {} + Lexicon(std::vector> entries_) + : entries(std::move(entries_)) {} + Lexicon(const Lexicon&) = delete; + Lexicon& operator=(const Lexicon&) = delete; + + // Lexicon will take the ownership of the entry. + void Add(DictEntry* entry) { entries.emplace_back(entry); } + + void Add(std::unique_ptr entry) { + entries.push_back(std::move(entry)); + } + + void Sort(); + + // Returns true if the lexicon is sorted by key. + bool IsSorted(); + + // Returns true if every key unique (after sorted). + // When dupkey is set, it is set to the duplicate key. + bool IsUnique(std::string* dupkey = nullptr); + + const DictEntry* At(size_t index) const { return entries.at(index).get(); } + + size_t Length() const { return entries.size(); } + + std::vector>::const_iterator begin() const { + return entries.begin(); + } + + std::vector>::const_iterator end() const { + return entries.end(); + } + +private: + std::vector> entries; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/MarisaDict.hpp b/opencc/x86/include/opencc/MarisaDict.hpp new file mode 100644 index 00000000..6917c23a --- /dev/null +++ b/opencc/x86/include/opencc/MarisaDict.hpp @@ -0,0 +1,63 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Darts dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict { +public: + virtual ~MarisaDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a MarisaDict from another dictionary. + */ + static MarisaDictPtr NewFromDict(const Dict& thatDict); + + static MarisaDictPtr NewFromFile(FILE* fp); + +private: + MarisaDict(); + + size_t maxLength; + LexiconPtr lexicon; + + class MarisaInternal; + std::unique_ptr internal; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/MaxMatchSegmentation.hpp b/opencc/x86/include/opencc/MaxMatchSegmentation.hpp new file mode 100644 index 00000000..1ecc227e --- /dev/null +++ b/opencc/x86/include/opencc/MaxMatchSegmentation.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictGroup.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Implementation of maximal match segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation { +public: + MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {} + + virtual ~MaxMatchSegmentation() {} + + virtual SegmentsPtr Segment(const std::string& text) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Optional.hpp b/opencc/x86/include/opencc/Optional.hpp new file mode 100644 index 00000000..8a510f43 --- /dev/null +++ b/opencc/x86/include/opencc/Optional.hpp @@ -0,0 +1,76 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace opencc { +/** + * A class that wraps type T into a nullable type. + * @ingroup opencc_cpp_api + */ +template class Optional { +public: + /** + * The constructor of Optional. + */ + Optional(T actual) : isNull(false), data(actual) {} + + /** + * Returns true if the instance is null. + */ + bool IsNull() const { return isNull; } + + /** + * Returns the containing data of the instance. + */ + const T& Get() const { return data; } + + /** + * Constructs a null instance. + */ + static Optional Null() { return Optional(); } + +private: + Optional() : isNull(true) {} + + bool isNull; + T data; +}; + +/** + * Specialization of Optional for pointers. + * + * Reduce a bool. + */ +template class Optional { +private: + Optional() : data(nullptr) {} + + typedef T* TPtr; + TPtr data; + +public: + Optional(TPtr actual) : data(actual) {} + + bool IsNull() const { return data == nullptr; } + + const TPtr& Get() const { return data; } + + static Optional Null() { return Optional(); } +}; +} // namespace opencc \ No newline at end of file diff --git a/opencc/x86/include/opencc/PhraseExtract.hpp b/opencc/x86/include/opencc/PhraseExtract.hpp new file mode 100644 index 00000000..75a165de --- /dev/null +++ b/opencc/x86/include/opencc/PhraseExtract.hpp @@ -0,0 +1,195 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Common.hpp" +#include "UTF8StringSlice.hpp" + +namespace opencc { + +class OPENCC_EXPORT PhraseExtract { +public: + typedef UTF8StringSlice::LengthType LengthType; + + typedef UTF8StringSliceBase UTF8StringSlice8Bit; + + PhraseExtract(); + + virtual ~PhraseExtract(); + + void Extract(const std::string& text) { + SetFullText(text); + ExtractSuffixes(); + CalculateFrequency(); + CalculateSuffixEntropy(); + ReleaseSuffixes(); + ExtractPrefixes(); + CalculatePrefixEntropy(); + ReleasePrefixes(); + ExtractWordCandidates(); + CalculateCohesions(); + SelectWords(); + } + + void SetFullText(const std::string& fullText) { + utf8FullText = UTF8StringSlice(fullText.c_str()); + } + + void SetFullText(const char* fullText) { + utf8FullText = UTF8StringSlice(fullText); + } + + void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; } + + void SetWordMinLength(const LengthType _wordMinLength) { + wordMinLength = _wordMinLength; + } + + void SetWordMaxLength(const LengthType _wordMaxLength) { + wordMaxLength = _wordMaxLength; + } + + void SetPrefixSetLength(const LengthType _prefixSetLength) { + prefixSetLength = _prefixSetLength; + } + + void SetSuffixSetLength(const LengthType _suffixSetLength) { + suffixSetLength = _suffixSetLength; + } + + // PreCalculationFilter is called after frequencies statistics. + void SetPreCalculationFilter( + const std::function& filter) { + preCalculationFilter = filter; + } + + void SetPostCalculationFilter( + const std::function& filter) { + postCalculationFilter = filter; + } + + void ReleaseSuffixes() { std::vector().swap(suffixes); } + + void ReleasePrefixes() { std::vector().swap(prefixes); } + + const std::vector& Words() const { return words; } + + const std::vector& WordCandidates() const { + return wordCandidates; + } + + struct Signals { + size_t frequency; + double cohesion; + double suffixEntropy; + double prefixEntropy; + }; + + const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const; + + double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double Entropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + size_t Frequency(const UTF8StringSlice8Bit& word) const; + + double Probability(const UTF8StringSlice8Bit& word) const; + + double LogProbability(const UTF8StringSlice8Bit& word) const; + + void Reset(); + + void ExtractSuffixes(); + + void ExtractPrefixes(); + + void ExtractWordCandidates(); + + void CalculateFrequency(); + + void CalculateCohesions(); + + void CalculateSuffixEntropy(); + + void CalculatePrefixEntropy(); + + void SelectWords(); + + static bool + DefaultPreCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + + static bool + DefaultPostCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + +private: + class DictType; + + // Pointwise Mutual Information + double PMI(const UTF8StringSlice8Bit& wordCandidate, + const UTF8StringSlice8Bit& part1, + const UTF8StringSlice8Bit& part2) const; + + double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double CalculateEntropy( + const std::unordered_map& choices) const; + + LengthType wordMinLength; + LengthType wordMaxLength; + LengthType prefixSetLength; + LengthType suffixSetLength; + std::function + preCalculationFilter; + std::function + postCalculationFilter; + + bool prefixesExtracted; + bool suffixesExtracted; + bool frequenciesCalculated; + bool wordCandidatesExtracted; + bool cohesionsCalculated; + bool prefixEntropiesCalculated; + bool suffixEntropiesCalculated; + bool wordsSelected; + + UTF8StringSlice utf8FullText; + size_t totalOccurrence; + double logTotalOccurrence; + std::vector prefixes; + std::vector suffixes; + std::vector wordCandidates; + std::vector words; + DictType* signals; + + friend class PhraseExtractTest; +}; + +} // namespace opencc diff --git a/opencc/x86/include/opencc/Segmentation.hpp b/opencc/x86/include/opencc/Segmentation.hpp new file mode 100644 index 00000000..c260ea6d --- /dev/null +++ b/opencc/x86/include/opencc/Segmentation.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Abstract segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segmentation { +public: + virtual SegmentsPtr Segment(const std::string& text) const = 0; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/Segments.hpp b/opencc/x86/include/opencc/Segments.hpp new file mode 100644 index 00000000..21c39813 --- /dev/null +++ b/opencc/x86/include/opencc/Segments.hpp @@ -0,0 +1,114 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" + +namespace opencc { +/** + * Segmented text + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segments { +public: + Segments() {} + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + void AddSegment(const char* unmanagedString) { + indexes.push_back(std::make_pair(unmanaged.size(), false)); + unmanaged.push_back(unmanagedString); + } + + void AddSegment(const std::string& str) { + indexes.push_back(std::make_pair(managed.size(), true)); + managed.push_back(str); + } + + class iterator : public std::iterator { + public: + iterator(const Segments* const _segments, size_t _cursor) + : segments(_segments), cursor(_cursor) {} + + iterator& operator++() { + cursor++; + return *this; + } + + bool operator==(const iterator& that) const { + return cursor == that.cursor && segments == that.segments; + } + + bool operator!=(const iterator& that) const { + return !this->operator==(that); + } + + const char* operator*() const { return segments->At(cursor); } + + private: + const Segments* const segments; + size_t cursor; + }; + + const char* At(size_t cursor) const { + const auto& index = indexes[cursor]; + if (index.second) { + return managed[index.first].c_str(); + } else { + return unmanaged[index.first]; + } + } + + size_t Length() const { return indexes.size(); } + + iterator begin() const { return iterator(this, 0); } + + iterator end() const { return iterator(this, indexes.size()); } + + std::string ToString() const { + // TODO implement a nested structure to reduce concatenation, + // like a purely functional differential list + std::ostringstream buffer; + for (const char* segment : *this) { + buffer << segment; + } + return buffer.str(); + } + +private: + Segments(const Segments&) {} + + std::vector unmanaged; + std::vector managed; + // index, managed + std::vector> indexes; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/SerializableDict.hpp b/opencc/x86/include/opencc/SerializableDict.hpp new file mode 100644 index 00000000..17cea890 --- /dev/null +++ b/opencc/x86/include/opencc/SerializableDict.hpp @@ -0,0 +1,77 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Dict.hpp" + +namespace opencc { +/** + * Serializable dictionary interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializableDict { +public: + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(FILE* fp) const = 0; + + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(const std::string& fileName) const { + FILE* fp = fopen(fileName.c_str(), "wb"); + if (fp == NULL) { + throw FileNotWritable(fileName); + } + SerializeToFile(fp); + fclose(fp); + } + + template + static bool TryLoadFromFile(const std::string& fileName, + std::shared_ptr* dict) { + FILE* fp = +#ifdef _MSC_VER + // well, the 'GetPlatformString' shall return a 'wstring' + _wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb") +#else + fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb") +#endif // _MSC_VER + ; + + if (fp == NULL) { + return false; + } + std::shared_ptr loadedDict = DICT::NewFromFile(fp); + fclose(fp); + *dict = loadedDict; + return true; + } + + template + static std::shared_ptr NewFromFile(const std::string& fileName) { + std::shared_ptr dict; + if (!TryLoadFromFile(fileName, &dict)) { + throw FileNotFound(fileName); + } + return dict; + } +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/SerializedValues.hpp b/opencc/x86/include/opencc/SerializedValues.hpp new file mode 100644 index 00000000..c17210ba --- /dev/null +++ b/opencc/x86/include/opencc/SerializedValues.hpp @@ -0,0 +1,50 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Binary format for dictionary values serialization. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializedValues : public SerializableDict { +public: + SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {} + + virtual ~SerializedValues() {} + + virtual void SerializeToFile(FILE* fp) const; + + static std::shared_ptr NewFromFile(FILE* fp); + + const LexiconPtr& GetLexicon() const { return lexicon; } + + size_t KeyMaxLength() const; + +private: + LexiconPtr lexicon; + + void ConstructBuffer(std::string* valueBuffer, + std::vector* valueBytes, + uint32_t* valueTotalLength) const; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/SimpleConverter.hpp b/opencc/x86/include/opencc/SimpleConverter.hpp new file mode 100644 index 00000000..56932b76 --- /dev/null +++ b/opencc/x86/include/opencc/SimpleConverter.hpp @@ -0,0 +1,93 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Export.hpp" +#include + +#ifndef __OPENCC_SIMPLECONVERTER_HPP_ +#define __OPENCC_SIMPLECONVERTER_HPP_ + +/** + * @defgroup opencc_simple_api OpenCC C++ Simple API + * + * Simple API in C++ language + */ + +namespace opencc { +/** + * A high level converter + * This interface does not require C++11 to compile. + * @ingroup opencc_simple_api + */ +class OPENCC_EXPORT SimpleConverter { +public: + /** + * Constructor of SimpleConverter + * @param configFileName File name of configuration. + */ + SimpleConverter(const std::string& configFileName); + + ~SimpleConverter(); + + /** + * Converts a text + * @param input Text to be converted. + */ + std::string Convert(const std::string& input) const; + + /** + * Converts a text + * @param input A C-Style std::string (terminated by '\0') to be converted. + */ + std::string Convert(const char* input) const; + + /** + * Converts a text + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + */ + std::string Convert(const char* input, size_t length) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string (terminated by '\0') to be converted. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, char* output) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, size_t length, char* output) const; + +private: + const void* internalData; +}; + +} // namespace opencc + +#endif diff --git a/opencc/x86/include/opencc/TextDict.hpp b/opencc/x86/include/opencc/TextDict.hpp new file mode 100644 index 00000000..f1cb67d9 --- /dev/null +++ b/opencc/x86/include/opencc/TextDict.hpp @@ -0,0 +1,60 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Text dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT TextDict : public Dict, public SerializableDict { +public: + /** + * Constructor of TextDict. + * _lexicon must be sorted. + */ + TextDict(const LexiconPtr& _lexicon); + + virtual ~TextDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a TextDict from another dictionary. + */ + static TextDictPtr NewFromDict(const Dict& dict); + + static TextDictPtr NewFromFile(FILE* fp); + + static TextDictPtr NewFromSortedFile(FILE* fp); + +private: + const size_t maxLength; + const LexiconPtr lexicon; +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/UTF8StringSlice.hpp b/opencc/x86/include/opencc/UTF8StringSlice.hpp new file mode 100644 index 00000000..0c9dfdc6 --- /dev/null +++ b/opencc/x86/include/opencc/UTF8StringSlice.hpp @@ -0,0 +1,246 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "Common.hpp" +#include "UTF8Util.hpp" + +namespace opencc { + +namespace internal { + +inline size_t FNVHash(const char* text, const size_t byteLength, + const size_t FNV_prime, const size_t FNV_offset_basis) { + size_t hash = FNV_offset_basis; + for (const char* pstr = text; pstr < text + byteLength; pstr++) { + hash ^= *pstr; + hash *= FNV_prime; + } + return hash; +} + +template size_t FNVHash(const char* text, const size_t byteLength); + +template <> +inline size_t FNVHash<4>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 16777619UL, 2166136261UL); +} + +#if SIZE_MAX == 0xffffffffffffffff +template <> +inline size_t FNVHash<8>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL); +} +#endif + +} // namespace internal + +template class UTF8StringSliceBase { +public: + typedef LENGTH_TYPE LengthType; + + UTF8StringSliceBase(const char* _str) + : str(_str), utf8Length(static_cast(UTF8Util::Length(_str))), + byteLength(static_cast(strlen(_str))) {} + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length) + : str(_str), utf8Length(_utf8Length) { + CalculateByteLength(); + } + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length, + const LengthType _byteLength) + : str(_str), utf8Length(_utf8Length), byteLength(_byteLength) { + CalculateByteLength(); + } + + LengthType UTF8Length() const { return utf8Length; } + + LengthType ByteLength() const { return byteLength; } + + UTF8StringSliceBase Left(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + return UTF8StringSliceBase(str, numberOfCharacters); + } + } + + UTF8StringSliceBase Right(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + const char* pstr = str + byteLength; + for (size_t i = 0; i < numberOfCharacters; i++) { + pstr = UTF8Util::PrevChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + UTF8StringSliceBase SubString(const LengthType offset, + const LengthType numberOfCharacters) const { + if (offset == 0) { + return Left(numberOfCharacters); + } else { + const char* pstr = str; + for (size_t i = 0; i < offset; i++) { + pstr = UTF8Util::NextChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + std::string ToString() const { return std::string(str, str + byteLength); } + + const char* CString() const { return str; } + + LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const { + if (str == that.str) { + return (std::min)(utf8Length, that.utf8Length); + } else { + const char* pstr1 = str; + const char* pstr2 = that.str; + for (size_t length = 0; length < utf8Length && length < that.utf8Length; + length++) { + size_t charLen1 = UTF8Util::NextCharLength(pstr1); + size_t charLen2 = UTF8Util::NextCharLength(pstr2); + if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) { + return length; + } + pstr1 += charLen1; + pstr2 += charLen2; + } + return 0; + } + } + + void MoveRight() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::NextCharLength(str); + str += charLen; + utf8Length--; + byteLength -= charLen; + } + } + + void MoveLeft() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::PrevCharLength(str + byteLength); + utf8Length--; + byteLength -= charLen; + } + } + + int ReverseCompare(const UTF8StringSliceBase& that) const { + const char* pstr1 = str + byteLength; + const char* pstr2 = that.str + that.byteLength; + const size_t length = (std::min)(utf8Length, that.utf8Length); + for (size_t i = 0; i < length; i++) { + const size_t charLen1 = UTF8Util::PrevCharLength(pstr1); + const size_t charLen2 = UTF8Util::PrevCharLength(pstr2); + pstr1 -= charLen1; + pstr2 -= charLen2; + const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2)); + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } else if (charLen1 < charLen2) { + return -1; + } else if (charLen1 > charLen2) { + return 1; + } + } + if (utf8Length < that.utf8Length) { + return -1; + } else if (utf8Length > that.utf8Length) { + return 1; + } else { + return 0; + } + } + + LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const { + return static_cast( + ToString().find(pattern.str, 0, pattern.byteLength)); + } + + bool operator<(const UTF8StringSliceBase& that) const { + return Compare(that) < 0; + } + + bool operator>(const UTF8StringSliceBase& that) const { + return Compare(that) > 0; + } + + bool operator==(const UTF8StringSliceBase& that) const { + return (str == that.str && utf8Length == that.utf8Length) || + Compare(that) == 0; + } + + bool operator!=(const UTF8StringSliceBase& that) const { + return !this->operator==(that); + } + + class Hasher { + public: + size_t operator()(const UTF8StringSliceBase& text) const { + return internal::FNVHash(text.CString(), + text.ByteLength()); + } + }; + +private: + inline int Compare(const UTF8StringSliceBase& that) const { + int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength)); + if (cmp == 0) { + if (utf8Length < that.utf8Length) { + cmp = -1; + } else if (utf8Length > that.utf8Length) { + cmp = 1; + } else { + cmp = 0; + } + } + return cmp; + } + + void CalculateByteLength() { + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + pstr = UTF8Util::NextChar(pstr); + } + byteLength = static_cast(pstr - str); + } + + const char* str; + LengthType utf8Length; + LengthType byteLength; +}; + +typedef UTF8StringSliceBase UTF8StringSlice; + +template +std::ostream& operator<<(::std::ostream& os, + const UTF8StringSliceBase& str) { + return os << str.ToString(); +} + +} // namespace opencc diff --git a/opencc/x86/include/opencc/UTF8Util.hpp b/opencc/x86/include/opencc/UTF8Util.hpp new file mode 100644 index 00000000..9d3e39b8 --- /dev/null +++ b/opencc/x86/include/opencc/UTF8Util.hpp @@ -0,0 +1,290 @@ +/* + * Open Chinese Convert + * + * Copyright 2013 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef _MSC_VER +#define NOMINMAX +#include +#undef NOMINMAX +#endif // _MSC_VER + +#include + +#include "Common.hpp" +#include "Exception.hpp" + +namespace opencc { +/** + * UTF8 std::string utilities + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT UTF8Util { +public: + /** + * Detect UTF8 BOM and skip it. + */ + static void SkipUtf8Bom(FILE* fp); + + /** + * Returns the length in byte for the next UTF8 character. + * On error returns 0. + */ + static size_t NextCharLengthNoException(const char* str) { + char ch = *str; + if ((ch & 0xF0) == 0xE0) { + return 3; + } else if ((ch & 0x80) == 0x00) { + return 1; + } else if ((ch & 0xE0) == 0xC0) { + return 2; + } else if ((ch & 0xF8) == 0xF0) { + return 4; + } else if ((ch & 0xFC) == 0xF8) { + return 5; + } else if ((ch & 0xFE) == 0xFC) { + return 6; + } + return 0; + } + + /** + * Returns the length in byte for the next UTF8 character. + */ + static size_t NextCharLength(const char* str) { + size_t length = NextCharLengthNoException(str); + if (length == 0) { + throw InvalidUTF8(str); + } + return length; + } + + /** + * Returns the length in byte for the previous UTF8 character. + */ + static size_t PrevCharLength(const char* str) { + { + const size_t length = NextCharLengthNoException(str - 3); + if (length == 3) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 1); + if (length == 1) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 2); + if (length == 2) { + return length; + } + } + for (size_t i = 4; i <= 6; i++) { + const size_t length = NextCharLengthNoException(str - i); + if (length == i) { + return length; + } + } + throw InvalidUTF8(str); + } + + /** + * Returns the char* pointer over the next UTF8 character. + */ + static const char* NextChar(const char* str) { + return str + NextCharLength(str); + } + + /** + * Move the char* pointer before the previous UTF8 character. + */ + static const char* PrevChar(const char* str) { + return str - PrevCharLength(str); + } + + /** + * Returns the UTF8 length of a valid UTF8 std::string. + */ + static size_t Length(const char* str) { + size_t length = 0; + while (*str != '\0') { + str = NextChar(str); + length++; + } + return length; + } + + /** + * Finds a character in the same line. + * @param str The text to be searched in. + * @param ch The character to find. + * @return The pointer that points to the found chacter in str or EOL/EOF. + */ + static const char* FindNextInline(const char* str, const char ch) { + while (!IsLineEndingOrFileEnding(*str) && *str != ch) { + str = NextChar(str); + } + return str; + } + + /** + * Returns true if the character is a line ending or end of file. + */ + static bool IsLineEndingOrFileEnding(const char ch) { + return ch == '\0' || ch == '\n' || ch == '\r'; + } + + /** + * Copies a substring with given length to a new string. + */ + static std::string FromSubstr(const char* str, size_t length) { + std::string newStr; + newStr.resize(length); + strncpy(const_cast(newStr.c_str()), str, length); + return newStr; + } + + /** + * Returns true if the given std::string is longer or as long as the given + * length. + */ + static bool NotShorterThan(const char* str, size_t byteLength) { + while (byteLength > 0) { + if (*str == '\0') { + return false; + } + byteLength--; + str++; + } + return true; + } + + /** + * Truncates a std::string with a maximal length in byte. + * No UTF8 character will be broken. + */ + static std::string TruncateUTF8(const char* str, size_t maxByteLength) { + std::string wordTrunc; + if (NotShorterThan(str, maxByteLength)) { + size_t len = 0; + const char* pStr = str; + for (;;) { + const size_t charLength = NextCharLength(pStr); + if (len + charLength > maxByteLength) { + break; + } + pStr += charLength; + len += charLength; + } + wordTrunc = FromSubstr(str, len); + } else { + wordTrunc = str; + } + return wordTrunc; + } + + /** + * Replaces all patterns in a std::string in place. + */ + static void ReplaceAll(std::string& str, const char* from, const char* to) { + std::string::size_type pos = 0; + std::string::size_type fromLen = strlen(from); + std::string::size_type toLen = strlen(to); + while ((pos = str.find(from, pos)) != std::string::npos) { + str.replace(pos, fromLen, to); + pos += toLen; + } + } + + /** + * Joins a std::string vector in to a std::string with a separator. + */ + static std::string Join(const std::vector& strings, + const std::string& separator) { + std::ostringstream buffer; + bool first = true; + for (const auto& str : strings) { + if (!first) { + buffer << separator; + } + buffer << str; + first = false; + } + return buffer.str(); + } + + /** + * Joins a std::string vector in to a std::string. + */ + static std::string Join(const std::vector& strings) { + std::ostringstream buffer; + for (const auto& str : strings) { + buffer << str; + } + return buffer.str(); + } + + static void GetByteMap(const char* str, const size_t utf8Length, + std::vector* byteMap) { + if (byteMap->size() < utf8Length) { + byteMap->resize(utf8Length); + } + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + (*byteMap)[i] = pstr - str; + pstr = NextChar(pstr); + } + } + +#ifdef _MSC_VER + static std::wstring GetPlatformString(const std::string& str) { + return U8ToU16(str); + } +#else + static std::string GetPlatformString(const std::string& str) { return str; } +#endif // _MSC_VER + +#ifdef _MSC_VER + static std::string U16ToU8(const std::wstring& wstr) { + std::string ret; + int length = static_cast(wstr.length()); + int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0, + NULL, NULL); + if (convcnt > 0) { + ret.resize(convcnt); + WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt, + NULL, NULL); + } + return ret; + } + + static std::wstring U8ToU16(const std::string& str) { + std::wstring ret; + int length = static_cast(str.length()); + int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0); + if (convcnt > 0) { + ret.resize(convcnt); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt); + } + return ret; + } +#endif // _MSC_VER +}; +} // namespace opencc diff --git a/opencc/x86/include/opencc/opencc.h b/opencc/x86/include/opencc/opencc.h new file mode 100644 index 00000000..bc3a9e97 --- /dev/null +++ b/opencc/x86/include/opencc/opencc.h @@ -0,0 +1,161 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __OPENCC_H_ +#define __OPENCC_H_ + +#ifdef __cplusplus + +#include "Export.hpp" +#include "SimpleConverter.hpp" +#include + +extern "C" { +#else +#include +#endif + +#ifndef OPENCC_EXPORT +#define OPENCC_EXPORT +#endif + +/** + * @defgroup opencc_c_api OpenCC C API + * + * API in C language + */ + +/** + * Filename of default Simplified to Traditional configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json" + +/** + * Filename of default Traditional to Simplified configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json" + +/** + * Type of opencc descriptor + * + * @ingroup opencc_c_api + */ +typedef void* opencc_t; + +/** + * Makes an instance of opencc + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open(const char* configFileName); +#ifdef _MSC_VER +/** + * Makes an instance of opencc (wide char / Unicode) + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName); +#endif /* _MSC_VER */ + +/** + * Destroys an instance of opencc + * + * @param opencc The description pointer. + * @return 0 on success or non-zero number on failure. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT int opencc_close(opencc_t opencc); + +/** + * Converts UTF-8 std::string + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * @param output The buffer to store converted text. You MUST make sure this + * buffer has sufficient space. + * + * @return The length of converted std::string or (size_t)-1 on error. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc, + const char* input, + size_t length, char* output); + +/** + * Converts UTF-8 std::string + * This function returns an allocated C-Style std::string, which stores + * the converted std::string. + * You MUST call opencc_convert_utf8_free() to release allocated memory. + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * + * @return The newly allocated UTF-8 std::string that stores text + * converted, or NULL on error. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input, + size_t length); + +/** + * Releases allocated buffer by opencc_convert_utf8 + * + * @param str Pointer to the allocated std::string buffer by + * opencc_convert_utf8. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT void opencc_convert_utf8_free(char* str); + +/** + * Returns the last error message + * + * Note that this function is the only one which is NOT thread-safe. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT const char* opencc_error(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +/** + * @defgroup opencc_cpp_api OpenCC C++ Comprehensive API + * + * Comprehensive API in C++ language + */ + +#endif diff --git a/opencc/x86/include/opencc/opencc_config.h b/opencc/x86/include/opencc/opencc_config.h new file mode 100644 index 00000000..8dad4489 --- /dev/null +++ b/opencc/x86/include/opencc/opencc_config.h @@ -0,0 +1,21 @@ +/* + * Open Chinese Convert + * + * Copyright 2021 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* #undef OPENCC_ENABLE_DARTS */ diff --git a/opencc/x86/lib/cmake/opencc/OpenCCConfig.cmake b/opencc/x86/lib/cmake/opencc/OpenCCConfig.cmake new file mode 100644 index 00000000..a360c2a4 --- /dev/null +++ b/opencc/x86/lib/cmake/opencc/OpenCCConfig.cmake @@ -0,0 +1,31 @@ + +####### Expanded from @PACKAGE_INIT@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run #### +####### The input file was OpenCCConfig.cmake.in ######## + +get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) + +macro(set_and_check _var _file) + set(${_var} "${_file}") + if(NOT EXISTS "${_file}") + message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") + endif() +endmacro() + +macro(check_required_components _NAME) + foreach(comp ${${_NAME}_FIND_COMPONENTS}) + if(NOT ${_NAME}_${comp}_FOUND) + if(${_NAME}_FIND_REQUIRED_${comp}) + set(${_NAME}_FOUND FALSE) + endif() + endif() + endforeach() +endmacro() + +#################################################################################### + +include(${CMAKE_CURRENT_LIST_DIR}/OpenCCTargets.cmake) + +set_and_check(OPENCC_INCLUDE_DIR ${PACKAGE_PREFIX_DIR}/include/opencc) + +check_required_components(OpenCC) diff --git a/opencc/x86/lib/cmake/opencc/OpenCCConfigVersion.cmake b/opencc/x86/lib/cmake/opencc/OpenCCConfigVersion.cmake new file mode 100644 index 00000000..5b32bd03 --- /dev/null +++ b/opencc/x86/lib/cmake/opencc/OpenCCConfigVersion.cmake @@ -0,0 +1,70 @@ +# This is a basic version file for the Config-mode of find_package(). +# It is used by write_basic_package_version_file() as input file for configure_file() +# to create a version-file which can be installed along a config.cmake file. +# +# The created file sets PACKAGE_VERSION_EXACT if the current version string and +# the requested version string are exactly the same and it sets +# PACKAGE_VERSION_COMPATIBLE if the current version is >= requested version, +# but only if the requested major version is the same as the current one. +# The variable CVF_VERSION must be set before calling configure_file(). + + +set(PACKAGE_VERSION "1.1.6") + +if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + + if("1.1.6" MATCHES "^([0-9]+)\\.") + set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}") + if(NOT CVF_VERSION_MAJOR VERSION_EQUAL 0) + string(REGEX REPLACE "^0+" "" CVF_VERSION_MAJOR "${CVF_VERSION_MAJOR}") + endif() + else() + set(CVF_VERSION_MAJOR "1.1.6") + endif() + + if(PACKAGE_FIND_VERSION_RANGE) + # both endpoints of the range must have the expected major version + math (EXPR CVF_VERSION_MAJOR_NEXT "${CVF_VERSION_MAJOR} + 1") + if (NOT PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + OR ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX_MAJOR STREQUAL CVF_VERSION_MAJOR) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX VERSION_LESS_EQUAL CVF_VERSION_MAJOR_NEXT))) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + elseif(PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + AND ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND PACKAGE_VERSION VERSION_LESS_EQUAL PACKAGE_FIND_VERSION_MAX) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MAX))) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + else() + if(PACKAGE_FIND_VERSION_MAJOR STREQUAL CVF_VERSION_MAJOR) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + + if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + endif() + endif() +endif() + + +# if the installed project requested no architecture check, don't perform the check +if("FALSE") + return() +endif() + +# if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it: +if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "" OR "4" STREQUAL "") + return() +endif() + +# check that the installed version has the same 32/64bit-ness as the one which is currently searching: +if(NOT CMAKE_SIZEOF_VOID_P STREQUAL "4") + math(EXPR installedBits "4 * 8") + set(PACKAGE_VERSION "${PACKAGE_VERSION} (${installedBits}bit)") + set(PACKAGE_VERSION_UNSUITABLE TRUE) +endif() diff --git a/opencc/x86/lib/cmake/opencc/OpenCCTargets-release.cmake b/opencc/x86/lib/cmake/opencc/OpenCCTargets-release.cmake new file mode 100644 index 00000000..199b87f5 --- /dev/null +++ b/opencc/x86/lib/cmake/opencc/OpenCCTargets-release.cmake @@ -0,0 +1,29 @@ +#---------------------------------------------------------------- +# Generated CMake target import file for configuration "Release". +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Import target "OpenCC::OpenCC" for configuration "Release" +set_property(TARGET OpenCC::OpenCC APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::OpenCC PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libopencc.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::OpenCC ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::OpenCC "${_IMPORT_PREFIX}/lib/libopencc.a" ) + +# Import target "OpenCC::marisa" for configuration "Release" +set_property(TARGET OpenCC::marisa APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::marisa PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libmarisa.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::marisa ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::marisa "${_IMPORT_PREFIX}/lib/libmarisa.a" ) + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) diff --git a/opencc/x86/lib/cmake/opencc/OpenCCTargets.cmake b/opencc/x86/lib/cmake/opencc/OpenCCTargets.cmake new file mode 100644 index 00000000..db461d58 --- /dev/null +++ b/opencc/x86/lib/cmake/opencc/OpenCCTargets.cmake @@ -0,0 +1,102 @@ +# Generated by CMake + +if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS 2.6) + message(FATAL_ERROR "CMake >= 2.6.0 required") +endif() +cmake_policy(PUSH) +cmake_policy(VERSION 2.6...3.20) +#---------------------------------------------------------------- +# Generated CMake target import file. +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Protect against multiple inclusion, which would fail when already imported targets are added once more. +set(_targetsDefined) +set(_targetsNotDefined) +set(_expectedTargets) +foreach(_expectedTarget OpenCC::OpenCC OpenCC::marisa) + list(APPEND _expectedTargets ${_expectedTarget}) + if(NOT TARGET ${_expectedTarget}) + list(APPEND _targetsNotDefined ${_expectedTarget}) + endif() + if(TARGET ${_expectedTarget}) + list(APPEND _targetsDefined ${_expectedTarget}) + endif() +endforeach() +if("${_targetsDefined}" STREQUAL "${_expectedTargets}") + unset(_targetsDefined) + unset(_targetsNotDefined) + unset(_expectedTargets) + set(CMAKE_IMPORT_FILE_VERSION) + cmake_policy(POP) + return() +endif() +if(NOT "${_targetsDefined}" STREQUAL "") + message(FATAL_ERROR "Some (but not all) targets in this export set were already defined.\nTargets Defined: ${_targetsDefined}\nTargets not yet defined: ${_targetsNotDefined}\n") +endif() +unset(_targetsDefined) +unset(_targetsNotDefined) +unset(_expectedTargets) + + +# Compute the installation prefix relative to this file. +get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +if(_IMPORT_PREFIX STREQUAL "/") + set(_IMPORT_PREFIX "") +endif() + +# Create imported target OpenCC::OpenCC +add_library(OpenCC::OpenCC STATIC IMPORTED) + +set_target_properties(OpenCC::OpenCC PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include/opencc" + INTERFACE_LINK_LIBRARIES "OpenCC::marisa" +) + +# Create imported target OpenCC::marisa +add_library(OpenCC::marisa STATIC IMPORTED) + +if(CMAKE_VERSION VERSION_LESS 2.8.12) + message(FATAL_ERROR "This file relies on consumers using CMake 2.8.12 or greater.") +endif() + +# Load information for each installed configuration. +get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +file(GLOB CONFIG_FILES "${_DIR}/OpenCCTargets-*.cmake") +foreach(f ${CONFIG_FILES}) + include(${f}) +endforeach() + +# Cleanup temporary variables. +set(_IMPORT_PREFIX) + +# Loop over all imported files and verify that they actually exist +foreach(target ${_IMPORT_CHECK_TARGETS} ) + foreach(file ${_IMPORT_CHECK_FILES_FOR_${target}} ) + if(NOT EXISTS "${file}" ) + message(FATAL_ERROR "The imported target \"${target}\" references the file + \"${file}\" +but this file does not exist. Possible reasons include: +* The file was deleted, renamed, or moved to another location. +* An install or uninstall procedure did not complete successfully. +* The installation package was faulty and contained + \"${CMAKE_CURRENT_LIST_FILE}\" +but not all the files it references. +") + endif() + endforeach() + unset(_IMPORT_CHECK_FILES_FOR_${target}) +endforeach() +unset(_IMPORT_CHECK_TARGETS) + +# This file does not depend on other imported targets which have +# been exported from the same project but in a separate export set. + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) +cmake_policy(POP) diff --git a/opencc/x86/lib/libmarisa.a b/opencc/x86/lib/libmarisa.a new file mode 100644 index 00000000..15f62acb Binary files /dev/null and b/opencc/x86/lib/libmarisa.a differ diff --git a/opencc/x86/lib/libopencc.a b/opencc/x86/lib/libopencc.a new file mode 100644 index 00000000..0be8594c Binary files /dev/null and b/opencc/x86/lib/libopencc.a differ diff --git a/opencc/x86/share/opencc b/opencc/x86/share/opencc new file mode 120000 index 00000000..e67b4559 --- /dev/null +++ b/opencc/x86/share/opencc @@ -0,0 +1 @@ +../../data \ No newline at end of file diff --git a/opencc/x86_64/include/opencc/Common.hpp b/opencc/x86_64/include/opencc/Common.hpp new file mode 100644 index 00000000..067966b5 --- /dev/null +++ b/opencc/x86_64/include/opencc/Common.hpp @@ -0,0 +1,82 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Microsoft Visual C++ specific +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820) +#endif + +#include +#include +#include +#include + +#include "Export.hpp" +#include "Optional.hpp" +#include "opencc_config.h" + +// Forward decalarations and alias +namespace opencc { +class Config; +class Conversion; +class ConversionChain; +class Converter; +class Dict; +class DictEntry; +class DictGroup; +class Lexicon; +class MarisaDict; +class MultiValueDictEntry; +class NoValueDictEntry; +class Segmentation; +class Segments; +class SerializableDict; +class SingleValueDictEntry; +class TextDict; +typedef std::shared_ptr ConversionPtr; +typedef std::shared_ptr ConversionChainPtr; +typedef std::shared_ptr ConverterPtr; +typedef std::shared_ptr DictPtr; +typedef std::shared_ptr DictGroupPtr; +typedef std::shared_ptr LexiconPtr; +typedef std::shared_ptr MarisaDictPtr; +typedef std::shared_ptr SegmentationPtr; +typedef std::shared_ptr SegmentsPtr; +typedef std::shared_ptr SerializableDictPtr; +typedef std::shared_ptr TextDictPtr; + +#ifdef OPENCC_ENABLE_DARTS +class BinaryDict; +class DartsDict; +typedef std::shared_ptr BinaryDictPtr; +typedef std::shared_ptr DartsDictPtr; +#endif + +} // namespace opencc + +#ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = ""; +#else // ifndef PKGDATADIR +const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/"; +#endif // ifndef PKGDATADIR + +#ifndef VERSION +#define VERSION "1.0.*" +#endif // ifndef VERSION diff --git a/opencc/x86_64/include/opencc/Config.hpp b/opencc/x86_64/include/opencc/Config.hpp new file mode 100644 index 00000000..7a904cee --- /dev/null +++ b/opencc/x86_64/include/opencc/Config.hpp @@ -0,0 +1,42 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Configuration loader + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Config { +public: + Config(); + + virtual ~Config(); + + ConverterPtr NewFromString(const std::string& json, + const std::string& configDirectory); + + ConverterPtr NewFromFile(const std::string& fileName); + +private: + void* internal; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Conversion.hpp b/opencc/x86_64/include/opencc/Conversion.hpp new file mode 100644 index 00000000..cf73a254 --- /dev/null +++ b/opencc/x86_64/include/opencc/Conversion.hpp @@ -0,0 +1,47 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Conversion interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Conversion { +public: + Conversion(DictPtr _dict) : dict(_dict) {} + + // Convert single phrase + std::string Convert(const std::string& phrase) const; + + // Convert single phrase + std::string Convert(const char* phrase) const; + + // Convert segmented text + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/ConversionChain.hpp b/opencc/x86_64/include/opencc/ConversionChain.hpp new file mode 100644 index 00000000..28853c67 --- /dev/null +++ b/opencc/x86_64/include/opencc/ConversionChain.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Conversion.hpp" + +namespace opencc { +/** + * Chain of conversions + * Consists of a list of conversions. Converts input in sequence. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT ConversionChain { +public: + ConversionChain(const std::list _conversions); + + SegmentsPtr Convert(const SegmentsPtr& input) const; + + const std::list GetConversions() const { return conversions; } + +private: + const std::list conversions; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Converter.hpp b/opencc/x86_64/include/opencc/Converter.hpp new file mode 100644 index 00000000..9f1f3f94 --- /dev/null +++ b/opencc/x86_64/include/opencc/Converter.hpp @@ -0,0 +1,51 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Controller of segmentation and conversion + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Converter { +public: + Converter(const std::string& _name, SegmentationPtr _segmentation, + ConversionChainPtr _conversionChain) + : name(_name), segmentation(_segmentation), + conversionChain(_conversionChain) {} + + std::string Convert(const std::string& text) const; + + size_t Convert(const char* input, char* output) const; + + const SegmentationPtr GetSegmentation() const { return segmentation; } + + const ConversionChainPtr GetConversionChain() const { + return conversionChain; + } + +private: + const std::string name; + const SegmentationPtr segmentation; + const ConversionChainPtr conversionChain; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Dict.hpp b/opencc/x86_64/include/opencc/Dict.hpp new file mode 100644 index 00000000..1c810346 --- /dev/null +++ b/opencc/x86_64/include/opencc/Dict.hpp @@ -0,0 +1,92 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Abstract class of dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Dict { +public: + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + virtual Optional Match(const char* word, + size_t len) const = 0; + + /** + * Matches a word exactly and returns the DictEntry or Optional::Null(). + */ + Optional Match(const std::string& word) const { + return Match(word.c_str(), word.length()); + } + + /** + * Matches the longest matched prefix of a word. + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * the longest prefix of "banana" matched is "bana". + */ + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const char* word) const { + return MatchPrefix(word, KeyMaxLength()); + } + + /** + * Matches the longest matched prefix of a word. + */ + Optional MatchPrefix(const std::string& word) const { + return MatchPrefix(word.c_str(), word.length()); + } + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana", + * all the matched prefixes of "banana" are "bana", "ban", "ba", "b". + */ + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + /** + * Returns all matched prefixes of a word, sorted by the length (desc). + */ + std::vector + MatchAllPrefixes(const std::string& word) const { + return MatchAllPrefixes(word.c_str(), word.length()); + } + + /** + * Returns the length of the longest key in the dictionary. + */ + virtual size_t KeyMaxLength() const = 0; + + /** + * Returns all entries in the dictionary. + */ + virtual LexiconPtr GetLexicon() const = 0; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/DictConverter.hpp b/opencc/x86_64/include/opencc/DictConverter.hpp new file mode 100644 index 00000000..f911c4fe --- /dev/null +++ b/opencc/x86_64/include/opencc/DictConverter.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2017 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Converts a dictionary from a format to another. + * @ingroup opencc_cpp_api + */ +OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName, + const std::string& outputFileName, + const std::string& formatFrom, + const std::string& formatTo); +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/DictEntry.hpp b/opencc/x86_64/include/opencc/DictEntry.hpp new file mode 100644 index 00000000..7b2babda --- /dev/null +++ b/opencc/x86_64/include/opencc/DictEntry.hpp @@ -0,0 +1,173 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "Segments.hpp" +#include "UTF8Util.hpp" + +namespace opencc { +/** + * Key-values pair entry + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictEntry { +public: + virtual ~DictEntry() {} + + virtual std::string Key() const = 0; + + virtual std::vector Values() const = 0; + + virtual std::string GetDefault() const = 0; + + virtual size_t NumValues() const = 0; + + virtual std::string ToString() const = 0; + + size_t KeyLength() const { return Key().length(); } + + bool operator<(const DictEntry& that) const { return Key() < that.Key(); } + + bool operator==(const DictEntry& that) const { return Key() == that.Key(); } + + static bool UPtrLessThan(const std::unique_ptr& a, + const std::unique_ptr& b) { + return *a < *b; + } +}; + +class OPENCC_EXPORT NoValueDictEntry : public DictEntry { +public: + NoValueDictEntry(const std::string& _key) : key(_key) {} + + virtual ~NoValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::vector Values() const { + return std::vector(); + } + + virtual std::string GetDefault() const { return key; } + + virtual size_t NumValues() const { return 0; } + + virtual std::string ToString() const { return key; } + +private: + std::string key; +}; + +class OPENCC_EXPORT SingleValueDictEntry : public DictEntry { +public: + virtual std::string Value() const = 0; + + virtual std::vector Values() const { + return std::vector{Value()}; + } + + virtual std::string GetDefault() const { return Value(); } + + virtual size_t NumValues() const { return 1; } + + virtual std::string ToString() const { + return std::string(Key()) + "\t" + Value(); + } +}; + +class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry { +public: + StrSingleValueDictEntry(const std::string& _key, const std::string& _value) + : key(_key), value(_value) {} + + virtual ~StrSingleValueDictEntry() {} + + virtual std::string Key() const { return key; } + + virtual std::string Value() const { return value; } + +private: + std::string key; + std::string value; +}; + +class OPENCC_EXPORT MultiValueDictEntry : public DictEntry { +public: + virtual std::string GetDefault() const { + if (NumValues() > 0) { + return Values().at(0); + } else { + return Key(); + } + } + + virtual std::string ToString() const; +}; + +class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry { +public: + StrMultiValueDictEntry(const std::string& _key, + const std::vector& _values) + : key(_key), values(_values) {} + + virtual ~StrMultiValueDictEntry() {} + + virtual std::string Key() const { return key; } + + size_t NumValues() const { return values.size(); } + + std::vector Values() const { return values; } + +private: + std::string key; + std::vector values; +}; + +class OPENCC_EXPORT DictEntryFactory { +public: + static DictEntry* New(const std::string& key) { + return new NoValueDictEntry(key); + } + + static DictEntry* New(const std::string& key, const std::string& value) { + return new StrSingleValueDictEntry(key, value); + } + + static DictEntry* New(const std::string& key, + const std::vector& values) { + if (values.size() == 0) { + return New(key); + } else if (values.size() == 1) { + return New(key, values.front()); + } + return new StrMultiValueDictEntry(key, values); + } + + static DictEntry* New(const DictEntry* entry) { + if (entry->NumValues() == 0) { + return new NoValueDictEntry(entry->Key()); + } else if (entry->NumValues() == 1) { + return new StrSingleValueDictEntry(entry->Key(), entry->Values().front()); + } else { + return new StrMultiValueDictEntry(entry->Key(), entry->Values()); + } + } +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/DictGroup.hpp b/opencc/x86_64/include/opencc/DictGroup.hpp new file mode 100644 index 00000000..fd51d913 --- /dev/null +++ b/opencc/x86_64/include/opencc/DictGroup.hpp @@ -0,0 +1,57 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" +#include "Dict.hpp" + +namespace opencc { +/** + * Group of dictionaries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT DictGroup : public Dict { +public: + DictGroup(const std::list& dicts); + + static DictGroupPtr NewFromDict(const Dict& dict); + + virtual ~DictGroup(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + const std::list GetDicts() const { return dicts; } + +private: + const size_t keyMaxLength; + const std::list dicts; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Exception.hpp b/opencc/x86_64/include/opencc/Exception.hpp new file mode 100644 index 00000000..7875537f --- /dev/null +++ b/opencc/x86_64/include/opencc/Exception.hpp @@ -0,0 +1,88 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Export.hpp" + +#if defined(_MSC_VER) && _MSC_VER < 1900 +// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not +// supported +#define noexcept +#endif // ifdef _MSC_VER + +namespace opencc { + +class OPENCC_EXPORT Exception { +public: + Exception() {} + + virtual ~Exception() throw() {} + + Exception(const std::string& _message) : message(_message) {} + + virtual const char* what() const noexcept { return message.c_str(); } + +protected: + std::string message; +}; + +class OPENCC_EXPORT FileNotFound : public Exception { +public: + FileNotFound(const std::string& fileName) + : Exception(fileName + " not found or not accessible.") {} +}; + +class OPENCC_EXPORT FileNotWritable : public Exception { +public: + FileNotWritable(const std::string& fileName) + : Exception(fileName + " not writable.") {} +}; + +class OPENCC_EXPORT InvalidFormat : public Exception { +public: + InvalidFormat(const std::string& message) + : Exception("Invalid format: " + message) {} +}; + +class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat { +public: + InvalidTextDictionary(const std::string& _message, size_t lineNum) + : InvalidFormat("") { + std::ostringstream buffer; + buffer << "Invalid text dictionary at line " << lineNum << ": " << _message; + message = buffer.str(); + } +}; + +class OPENCC_EXPORT InvalidUTF8 : public Exception { +public: + InvalidUTF8(const std::string& _message) + : Exception("Invalid UTF8: " + _message) {} +}; + +class OPENCC_EXPORT ShouldNotBeHere : public Exception { +public: + ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {} +}; + +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Export.hpp b/opencc/x86_64/include/opencc/Export.hpp new file mode 100644 index 00000000..0015e880 --- /dev/null +++ b/opencc/x86_64/include/opencc/Export.hpp @@ -0,0 +1,40 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#define OPENCC_EXPORT +#define OPENCC_NO_EXPORT +#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) +#ifndef OPENCC_EXPORT +#ifdef libopencc_EXPORTS + +/* We are building this library */ +#define OPENCC_EXPORT __declspec(dllexport) +#else // ifdef libopencc_EXPORTS + +/* We are using this library */ +#define OPENCC_EXPORT __declspec(dllimport) +#endif // ifdef libopencc_EXPORTS +#endif // ifndef OPENCC_EXPORT + +#ifndef OPENCC_NO_EXPORT +#define OPENCC_NO_EXPORT +#endif // ifndef OPENCC_NO_EXPORT +#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32) diff --git a/opencc/x86_64/include/opencc/Lexicon.hpp b/opencc/x86_64/include/opencc/Lexicon.hpp new file mode 100644 index 00000000..68891287 --- /dev/null +++ b/opencc/x86_64/include/opencc/Lexicon.hpp @@ -0,0 +1,68 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictEntry.hpp" + +namespace opencc { +/** + * Storage of all entries + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Lexicon { +public: + Lexicon() {} + Lexicon(std::vector> entries_) + : entries(std::move(entries_)) {} + Lexicon(const Lexicon&) = delete; + Lexicon& operator=(const Lexicon&) = delete; + + // Lexicon will take the ownership of the entry. + void Add(DictEntry* entry) { entries.emplace_back(entry); } + + void Add(std::unique_ptr entry) { + entries.push_back(std::move(entry)); + } + + void Sort(); + + // Returns true if the lexicon is sorted by key. + bool IsSorted(); + + // Returns true if every key unique (after sorted). + // When dupkey is set, it is set to the duplicate key. + bool IsUnique(std::string* dupkey = nullptr); + + const DictEntry* At(size_t index) const { return entries.at(index).get(); } + + size_t Length() const { return entries.size(); } + + std::vector>::const_iterator begin() const { + return entries.begin(); + } + + std::vector>::const_iterator end() const { + return entries.end(); + } + +private: + std::vector> entries; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/MarisaDict.hpp b/opencc/x86_64/include/opencc/MarisaDict.hpp new file mode 100644 index 00000000..6917c23a --- /dev/null +++ b/opencc/x86_64/include/opencc/MarisaDict.hpp @@ -0,0 +1,63 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Darts dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict { +public: + virtual ~MarisaDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual Optional MatchPrefix(const char* word, + size_t len) const; + + virtual std::vector MatchAllPrefixes(const char* word, + size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a MarisaDict from another dictionary. + */ + static MarisaDictPtr NewFromDict(const Dict& thatDict); + + static MarisaDictPtr NewFromFile(FILE* fp); + +private: + MarisaDict(); + + size_t maxLength; + LexiconPtr lexicon; + + class MarisaInternal; + std::unique_ptr internal; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/MaxMatchSegmentation.hpp b/opencc/x86_64/include/opencc/MaxMatchSegmentation.hpp new file mode 100644 index 00000000..1ecc227e --- /dev/null +++ b/opencc/x86_64/include/opencc/MaxMatchSegmentation.hpp @@ -0,0 +1,43 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "DictGroup.hpp" +#include "Segmentation.hpp" + +namespace opencc { +/** + * Implementation of maximal match segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation { +public: + MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {} + + virtual ~MaxMatchSegmentation() {} + + virtual SegmentsPtr Segment(const std::string& text) const; + + const DictPtr GetDict() const { return dict; } + +private: + const DictPtr dict; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Optional.hpp b/opencc/x86_64/include/opencc/Optional.hpp new file mode 100644 index 00000000..8a510f43 --- /dev/null +++ b/opencc/x86_64/include/opencc/Optional.hpp @@ -0,0 +1,76 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace opencc { +/** + * A class that wraps type T into a nullable type. + * @ingroup opencc_cpp_api + */ +template class Optional { +public: + /** + * The constructor of Optional. + */ + Optional(T actual) : isNull(false), data(actual) {} + + /** + * Returns true if the instance is null. + */ + bool IsNull() const { return isNull; } + + /** + * Returns the containing data of the instance. + */ + const T& Get() const { return data; } + + /** + * Constructs a null instance. + */ + static Optional Null() { return Optional(); } + +private: + Optional() : isNull(true) {} + + bool isNull; + T data; +}; + +/** + * Specialization of Optional for pointers. + * + * Reduce a bool. + */ +template class Optional { +private: + Optional() : data(nullptr) {} + + typedef T* TPtr; + TPtr data; + +public: + Optional(TPtr actual) : data(actual) {} + + bool IsNull() const { return data == nullptr; } + + const TPtr& Get() const { return data; } + + static Optional Null() { return Optional(); } +}; +} // namespace opencc \ No newline at end of file diff --git a/opencc/x86_64/include/opencc/PhraseExtract.hpp b/opencc/x86_64/include/opencc/PhraseExtract.hpp new file mode 100644 index 00000000..75a165de --- /dev/null +++ b/opencc/x86_64/include/opencc/PhraseExtract.hpp @@ -0,0 +1,195 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Common.hpp" +#include "UTF8StringSlice.hpp" + +namespace opencc { + +class OPENCC_EXPORT PhraseExtract { +public: + typedef UTF8StringSlice::LengthType LengthType; + + typedef UTF8StringSliceBase UTF8StringSlice8Bit; + + PhraseExtract(); + + virtual ~PhraseExtract(); + + void Extract(const std::string& text) { + SetFullText(text); + ExtractSuffixes(); + CalculateFrequency(); + CalculateSuffixEntropy(); + ReleaseSuffixes(); + ExtractPrefixes(); + CalculatePrefixEntropy(); + ReleasePrefixes(); + ExtractWordCandidates(); + CalculateCohesions(); + SelectWords(); + } + + void SetFullText(const std::string& fullText) { + utf8FullText = UTF8StringSlice(fullText.c_str()); + } + + void SetFullText(const char* fullText) { + utf8FullText = UTF8StringSlice(fullText); + } + + void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; } + + void SetWordMinLength(const LengthType _wordMinLength) { + wordMinLength = _wordMinLength; + } + + void SetWordMaxLength(const LengthType _wordMaxLength) { + wordMaxLength = _wordMaxLength; + } + + void SetPrefixSetLength(const LengthType _prefixSetLength) { + prefixSetLength = _prefixSetLength; + } + + void SetSuffixSetLength(const LengthType _suffixSetLength) { + suffixSetLength = _suffixSetLength; + } + + // PreCalculationFilter is called after frequencies statistics. + void SetPreCalculationFilter( + const std::function& filter) { + preCalculationFilter = filter; + } + + void SetPostCalculationFilter( + const std::function& filter) { + postCalculationFilter = filter; + } + + void ReleaseSuffixes() { std::vector().swap(suffixes); } + + void ReleasePrefixes() { std::vector().swap(prefixes); } + + const std::vector& Words() const { return words; } + + const std::vector& WordCandidates() const { + return wordCandidates; + } + + struct Signals { + size_t frequency; + double cohesion; + double suffixEntropy; + double prefixEntropy; + }; + + const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const; + + double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double Entropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const; + + size_t Frequency(const UTF8StringSlice8Bit& word) const; + + double Probability(const UTF8StringSlice8Bit& word) const; + + double LogProbability(const UTF8StringSlice8Bit& word) const; + + void Reset(); + + void ExtractSuffixes(); + + void ExtractPrefixes(); + + void ExtractWordCandidates(); + + void CalculateFrequency(); + + void CalculateCohesions(); + + void CalculateSuffixEntropy(); + + void CalculatePrefixEntropy(); + + void SelectWords(); + + static bool + DefaultPreCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + + static bool + DefaultPostCalculationFilter(const PhraseExtract&, + const PhraseExtract::UTF8StringSlice8Bit&); + +private: + class DictType; + + // Pointwise Mutual Information + double PMI(const UTF8StringSlice8Bit& wordCandidate, + const UTF8StringSlice8Bit& part1, + const UTF8StringSlice8Bit& part2) const; + + double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const; + + double CalculateEntropy( + const std::unordered_map& choices) const; + + LengthType wordMinLength; + LengthType wordMaxLength; + LengthType prefixSetLength; + LengthType suffixSetLength; + std::function + preCalculationFilter; + std::function + postCalculationFilter; + + bool prefixesExtracted; + bool suffixesExtracted; + bool frequenciesCalculated; + bool wordCandidatesExtracted; + bool cohesionsCalculated; + bool prefixEntropiesCalculated; + bool suffixEntropiesCalculated; + bool wordsSelected; + + UTF8StringSlice utf8FullText; + size_t totalOccurrence; + double logTotalOccurrence; + std::vector prefixes; + std::vector suffixes; + std::vector wordCandidates; + std::vector words; + DictType* signals; + + friend class PhraseExtractTest; +}; + +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Segmentation.hpp b/opencc/x86_64/include/opencc/Segmentation.hpp new file mode 100644 index 00000000..c260ea6d --- /dev/null +++ b/opencc/x86_64/include/opencc/Segmentation.hpp @@ -0,0 +1,32 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" + +namespace opencc { +/** + * Abstract segmentation + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segmentation { +public: + virtual SegmentsPtr Segment(const std::string& text) const = 0; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/Segments.hpp b/opencc/x86_64/include/opencc/Segments.hpp new file mode 100644 index 00000000..21c39813 --- /dev/null +++ b/opencc/x86_64/include/opencc/Segments.hpp @@ -0,0 +1,114 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "Common.hpp" + +namespace opencc { +/** + * Segmented text + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT Segments { +public: + Segments() {} + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + Segments(std::initializer_list initList) { + for (const std::string& item : initList) { + AddSegment(item); + } + } + + void AddSegment(const char* unmanagedString) { + indexes.push_back(std::make_pair(unmanaged.size(), false)); + unmanaged.push_back(unmanagedString); + } + + void AddSegment(const std::string& str) { + indexes.push_back(std::make_pair(managed.size(), true)); + managed.push_back(str); + } + + class iterator : public std::iterator { + public: + iterator(const Segments* const _segments, size_t _cursor) + : segments(_segments), cursor(_cursor) {} + + iterator& operator++() { + cursor++; + return *this; + } + + bool operator==(const iterator& that) const { + return cursor == that.cursor && segments == that.segments; + } + + bool operator!=(const iterator& that) const { + return !this->operator==(that); + } + + const char* operator*() const { return segments->At(cursor); } + + private: + const Segments* const segments; + size_t cursor; + }; + + const char* At(size_t cursor) const { + const auto& index = indexes[cursor]; + if (index.second) { + return managed[index.first].c_str(); + } else { + return unmanaged[index.first]; + } + } + + size_t Length() const { return indexes.size(); } + + iterator begin() const { return iterator(this, 0); } + + iterator end() const { return iterator(this, indexes.size()); } + + std::string ToString() const { + // TODO implement a nested structure to reduce concatenation, + // like a purely functional differential list + std::ostringstream buffer; + for (const char* segment : *this) { + buffer << segment; + } + return buffer.str(); + } + +private: + Segments(const Segments&) {} + + std::vector unmanaged; + std::vector managed; + // index, managed + std::vector> indexes; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/SerializableDict.hpp b/opencc/x86_64/include/opencc/SerializableDict.hpp new file mode 100644 index 00000000..17cea890 --- /dev/null +++ b/opencc/x86_64/include/opencc/SerializableDict.hpp @@ -0,0 +1,77 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Dict.hpp" + +namespace opencc { +/** + * Serializable dictionary interface + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializableDict { +public: + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(FILE* fp) const = 0; + + /** + * Serializes the dictionary and writes in to a file. + */ + virtual void SerializeToFile(const std::string& fileName) const { + FILE* fp = fopen(fileName.c_str(), "wb"); + if (fp == NULL) { + throw FileNotWritable(fileName); + } + SerializeToFile(fp); + fclose(fp); + } + + template + static bool TryLoadFromFile(const std::string& fileName, + std::shared_ptr* dict) { + FILE* fp = +#ifdef _MSC_VER + // well, the 'GetPlatformString' shall return a 'wstring' + _wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb") +#else + fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb") +#endif // _MSC_VER + ; + + if (fp == NULL) { + return false; + } + std::shared_ptr loadedDict = DICT::NewFromFile(fp); + fclose(fp); + *dict = loadedDict; + return true; + } + + template + static std::shared_ptr NewFromFile(const std::string& fileName) { + std::shared_ptr dict; + if (!TryLoadFromFile(fileName, &dict)) { + throw FileNotFound(fileName); + } + return dict; + } +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/SerializedValues.hpp b/opencc/x86_64/include/opencc/SerializedValues.hpp new file mode 100644 index 00000000..c17210ba --- /dev/null +++ b/opencc/x86_64/include/opencc/SerializedValues.hpp @@ -0,0 +1,50 @@ +/* + * Open Chinese Convert + * + * Copyright 2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Binary format for dictionary values serialization. + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT SerializedValues : public SerializableDict { +public: + SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {} + + virtual ~SerializedValues() {} + + virtual void SerializeToFile(FILE* fp) const; + + static std::shared_ptr NewFromFile(FILE* fp); + + const LexiconPtr& GetLexicon() const { return lexicon; } + + size_t KeyMaxLength() const; + +private: + LexiconPtr lexicon; + + void ConstructBuffer(std::string* valueBuffer, + std::vector* valueBytes, + uint32_t* valueTotalLength) const; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/SimpleConverter.hpp b/opencc/x86_64/include/opencc/SimpleConverter.hpp new file mode 100644 index 00000000..56932b76 --- /dev/null +++ b/opencc/x86_64/include/opencc/SimpleConverter.hpp @@ -0,0 +1,93 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Export.hpp" +#include + +#ifndef __OPENCC_SIMPLECONVERTER_HPP_ +#define __OPENCC_SIMPLECONVERTER_HPP_ + +/** + * @defgroup opencc_simple_api OpenCC C++ Simple API + * + * Simple API in C++ language + */ + +namespace opencc { +/** + * A high level converter + * This interface does not require C++11 to compile. + * @ingroup opencc_simple_api + */ +class OPENCC_EXPORT SimpleConverter { +public: + /** + * Constructor of SimpleConverter + * @param configFileName File name of configuration. + */ + SimpleConverter(const std::string& configFileName); + + ~SimpleConverter(); + + /** + * Converts a text + * @param input Text to be converted. + */ + std::string Convert(const std::string& input) const; + + /** + * Converts a text + * @param input A C-Style std::string (terminated by '\0') to be converted. + */ + std::string Convert(const char* input) const; + + /** + * Converts a text + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + */ + std::string Convert(const char* input, size_t length) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string (terminated by '\0') to be converted. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, char* output) const; + + /** + * Converts a text and writes to an allocated buffer + * Please make sure the buffer has sufficient space. + * @param input A C-Style std::string limited by a given length to be + * converted. + * @param length Maximal length in byte of the input std::string. + * @param output Buffer to write the converted text. + * @return Length of converted text. + */ + size_t Convert(const char* input, size_t length, char* output) const; + +private: + const void* internalData; +}; + +} // namespace opencc + +#endif diff --git a/opencc/x86_64/include/opencc/TextDict.hpp b/opencc/x86_64/include/opencc/TextDict.hpp new file mode 100644 index 00000000..f1cb67d9 --- /dev/null +++ b/opencc/x86_64/include/opencc/TextDict.hpp @@ -0,0 +1,60 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2020 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Common.hpp" +#include "SerializableDict.hpp" + +namespace opencc { +/** + * Text dictionary + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT TextDict : public Dict, public SerializableDict { +public: + /** + * Constructor of TextDict. + * _lexicon must be sorted. + */ + TextDict(const LexiconPtr& _lexicon); + + virtual ~TextDict(); + + virtual size_t KeyMaxLength() const; + + virtual Optional Match(const char* word, size_t len) const; + + virtual LexiconPtr GetLexicon() const; + + virtual void SerializeToFile(FILE* fp) const; + + /** + * Constructs a TextDict from another dictionary. + */ + static TextDictPtr NewFromDict(const Dict& dict); + + static TextDictPtr NewFromFile(FILE* fp); + + static TextDictPtr NewFromSortedFile(FILE* fp); + +private: + const size_t maxLength; + const LexiconPtr lexicon; +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/UTF8StringSlice.hpp b/opencc/x86_64/include/opencc/UTF8StringSlice.hpp new file mode 100644 index 00000000..0c9dfdc6 --- /dev/null +++ b/opencc/x86_64/include/opencc/UTF8StringSlice.hpp @@ -0,0 +1,246 @@ +/* + * Open Chinese Convert + * + * Copyright 2015 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "Common.hpp" +#include "UTF8Util.hpp" + +namespace opencc { + +namespace internal { + +inline size_t FNVHash(const char* text, const size_t byteLength, + const size_t FNV_prime, const size_t FNV_offset_basis) { + size_t hash = FNV_offset_basis; + for (const char* pstr = text; pstr < text + byteLength; pstr++) { + hash ^= *pstr; + hash *= FNV_prime; + } + return hash; +} + +template size_t FNVHash(const char* text, const size_t byteLength); + +template <> +inline size_t FNVHash<4>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 16777619UL, 2166136261UL); +} + +#if SIZE_MAX == 0xffffffffffffffff +template <> +inline size_t FNVHash<8>(const char* text, const size_t byteLength) { + return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL); +} +#endif + +} // namespace internal + +template class UTF8StringSliceBase { +public: + typedef LENGTH_TYPE LengthType; + + UTF8StringSliceBase(const char* _str) + : str(_str), utf8Length(static_cast(UTF8Util::Length(_str))), + byteLength(static_cast(strlen(_str))) {} + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length) + : str(_str), utf8Length(_utf8Length) { + CalculateByteLength(); + } + + UTF8StringSliceBase(const char* _str, const LengthType _utf8Length, + const LengthType _byteLength) + : str(_str), utf8Length(_utf8Length), byteLength(_byteLength) { + CalculateByteLength(); + } + + LengthType UTF8Length() const { return utf8Length; } + + LengthType ByteLength() const { return byteLength; } + + UTF8StringSliceBase Left(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + return UTF8StringSliceBase(str, numberOfCharacters); + } + } + + UTF8StringSliceBase Right(const LengthType numberOfCharacters) const { + if (numberOfCharacters == UTF8Length()) { + return *this; + } else { + const char* pstr = str + byteLength; + for (size_t i = 0; i < numberOfCharacters; i++) { + pstr = UTF8Util::PrevChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + UTF8StringSliceBase SubString(const LengthType offset, + const LengthType numberOfCharacters) const { + if (offset == 0) { + return Left(numberOfCharacters); + } else { + const char* pstr = str; + for (size_t i = 0; i < offset; i++) { + pstr = UTF8Util::NextChar(pstr); + } + return UTF8StringSliceBase(pstr, numberOfCharacters); + } + } + + std::string ToString() const { return std::string(str, str + byteLength); } + + const char* CString() const { return str; } + + LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const { + if (str == that.str) { + return (std::min)(utf8Length, that.utf8Length); + } else { + const char* pstr1 = str; + const char* pstr2 = that.str; + for (size_t length = 0; length < utf8Length && length < that.utf8Length; + length++) { + size_t charLen1 = UTF8Util::NextCharLength(pstr1); + size_t charLen2 = UTF8Util::NextCharLength(pstr2); + if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) { + return length; + } + pstr1 += charLen1; + pstr2 += charLen2; + } + return 0; + } + } + + void MoveRight() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::NextCharLength(str); + str += charLen; + utf8Length--; + byteLength -= charLen; + } + } + + void MoveLeft() { + if (utf8Length > 0) { + const size_t charLen = UTF8Util::PrevCharLength(str + byteLength); + utf8Length--; + byteLength -= charLen; + } + } + + int ReverseCompare(const UTF8StringSliceBase& that) const { + const char* pstr1 = str + byteLength; + const char* pstr2 = that.str + that.byteLength; + const size_t length = (std::min)(utf8Length, that.utf8Length); + for (size_t i = 0; i < length; i++) { + const size_t charLen1 = UTF8Util::PrevCharLength(pstr1); + const size_t charLen2 = UTF8Util::PrevCharLength(pstr2); + pstr1 -= charLen1; + pstr2 -= charLen2; + const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2)); + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } else if (charLen1 < charLen2) { + return -1; + } else if (charLen1 > charLen2) { + return 1; + } + } + if (utf8Length < that.utf8Length) { + return -1; + } else if (utf8Length > that.utf8Length) { + return 1; + } else { + return 0; + } + } + + LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const { + return static_cast( + ToString().find(pattern.str, 0, pattern.byteLength)); + } + + bool operator<(const UTF8StringSliceBase& that) const { + return Compare(that) < 0; + } + + bool operator>(const UTF8StringSliceBase& that) const { + return Compare(that) > 0; + } + + bool operator==(const UTF8StringSliceBase& that) const { + return (str == that.str && utf8Length == that.utf8Length) || + Compare(that) == 0; + } + + bool operator!=(const UTF8StringSliceBase& that) const { + return !this->operator==(that); + } + + class Hasher { + public: + size_t operator()(const UTF8StringSliceBase& text) const { + return internal::FNVHash(text.CString(), + text.ByteLength()); + } + }; + +private: + inline int Compare(const UTF8StringSliceBase& that) const { + int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength)); + if (cmp == 0) { + if (utf8Length < that.utf8Length) { + cmp = -1; + } else if (utf8Length > that.utf8Length) { + cmp = 1; + } else { + cmp = 0; + } + } + return cmp; + } + + void CalculateByteLength() { + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + pstr = UTF8Util::NextChar(pstr); + } + byteLength = static_cast(pstr - str); + } + + const char* str; + LengthType utf8Length; + LengthType byteLength; +}; + +typedef UTF8StringSliceBase UTF8StringSlice; + +template +std::ostream& operator<<(::std::ostream& os, + const UTF8StringSliceBase& str) { + return os << str.ToString(); +} + +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/UTF8Util.hpp b/opencc/x86_64/include/opencc/UTF8Util.hpp new file mode 100644 index 00000000..9d3e39b8 --- /dev/null +++ b/opencc/x86_64/include/opencc/UTF8Util.hpp @@ -0,0 +1,290 @@ +/* + * Open Chinese Convert + * + * Copyright 2013 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef _MSC_VER +#define NOMINMAX +#include +#undef NOMINMAX +#endif // _MSC_VER + +#include + +#include "Common.hpp" +#include "Exception.hpp" + +namespace opencc { +/** + * UTF8 std::string utilities + * @ingroup opencc_cpp_api + */ +class OPENCC_EXPORT UTF8Util { +public: + /** + * Detect UTF8 BOM and skip it. + */ + static void SkipUtf8Bom(FILE* fp); + + /** + * Returns the length in byte for the next UTF8 character. + * On error returns 0. + */ + static size_t NextCharLengthNoException(const char* str) { + char ch = *str; + if ((ch & 0xF0) == 0xE0) { + return 3; + } else if ((ch & 0x80) == 0x00) { + return 1; + } else if ((ch & 0xE0) == 0xC0) { + return 2; + } else if ((ch & 0xF8) == 0xF0) { + return 4; + } else if ((ch & 0xFC) == 0xF8) { + return 5; + } else if ((ch & 0xFE) == 0xFC) { + return 6; + } + return 0; + } + + /** + * Returns the length in byte for the next UTF8 character. + */ + static size_t NextCharLength(const char* str) { + size_t length = NextCharLengthNoException(str); + if (length == 0) { + throw InvalidUTF8(str); + } + return length; + } + + /** + * Returns the length in byte for the previous UTF8 character. + */ + static size_t PrevCharLength(const char* str) { + { + const size_t length = NextCharLengthNoException(str - 3); + if (length == 3) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 1); + if (length == 1) { + return length; + } + } + { + const size_t length = NextCharLengthNoException(str - 2); + if (length == 2) { + return length; + } + } + for (size_t i = 4; i <= 6; i++) { + const size_t length = NextCharLengthNoException(str - i); + if (length == i) { + return length; + } + } + throw InvalidUTF8(str); + } + + /** + * Returns the char* pointer over the next UTF8 character. + */ + static const char* NextChar(const char* str) { + return str + NextCharLength(str); + } + + /** + * Move the char* pointer before the previous UTF8 character. + */ + static const char* PrevChar(const char* str) { + return str - PrevCharLength(str); + } + + /** + * Returns the UTF8 length of a valid UTF8 std::string. + */ + static size_t Length(const char* str) { + size_t length = 0; + while (*str != '\0') { + str = NextChar(str); + length++; + } + return length; + } + + /** + * Finds a character in the same line. + * @param str The text to be searched in. + * @param ch The character to find. + * @return The pointer that points to the found chacter in str or EOL/EOF. + */ + static const char* FindNextInline(const char* str, const char ch) { + while (!IsLineEndingOrFileEnding(*str) && *str != ch) { + str = NextChar(str); + } + return str; + } + + /** + * Returns true if the character is a line ending or end of file. + */ + static bool IsLineEndingOrFileEnding(const char ch) { + return ch == '\0' || ch == '\n' || ch == '\r'; + } + + /** + * Copies a substring with given length to a new string. + */ + static std::string FromSubstr(const char* str, size_t length) { + std::string newStr; + newStr.resize(length); + strncpy(const_cast(newStr.c_str()), str, length); + return newStr; + } + + /** + * Returns true if the given std::string is longer or as long as the given + * length. + */ + static bool NotShorterThan(const char* str, size_t byteLength) { + while (byteLength > 0) { + if (*str == '\0') { + return false; + } + byteLength--; + str++; + } + return true; + } + + /** + * Truncates a std::string with a maximal length in byte. + * No UTF8 character will be broken. + */ + static std::string TruncateUTF8(const char* str, size_t maxByteLength) { + std::string wordTrunc; + if (NotShorterThan(str, maxByteLength)) { + size_t len = 0; + const char* pStr = str; + for (;;) { + const size_t charLength = NextCharLength(pStr); + if (len + charLength > maxByteLength) { + break; + } + pStr += charLength; + len += charLength; + } + wordTrunc = FromSubstr(str, len); + } else { + wordTrunc = str; + } + return wordTrunc; + } + + /** + * Replaces all patterns in a std::string in place. + */ + static void ReplaceAll(std::string& str, const char* from, const char* to) { + std::string::size_type pos = 0; + std::string::size_type fromLen = strlen(from); + std::string::size_type toLen = strlen(to); + while ((pos = str.find(from, pos)) != std::string::npos) { + str.replace(pos, fromLen, to); + pos += toLen; + } + } + + /** + * Joins a std::string vector in to a std::string with a separator. + */ + static std::string Join(const std::vector& strings, + const std::string& separator) { + std::ostringstream buffer; + bool first = true; + for (const auto& str : strings) { + if (!first) { + buffer << separator; + } + buffer << str; + first = false; + } + return buffer.str(); + } + + /** + * Joins a std::string vector in to a std::string. + */ + static std::string Join(const std::vector& strings) { + std::ostringstream buffer; + for (const auto& str : strings) { + buffer << str; + } + return buffer.str(); + } + + static void GetByteMap(const char* str, const size_t utf8Length, + std::vector* byteMap) { + if (byteMap->size() < utf8Length) { + byteMap->resize(utf8Length); + } + const char* pstr = str; + for (size_t i = 0; i < utf8Length; i++) { + (*byteMap)[i] = pstr - str; + pstr = NextChar(pstr); + } + } + +#ifdef _MSC_VER + static std::wstring GetPlatformString(const std::string& str) { + return U8ToU16(str); + } +#else + static std::string GetPlatformString(const std::string& str) { return str; } +#endif // _MSC_VER + +#ifdef _MSC_VER + static std::string U16ToU8(const std::wstring& wstr) { + std::string ret; + int length = static_cast(wstr.length()); + int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0, + NULL, NULL); + if (convcnt > 0) { + ret.resize(convcnt); + WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt, + NULL, NULL); + } + return ret; + } + + static std::wstring U8ToU16(const std::string& str) { + std::wstring ret; + int length = static_cast(str.length()); + int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0); + if (convcnt > 0) { + ret.resize(convcnt); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt); + } + return ret; + } +#endif // _MSC_VER +}; +} // namespace opencc diff --git a/opencc/x86_64/include/opencc/opencc.h b/opencc/x86_64/include/opencc/opencc.h new file mode 100644 index 00000000..bc3a9e97 --- /dev/null +++ b/opencc/x86_64/include/opencc/opencc.h @@ -0,0 +1,161 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2014 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __OPENCC_H_ +#define __OPENCC_H_ + +#ifdef __cplusplus + +#include "Export.hpp" +#include "SimpleConverter.hpp" +#include + +extern "C" { +#else +#include +#endif + +#ifndef OPENCC_EXPORT +#define OPENCC_EXPORT +#endif + +/** + * @defgroup opencc_c_api OpenCC C API + * + * API in C language + */ + +/** + * Filename of default Simplified to Traditional configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json" + +/** + * Filename of default Traditional to Simplified configuration + * + * @ingroup opencc_c_api + */ +#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json" + +/** + * Type of opencc descriptor + * + * @ingroup opencc_c_api + */ +typedef void* opencc_t; + +/** + * Makes an instance of opencc + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open(const char* configFileName); +#ifdef _MSC_VER +/** + * Makes an instance of opencc (wide char / Unicode) + * + * @param configFileName Location of configuration file. If this is set to NULL, + * OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded. + * @return A description pointer of the newly allocated instance of + * opencc. On error the return value will be (opencc_t) -1. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName); +#endif /* _MSC_VER */ + +/** + * Destroys an instance of opencc + * + * @param opencc The description pointer. + * @return 0 on success or non-zero number on failure. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT int opencc_close(opencc_t opencc); + +/** + * Converts UTF-8 std::string + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * @param output The buffer to store converted text. You MUST make sure this + * buffer has sufficient space. + * + * @return The length of converted std::string or (size_t)-1 on error. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc, + const char* input, + size_t length, char* output); + +/** + * Converts UTF-8 std::string + * This function returns an allocated C-Style std::string, which stores + * the converted std::string. + * You MUST call opencc_convert_utf8_free() to release allocated memory. + * + * @param opencc The opencc description pointer. + * @param input The UTF-8 encoded std::string. + * @param length The maximum length in byte to convert. If length is (size_t)-1, + * the whole std::string (terminated by '\0') will be converted. + * + * @return The newly allocated UTF-8 std::string that stores text + * converted, or NULL on error. + * @ingroup opencc_c_api + */ +OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input, + size_t length); + +/** + * Releases allocated buffer by opencc_convert_utf8 + * + * @param str Pointer to the allocated std::string buffer by + * opencc_convert_utf8. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT void opencc_convert_utf8_free(char* str); + +/** + * Returns the last error message + * + * Note that this function is the only one which is NOT thread-safe. + * + * @ingroup opencc_c_api + */ +OPENCC_EXPORT const char* opencc_error(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +/** + * @defgroup opencc_cpp_api OpenCC C++ Comprehensive API + * + * Comprehensive API in C++ language + */ + +#endif diff --git a/opencc/x86_64/include/opencc/opencc_config.h b/opencc/x86_64/include/opencc/opencc_config.h new file mode 100644 index 00000000..8dad4489 --- /dev/null +++ b/opencc/x86_64/include/opencc/opencc_config.h @@ -0,0 +1,21 @@ +/* + * Open Chinese Convert + * + * Copyright 2021 Carbo Kuo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* #undef OPENCC_ENABLE_DARTS */ diff --git a/opencc/x86_64/lib/cmake/opencc/OpenCCConfig.cmake b/opencc/x86_64/lib/cmake/opencc/OpenCCConfig.cmake new file mode 100644 index 00000000..a360c2a4 --- /dev/null +++ b/opencc/x86_64/lib/cmake/opencc/OpenCCConfig.cmake @@ -0,0 +1,31 @@ + +####### Expanded from @PACKAGE_INIT@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run #### +####### The input file was OpenCCConfig.cmake.in ######## + +get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) + +macro(set_and_check _var _file) + set(${_var} "${_file}") + if(NOT EXISTS "${_file}") + message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") + endif() +endmacro() + +macro(check_required_components _NAME) + foreach(comp ${${_NAME}_FIND_COMPONENTS}) + if(NOT ${_NAME}_${comp}_FOUND) + if(${_NAME}_FIND_REQUIRED_${comp}) + set(${_NAME}_FOUND FALSE) + endif() + endif() + endforeach() +endmacro() + +#################################################################################### + +include(${CMAKE_CURRENT_LIST_DIR}/OpenCCTargets.cmake) + +set_and_check(OPENCC_INCLUDE_DIR ${PACKAGE_PREFIX_DIR}/include/opencc) + +check_required_components(OpenCC) diff --git a/opencc/x86_64/lib/cmake/opencc/OpenCCConfigVersion.cmake b/opencc/x86_64/lib/cmake/opencc/OpenCCConfigVersion.cmake new file mode 100644 index 00000000..7fa638f2 --- /dev/null +++ b/opencc/x86_64/lib/cmake/opencc/OpenCCConfigVersion.cmake @@ -0,0 +1,70 @@ +# This is a basic version file for the Config-mode of find_package(). +# It is used by write_basic_package_version_file() as input file for configure_file() +# to create a version-file which can be installed along a config.cmake file. +# +# The created file sets PACKAGE_VERSION_EXACT if the current version string and +# the requested version string are exactly the same and it sets +# PACKAGE_VERSION_COMPATIBLE if the current version is >= requested version, +# but only if the requested major version is the same as the current one. +# The variable CVF_VERSION must be set before calling configure_file(). + + +set(PACKAGE_VERSION "1.1.6") + +if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + + if("1.1.6" MATCHES "^([0-9]+)\\.") + set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}") + if(NOT CVF_VERSION_MAJOR VERSION_EQUAL 0) + string(REGEX REPLACE "^0+" "" CVF_VERSION_MAJOR "${CVF_VERSION_MAJOR}") + endif() + else() + set(CVF_VERSION_MAJOR "1.1.6") + endif() + + if(PACKAGE_FIND_VERSION_RANGE) + # both endpoints of the range must have the expected major version + math (EXPR CVF_VERSION_MAJOR_NEXT "${CVF_VERSION_MAJOR} + 1") + if (NOT PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + OR ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX_MAJOR STREQUAL CVF_VERSION_MAJOR) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND NOT PACKAGE_FIND_VERSION_MAX VERSION_LESS_EQUAL CVF_VERSION_MAJOR_NEXT))) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + elseif(PACKAGE_FIND_VERSION_MIN_MAJOR STREQUAL CVF_VERSION_MAJOR + AND ((PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE" AND PACKAGE_VERSION VERSION_LESS_EQUAL PACKAGE_FIND_VERSION_MAX) + OR (PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE" AND PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MAX))) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + else() + if(PACKAGE_FIND_VERSION_MAJOR STREQUAL CVF_VERSION_MAJOR) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + + if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + endif() + endif() +endif() + + +# if the installed project requested no architecture check, don't perform the check +if("FALSE") + return() +endif() + +# if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it: +if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "" OR "8" STREQUAL "") + return() +endif() + +# check that the installed version has the same 32/64bit-ness as the one which is currently searching: +if(NOT CMAKE_SIZEOF_VOID_P STREQUAL "8") + math(EXPR installedBits "8 * 8") + set(PACKAGE_VERSION "${PACKAGE_VERSION} (${installedBits}bit)") + set(PACKAGE_VERSION_UNSUITABLE TRUE) +endif() diff --git a/opencc/x86_64/lib/cmake/opencc/OpenCCTargets-release.cmake b/opencc/x86_64/lib/cmake/opencc/OpenCCTargets-release.cmake new file mode 100644 index 00000000..199b87f5 --- /dev/null +++ b/opencc/x86_64/lib/cmake/opencc/OpenCCTargets-release.cmake @@ -0,0 +1,29 @@ +#---------------------------------------------------------------- +# Generated CMake target import file for configuration "Release". +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Import target "OpenCC::OpenCC" for configuration "Release" +set_property(TARGET OpenCC::OpenCC APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::OpenCC PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libopencc.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::OpenCC ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::OpenCC "${_IMPORT_PREFIX}/lib/libopencc.a" ) + +# Import target "OpenCC::marisa" for configuration "Release" +set_property(TARGET OpenCC::marisa APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) +set_target_properties(OpenCC::marisa PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libmarisa.a" + ) + +list(APPEND _IMPORT_CHECK_TARGETS OpenCC::marisa ) +list(APPEND _IMPORT_CHECK_FILES_FOR_OpenCC::marisa "${_IMPORT_PREFIX}/lib/libmarisa.a" ) + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) diff --git a/opencc/x86_64/lib/cmake/opencc/OpenCCTargets.cmake b/opencc/x86_64/lib/cmake/opencc/OpenCCTargets.cmake new file mode 100644 index 00000000..db461d58 --- /dev/null +++ b/opencc/x86_64/lib/cmake/opencc/OpenCCTargets.cmake @@ -0,0 +1,102 @@ +# Generated by CMake + +if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS 2.6) + message(FATAL_ERROR "CMake >= 2.6.0 required") +endif() +cmake_policy(PUSH) +cmake_policy(VERSION 2.6...3.20) +#---------------------------------------------------------------- +# Generated CMake target import file. +#---------------------------------------------------------------- + +# Commands may need to know the format version. +set(CMAKE_IMPORT_FILE_VERSION 1) + +# Protect against multiple inclusion, which would fail when already imported targets are added once more. +set(_targetsDefined) +set(_targetsNotDefined) +set(_expectedTargets) +foreach(_expectedTarget OpenCC::OpenCC OpenCC::marisa) + list(APPEND _expectedTargets ${_expectedTarget}) + if(NOT TARGET ${_expectedTarget}) + list(APPEND _targetsNotDefined ${_expectedTarget}) + endif() + if(TARGET ${_expectedTarget}) + list(APPEND _targetsDefined ${_expectedTarget}) + endif() +endforeach() +if("${_targetsDefined}" STREQUAL "${_expectedTargets}") + unset(_targetsDefined) + unset(_targetsNotDefined) + unset(_expectedTargets) + set(CMAKE_IMPORT_FILE_VERSION) + cmake_policy(POP) + return() +endif() +if(NOT "${_targetsDefined}" STREQUAL "") + message(FATAL_ERROR "Some (but not all) targets in this export set were already defined.\nTargets Defined: ${_targetsDefined}\nTargets not yet defined: ${_targetsNotDefined}\n") +endif() +unset(_targetsDefined) +unset(_targetsNotDefined) +unset(_expectedTargets) + + +# Compute the installation prefix relative to this file. +get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +if(_IMPORT_PREFIX STREQUAL "/") + set(_IMPORT_PREFIX "") +endif() + +# Create imported target OpenCC::OpenCC +add_library(OpenCC::OpenCC STATIC IMPORTED) + +set_target_properties(OpenCC::OpenCC PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include/opencc" + INTERFACE_LINK_LIBRARIES "OpenCC::marisa" +) + +# Create imported target OpenCC::marisa +add_library(OpenCC::marisa STATIC IMPORTED) + +if(CMAKE_VERSION VERSION_LESS 2.8.12) + message(FATAL_ERROR "This file relies on consumers using CMake 2.8.12 or greater.") +endif() + +# Load information for each installed configuration. +get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +file(GLOB CONFIG_FILES "${_DIR}/OpenCCTargets-*.cmake") +foreach(f ${CONFIG_FILES}) + include(${f}) +endforeach() + +# Cleanup temporary variables. +set(_IMPORT_PREFIX) + +# Loop over all imported files and verify that they actually exist +foreach(target ${_IMPORT_CHECK_TARGETS} ) + foreach(file ${_IMPORT_CHECK_FILES_FOR_${target}} ) + if(NOT EXISTS "${file}" ) + message(FATAL_ERROR "The imported target \"${target}\" references the file + \"${file}\" +but this file does not exist. Possible reasons include: +* The file was deleted, renamed, or moved to another location. +* An install or uninstall procedure did not complete successfully. +* The installation package was faulty and contained + \"${CMAKE_CURRENT_LIST_FILE}\" +but not all the files it references. +") + endif() + endforeach() + unset(_IMPORT_CHECK_FILES_FOR_${target}) +endforeach() +unset(_IMPORT_CHECK_TARGETS) + +# This file does not depend on other imported targets which have +# been exported from the same project but in a separate export set. + +# Commands beyond this point should not need to know the version. +set(CMAKE_IMPORT_FILE_VERSION) +cmake_policy(POP) diff --git a/opencc/x86_64/lib/libmarisa.a b/opencc/x86_64/lib/libmarisa.a new file mode 100644 index 00000000..64c9a085 Binary files /dev/null and b/opencc/x86_64/lib/libmarisa.a differ diff --git a/opencc/x86_64/lib/libopencc.a b/opencc/x86_64/lib/libopencc.a new file mode 100644 index 00000000..4926abc1 Binary files /dev/null and b/opencc/x86_64/lib/libopencc.a differ diff --git a/opencc/x86_64/share/opencc b/opencc/x86_64/share/opencc new file mode 120000 index 00000000..e67b4559 --- /dev/null +++ b/opencc/x86_64/share/opencc @@ -0,0 +1 @@ +../../data \ No newline at end of file