Auto update
This commit is contained in:
parent
31c40e70b1
commit
95021a13ed
164 changed files with 10625 additions and 0 deletions
82
opencc/x86_64/include/opencc/Common.hpp
Normal file
82
opencc/x86_64/include/opencc/Common.hpp
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
// Microsoft Visual C++ specific
|
||||
#if defined(_MSC_VER) && (_MSC_VER >= 1020)
|
||||
#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820)
|
||||
#endif
|
||||
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Export.hpp"
|
||||
#include "Optional.hpp"
|
||||
#include "opencc_config.h"
|
||||
|
||||
// Forward decalarations and alias
|
||||
namespace opencc {
|
||||
class Config;
|
||||
class Conversion;
|
||||
class ConversionChain;
|
||||
class Converter;
|
||||
class Dict;
|
||||
class DictEntry;
|
||||
class DictGroup;
|
||||
class Lexicon;
|
||||
class MarisaDict;
|
||||
class MultiValueDictEntry;
|
||||
class NoValueDictEntry;
|
||||
class Segmentation;
|
||||
class Segments;
|
||||
class SerializableDict;
|
||||
class SingleValueDictEntry;
|
||||
class TextDict;
|
||||
typedef std::shared_ptr<Conversion> ConversionPtr;
|
||||
typedef std::shared_ptr<ConversionChain> ConversionChainPtr;
|
||||
typedef std::shared_ptr<Converter> ConverterPtr;
|
||||
typedef std::shared_ptr<Dict> DictPtr;
|
||||
typedef std::shared_ptr<DictGroup> DictGroupPtr;
|
||||
typedef std::shared_ptr<Lexicon> LexiconPtr;
|
||||
typedef std::shared_ptr<MarisaDict> MarisaDictPtr;
|
||||
typedef std::shared_ptr<Segmentation> SegmentationPtr;
|
||||
typedef std::shared_ptr<Segments> SegmentsPtr;
|
||||
typedef std::shared_ptr<SerializableDict> SerializableDictPtr;
|
||||
typedef std::shared_ptr<TextDict> TextDictPtr;
|
||||
|
||||
#ifdef OPENCC_ENABLE_DARTS
|
||||
class BinaryDict;
|
||||
class DartsDict;
|
||||
typedef std::shared_ptr<BinaryDict> BinaryDictPtr;
|
||||
typedef std::shared_ptr<DartsDict> DartsDictPtr;
|
||||
#endif
|
||||
|
||||
} // namespace opencc
|
||||
|
||||
#ifndef PKGDATADIR
|
||||
const std::string PACKAGE_DATA_DIRECTORY = "";
|
||||
#else // ifndef PKGDATADIR
|
||||
const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/";
|
||||
#endif // ifndef PKGDATADIR
|
||||
|
||||
#ifndef VERSION
|
||||
#define VERSION "1.0.*"
|
||||
#endif // ifndef VERSION
|
||||
42
opencc/x86_64/include/opencc/Config.hpp
Normal file
42
opencc/x86_64/include/opencc/Config.hpp
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Configuration loader
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Config {
|
||||
public:
|
||||
Config();
|
||||
|
||||
virtual ~Config();
|
||||
|
||||
ConverterPtr NewFromString(const std::string& json,
|
||||
const std::string& configDirectory);
|
||||
|
||||
ConverterPtr NewFromFile(const std::string& fileName);
|
||||
|
||||
private:
|
||||
void* internal;
|
||||
};
|
||||
} // namespace opencc
|
||||
47
opencc/x86_64/include/opencc/Conversion.hpp
Normal file
47
opencc/x86_64/include/opencc/Conversion.hpp
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Segmentation.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Conversion interface
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Conversion {
|
||||
public:
|
||||
Conversion(DictPtr _dict) : dict(_dict) {}
|
||||
|
||||
// Convert single phrase
|
||||
std::string Convert(const std::string& phrase) const;
|
||||
|
||||
// Convert single phrase
|
||||
std::string Convert(const char* phrase) const;
|
||||
|
||||
// Convert segmented text
|
||||
SegmentsPtr Convert(const SegmentsPtr& input) const;
|
||||
|
||||
const DictPtr GetDict() const { return dict; }
|
||||
|
||||
private:
|
||||
const DictPtr dict;
|
||||
};
|
||||
} // namespace opencc
|
||||
43
opencc/x86_64/include/opencc/ConversionChain.hpp
Normal file
43
opencc/x86_64/include/opencc/ConversionChain.hpp
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <list>
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Conversion.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Chain of conversions
|
||||
* Consists of a list of conversions. Converts input in sequence.
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT ConversionChain {
|
||||
public:
|
||||
ConversionChain(const std::list<ConversionPtr> _conversions);
|
||||
|
||||
SegmentsPtr Convert(const SegmentsPtr& input) const;
|
||||
|
||||
const std::list<ConversionPtr> GetConversions() const { return conversions; }
|
||||
|
||||
private:
|
||||
const std::list<ConversionPtr> conversions;
|
||||
};
|
||||
} // namespace opencc
|
||||
51
opencc/x86_64/include/opencc/Converter.hpp
Normal file
51
opencc/x86_64/include/opencc/Converter.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Segmentation.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Controller of segmentation and conversion
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Converter {
|
||||
public:
|
||||
Converter(const std::string& _name, SegmentationPtr _segmentation,
|
||||
ConversionChainPtr _conversionChain)
|
||||
: name(_name), segmentation(_segmentation),
|
||||
conversionChain(_conversionChain) {}
|
||||
|
||||
std::string Convert(const std::string& text) const;
|
||||
|
||||
size_t Convert(const char* input, char* output) const;
|
||||
|
||||
const SegmentationPtr GetSegmentation() const { return segmentation; }
|
||||
|
||||
const ConversionChainPtr GetConversionChain() const {
|
||||
return conversionChain;
|
||||
}
|
||||
|
||||
private:
|
||||
const std::string name;
|
||||
const SegmentationPtr segmentation;
|
||||
const ConversionChainPtr conversionChain;
|
||||
};
|
||||
} // namespace opencc
|
||||
92
opencc/x86_64/include/opencc/Dict.hpp
Normal file
92
opencc/x86_64/include/opencc/Dict.hpp
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "DictEntry.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Abstract class of dictionary
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Dict {
|
||||
public:
|
||||
/**
|
||||
* Matches a word exactly and returns the DictEntry or Optional::Null().
|
||||
*/
|
||||
virtual Optional<const DictEntry*> Match(const char* word,
|
||||
size_t len) const = 0;
|
||||
|
||||
/**
|
||||
* Matches a word exactly and returns the DictEntry or Optional::Null().
|
||||
*/
|
||||
Optional<const DictEntry*> Match(const std::string& word) const {
|
||||
return Match(word.c_str(), word.length());
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches the longest matched prefix of a word.
|
||||
* For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
|
||||
* the longest prefix of "banana" matched is "bana".
|
||||
*/
|
||||
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
/**
|
||||
* Matches the longest matched prefix of a word.
|
||||
*/
|
||||
Optional<const DictEntry*> MatchPrefix(const char* word) const {
|
||||
return MatchPrefix(word, KeyMaxLength());
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches the longest matched prefix of a word.
|
||||
*/
|
||||
Optional<const DictEntry*> MatchPrefix(const std::string& word) const {
|
||||
return MatchPrefix(word.c_str(), word.length());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all matched prefixes of a word, sorted by the length (desc).
|
||||
* For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
|
||||
* all the matched prefixes of "banana" are "bana", "ban", "ba", "b".
|
||||
*/
|
||||
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
/**
|
||||
* Returns all matched prefixes of a word, sorted by the length (desc).
|
||||
*/
|
||||
std::vector<const DictEntry*>
|
||||
MatchAllPrefixes(const std::string& word) const {
|
||||
return MatchAllPrefixes(word.c_str(), word.length());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length of the longest key in the dictionary.
|
||||
*/
|
||||
virtual size_t KeyMaxLength() const = 0;
|
||||
|
||||
/**
|
||||
* Returns all entries in the dictionary.
|
||||
*/
|
||||
virtual LexiconPtr GetLexicon() const = 0;
|
||||
};
|
||||
} // namespace opencc
|
||||
32
opencc/x86_64/include/opencc/DictConverter.hpp
Normal file
32
opencc/x86_64/include/opencc/DictConverter.hpp
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2017 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Converts a dictionary from a format to another.
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName,
|
||||
const std::string& outputFileName,
|
||||
const std::string& formatFrom,
|
||||
const std::string& formatTo);
|
||||
} // namespace opencc
|
||||
173
opencc/x86_64/include/opencc/DictEntry.hpp
Normal file
173
opencc/x86_64/include/opencc/DictEntry.hpp
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Segments.hpp"
|
||||
#include "UTF8Util.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Key-values pair entry
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT DictEntry {
|
||||
public:
|
||||
virtual ~DictEntry() {}
|
||||
|
||||
virtual std::string Key() const = 0;
|
||||
|
||||
virtual std::vector<std::string> Values() const = 0;
|
||||
|
||||
virtual std::string GetDefault() const = 0;
|
||||
|
||||
virtual size_t NumValues() const = 0;
|
||||
|
||||
virtual std::string ToString() const = 0;
|
||||
|
||||
size_t KeyLength() const { return Key().length(); }
|
||||
|
||||
bool operator<(const DictEntry& that) const { return Key() < that.Key(); }
|
||||
|
||||
bool operator==(const DictEntry& that) const { return Key() == that.Key(); }
|
||||
|
||||
static bool UPtrLessThan(const std::unique_ptr<DictEntry>& a,
|
||||
const std::unique_ptr<DictEntry>& b) {
|
||||
return *a < *b;
|
||||
}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT NoValueDictEntry : public DictEntry {
|
||||
public:
|
||||
NoValueDictEntry(const std::string& _key) : key(_key) {}
|
||||
|
||||
virtual ~NoValueDictEntry() {}
|
||||
|
||||
virtual std::string Key() const { return key; }
|
||||
|
||||
virtual std::vector<std::string> Values() const {
|
||||
return std::vector<std::string>();
|
||||
}
|
||||
|
||||
virtual std::string GetDefault() const { return key; }
|
||||
|
||||
virtual size_t NumValues() const { return 0; }
|
||||
|
||||
virtual std::string ToString() const { return key; }
|
||||
|
||||
private:
|
||||
std::string key;
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT SingleValueDictEntry : public DictEntry {
|
||||
public:
|
||||
virtual std::string Value() const = 0;
|
||||
|
||||
virtual std::vector<std::string> Values() const {
|
||||
return std::vector<std::string>{Value()};
|
||||
}
|
||||
|
||||
virtual std::string GetDefault() const { return Value(); }
|
||||
|
||||
virtual size_t NumValues() const { return 1; }
|
||||
|
||||
virtual std::string ToString() const {
|
||||
return std::string(Key()) + "\t" + Value();
|
||||
}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry {
|
||||
public:
|
||||
StrSingleValueDictEntry(const std::string& _key, const std::string& _value)
|
||||
: key(_key), value(_value) {}
|
||||
|
||||
virtual ~StrSingleValueDictEntry() {}
|
||||
|
||||
virtual std::string Key() const { return key; }
|
||||
|
||||
virtual std::string Value() const { return value; }
|
||||
|
||||
private:
|
||||
std::string key;
|
||||
std::string value;
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT MultiValueDictEntry : public DictEntry {
|
||||
public:
|
||||
virtual std::string GetDefault() const {
|
||||
if (NumValues() > 0) {
|
||||
return Values().at(0);
|
||||
} else {
|
||||
return Key();
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::string ToString() const;
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry {
|
||||
public:
|
||||
StrMultiValueDictEntry(const std::string& _key,
|
||||
const std::vector<std::string>& _values)
|
||||
: key(_key), values(_values) {}
|
||||
|
||||
virtual ~StrMultiValueDictEntry() {}
|
||||
|
||||
virtual std::string Key() const { return key; }
|
||||
|
||||
size_t NumValues() const { return values.size(); }
|
||||
|
||||
std::vector<std::string> Values() const { return values; }
|
||||
|
||||
private:
|
||||
std::string key;
|
||||
std::vector<std::string> values;
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT DictEntryFactory {
|
||||
public:
|
||||
static DictEntry* New(const std::string& key) {
|
||||
return new NoValueDictEntry(key);
|
||||
}
|
||||
|
||||
static DictEntry* New(const std::string& key, const std::string& value) {
|
||||
return new StrSingleValueDictEntry(key, value);
|
||||
}
|
||||
|
||||
static DictEntry* New(const std::string& key,
|
||||
const std::vector<std::string>& values) {
|
||||
if (values.size() == 0) {
|
||||
return New(key);
|
||||
} else if (values.size() == 1) {
|
||||
return New(key, values.front());
|
||||
}
|
||||
return new StrMultiValueDictEntry(key, values);
|
||||
}
|
||||
|
||||
static DictEntry* New(const DictEntry* entry) {
|
||||
if (entry->NumValues() == 0) {
|
||||
return new NoValueDictEntry(entry->Key());
|
||||
} else if (entry->NumValues() == 1) {
|
||||
return new StrSingleValueDictEntry(entry->Key(), entry->Values().front());
|
||||
} else {
|
||||
return new StrMultiValueDictEntry(entry->Key(), entry->Values());
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace opencc
|
||||
57
opencc/x86_64/include/opencc/DictGroup.hpp
Normal file
57
opencc/x86_64/include/opencc/DictGroup.hpp
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <list>
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Dict.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Group of dictionaries
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT DictGroup : public Dict {
|
||||
public:
|
||||
DictGroup(const std::list<DictPtr>& dicts);
|
||||
|
||||
static DictGroupPtr NewFromDict(const Dict& dict);
|
||||
|
||||
virtual ~DictGroup();
|
||||
|
||||
virtual size_t KeyMaxLength() const;
|
||||
|
||||
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
|
||||
|
||||
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
virtual LexiconPtr GetLexicon() const;
|
||||
|
||||
const std::list<DictPtr> GetDicts() const { return dicts; }
|
||||
|
||||
private:
|
||||
const size_t keyMaxLength;
|
||||
const std::list<DictPtr> dicts;
|
||||
};
|
||||
} // namespace opencc
|
||||
88
opencc/x86_64/include/opencc/Exception.hpp
Normal file
88
opencc/x86_64/include/opencc/Exception.hpp
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include "Export.hpp"
|
||||
|
||||
#if defined(_MSC_VER) && _MSC_VER < 1900
|
||||
// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not
|
||||
// supported
|
||||
#define noexcept
|
||||
#endif // ifdef _MSC_VER
|
||||
|
||||
namespace opencc {
|
||||
|
||||
class OPENCC_EXPORT Exception {
|
||||
public:
|
||||
Exception() {}
|
||||
|
||||
virtual ~Exception() throw() {}
|
||||
|
||||
Exception(const std::string& _message) : message(_message) {}
|
||||
|
||||
virtual const char* what() const noexcept { return message.c_str(); }
|
||||
|
||||
protected:
|
||||
std::string message;
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT FileNotFound : public Exception {
|
||||
public:
|
||||
FileNotFound(const std::string& fileName)
|
||||
: Exception(fileName + " not found or not accessible.") {}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT FileNotWritable : public Exception {
|
||||
public:
|
||||
FileNotWritable(const std::string& fileName)
|
||||
: Exception(fileName + " not writable.") {}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT InvalidFormat : public Exception {
|
||||
public:
|
||||
InvalidFormat(const std::string& message)
|
||||
: Exception("Invalid format: " + message) {}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat {
|
||||
public:
|
||||
InvalidTextDictionary(const std::string& _message, size_t lineNum)
|
||||
: InvalidFormat("") {
|
||||
std::ostringstream buffer;
|
||||
buffer << "Invalid text dictionary at line " << lineNum << ": " << _message;
|
||||
message = buffer.str();
|
||||
}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT InvalidUTF8 : public Exception {
|
||||
public:
|
||||
InvalidUTF8(const std::string& _message)
|
||||
: Exception("Invalid UTF8: " + _message) {}
|
||||
};
|
||||
|
||||
class OPENCC_EXPORT ShouldNotBeHere : public Exception {
|
||||
public:
|
||||
ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {}
|
||||
};
|
||||
|
||||
} // namespace opencc
|
||||
40
opencc/x86_64/include/opencc/Export.hpp
Normal file
40
opencc/x86_64/include/opencc/Export.hpp
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
|
||||
#define OPENCC_EXPORT
|
||||
#define OPENCC_NO_EXPORT
|
||||
#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
|
||||
#ifndef OPENCC_EXPORT
|
||||
#ifdef libopencc_EXPORTS
|
||||
|
||||
/* We are building this library */
|
||||
#define OPENCC_EXPORT __declspec(dllexport)
|
||||
#else // ifdef libopencc_EXPORTS
|
||||
|
||||
/* We are using this library */
|
||||
#define OPENCC_EXPORT __declspec(dllimport)
|
||||
#endif // ifdef libopencc_EXPORTS
|
||||
#endif // ifndef OPENCC_EXPORT
|
||||
|
||||
#ifndef OPENCC_NO_EXPORT
|
||||
#define OPENCC_NO_EXPORT
|
||||
#endif // ifndef OPENCC_NO_EXPORT
|
||||
#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
|
||||
68
opencc/x86_64/include/opencc/Lexicon.hpp
Normal file
68
opencc/x86_64/include/opencc/Lexicon.hpp
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "DictEntry.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Storage of all entries
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Lexicon {
|
||||
public:
|
||||
Lexicon() {}
|
||||
Lexicon(std::vector<std::unique_ptr<DictEntry>> entries_)
|
||||
: entries(std::move(entries_)) {}
|
||||
Lexicon(const Lexicon&) = delete;
|
||||
Lexicon& operator=(const Lexicon&) = delete;
|
||||
|
||||
// Lexicon will take the ownership of the entry.
|
||||
void Add(DictEntry* entry) { entries.emplace_back(entry); }
|
||||
|
||||
void Add(std::unique_ptr<DictEntry> entry) {
|
||||
entries.push_back(std::move(entry));
|
||||
}
|
||||
|
||||
void Sort();
|
||||
|
||||
// Returns true if the lexicon is sorted by key.
|
||||
bool IsSorted();
|
||||
|
||||
// Returns true if every key unique (after sorted).
|
||||
// When dupkey is set, it is set to the duplicate key.
|
||||
bool IsUnique(std::string* dupkey = nullptr);
|
||||
|
||||
const DictEntry* At(size_t index) const { return entries.at(index).get(); }
|
||||
|
||||
size_t Length() const { return entries.size(); }
|
||||
|
||||
std::vector<std::unique_ptr<DictEntry>>::const_iterator begin() const {
|
||||
return entries.begin();
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<DictEntry>>::const_iterator end() const {
|
||||
return entries.end();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::unique_ptr<DictEntry>> entries;
|
||||
};
|
||||
} // namespace opencc
|
||||
63
opencc/x86_64/include/opencc/MarisaDict.hpp
Normal file
63
opencc/x86_64/include/opencc/MarisaDict.hpp
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "SerializableDict.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Darts dictionary
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict {
|
||||
public:
|
||||
virtual ~MarisaDict();
|
||||
|
||||
virtual size_t KeyMaxLength() const;
|
||||
|
||||
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
|
||||
|
||||
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
|
||||
size_t len) const;
|
||||
|
||||
virtual LexiconPtr GetLexicon() const;
|
||||
|
||||
virtual void SerializeToFile(FILE* fp) const;
|
||||
|
||||
/**
|
||||
* Constructs a MarisaDict from another dictionary.
|
||||
*/
|
||||
static MarisaDictPtr NewFromDict(const Dict& thatDict);
|
||||
|
||||
static MarisaDictPtr NewFromFile(FILE* fp);
|
||||
|
||||
private:
|
||||
MarisaDict();
|
||||
|
||||
size_t maxLength;
|
||||
LexiconPtr lexicon;
|
||||
|
||||
class MarisaInternal;
|
||||
std::unique_ptr<MarisaInternal> internal;
|
||||
};
|
||||
} // namespace opencc
|
||||
43
opencc/x86_64/include/opencc/MaxMatchSegmentation.hpp
Normal file
43
opencc/x86_64/include/opencc/MaxMatchSegmentation.hpp
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "DictGroup.hpp"
|
||||
#include "Segmentation.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Implementation of maximal match segmentation
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation {
|
||||
public:
|
||||
MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {}
|
||||
|
||||
virtual ~MaxMatchSegmentation() {}
|
||||
|
||||
virtual SegmentsPtr Segment(const std::string& text) const;
|
||||
|
||||
const DictPtr GetDict() const { return dict; }
|
||||
|
||||
private:
|
||||
const DictPtr dict;
|
||||
};
|
||||
} // namespace opencc
|
||||
76
opencc/x86_64/include/opencc/Optional.hpp
Normal file
76
opencc/x86_64/include/opencc/Optional.hpp
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* A class that wraps type T into a nullable type.
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
template <typename T> class Optional {
|
||||
public:
|
||||
/**
|
||||
* The constructor of Optional.
|
||||
*/
|
||||
Optional(T actual) : isNull(false), data(actual) {}
|
||||
|
||||
/**
|
||||
* Returns true if the instance is null.
|
||||
*/
|
||||
bool IsNull() const { return isNull; }
|
||||
|
||||
/**
|
||||
* Returns the containing data of the instance.
|
||||
*/
|
||||
const T& Get() const { return data; }
|
||||
|
||||
/**
|
||||
* Constructs a null instance.
|
||||
*/
|
||||
static Optional<T> Null() { return Optional(); }
|
||||
|
||||
private:
|
||||
Optional() : isNull(true) {}
|
||||
|
||||
bool isNull;
|
||||
T data;
|
||||
};
|
||||
|
||||
/**
|
||||
* Specialization of Optional for pointers.
|
||||
*
|
||||
* Reduce a bool.
|
||||
*/
|
||||
template <typename T> class Optional<T*> {
|
||||
private:
|
||||
Optional() : data(nullptr) {}
|
||||
|
||||
typedef T* TPtr;
|
||||
TPtr data;
|
||||
|
||||
public:
|
||||
Optional(TPtr actual) : data(actual) {}
|
||||
|
||||
bool IsNull() const { return data == nullptr; }
|
||||
|
||||
const TPtr& Get() const { return data; }
|
||||
|
||||
static Optional<TPtr> Null() { return Optional(); }
|
||||
};
|
||||
} // namespace opencc
|
||||
195
opencc/x86_64/include/opencc/PhraseExtract.hpp
Normal file
195
opencc/x86_64/include/opencc/PhraseExtract.hpp
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "UTF8StringSlice.hpp"
|
||||
|
||||
namespace opencc {
|
||||
|
||||
class OPENCC_EXPORT PhraseExtract {
|
||||
public:
|
||||
typedef UTF8StringSlice::LengthType LengthType;
|
||||
|
||||
typedef UTF8StringSliceBase<unsigned char> UTF8StringSlice8Bit;
|
||||
|
||||
PhraseExtract();
|
||||
|
||||
virtual ~PhraseExtract();
|
||||
|
||||
void Extract(const std::string& text) {
|
||||
SetFullText(text);
|
||||
ExtractSuffixes();
|
||||
CalculateFrequency();
|
||||
CalculateSuffixEntropy();
|
||||
ReleaseSuffixes();
|
||||
ExtractPrefixes();
|
||||
CalculatePrefixEntropy();
|
||||
ReleasePrefixes();
|
||||
ExtractWordCandidates();
|
||||
CalculateCohesions();
|
||||
SelectWords();
|
||||
}
|
||||
|
||||
void SetFullText(const std::string& fullText) {
|
||||
utf8FullText = UTF8StringSlice(fullText.c_str());
|
||||
}
|
||||
|
||||
void SetFullText(const char* fullText) {
|
||||
utf8FullText = UTF8StringSlice(fullText);
|
||||
}
|
||||
|
||||
void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; }
|
||||
|
||||
void SetWordMinLength(const LengthType _wordMinLength) {
|
||||
wordMinLength = _wordMinLength;
|
||||
}
|
||||
|
||||
void SetWordMaxLength(const LengthType _wordMaxLength) {
|
||||
wordMaxLength = _wordMaxLength;
|
||||
}
|
||||
|
||||
void SetPrefixSetLength(const LengthType _prefixSetLength) {
|
||||
prefixSetLength = _prefixSetLength;
|
||||
}
|
||||
|
||||
void SetSuffixSetLength(const LengthType _suffixSetLength) {
|
||||
suffixSetLength = _suffixSetLength;
|
||||
}
|
||||
|
||||
// PreCalculationFilter is called after frequencies statistics.
|
||||
void SetPreCalculationFilter(
|
||||
const std::function<bool(const PhraseExtract&,
|
||||
const UTF8StringSlice8Bit&)>& filter) {
|
||||
preCalculationFilter = filter;
|
||||
}
|
||||
|
||||
void SetPostCalculationFilter(
|
||||
const std::function<bool(const PhraseExtract&,
|
||||
const UTF8StringSlice8Bit&)>& filter) {
|
||||
postCalculationFilter = filter;
|
||||
}
|
||||
|
||||
void ReleaseSuffixes() { std::vector<UTF8StringSlice8Bit>().swap(suffixes); }
|
||||
|
||||
void ReleasePrefixes() { std::vector<UTF8StringSlice8Bit>().swap(prefixes); }
|
||||
|
||||
const std::vector<UTF8StringSlice8Bit>& Words() const { return words; }
|
||||
|
||||
const std::vector<UTF8StringSlice8Bit>& WordCandidates() const {
|
||||
return wordCandidates;
|
||||
}
|
||||
|
||||
struct Signals {
|
||||
size_t frequency;
|
||||
double cohesion;
|
||||
double suffixEntropy;
|
||||
double prefixEntropy;
|
||||
};
|
||||
|
||||
const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
double Entropy(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
size_t Frequency(const UTF8StringSlice8Bit& word) const;
|
||||
|
||||
double Probability(const UTF8StringSlice8Bit& word) const;
|
||||
|
||||
double LogProbability(const UTF8StringSlice8Bit& word) const;
|
||||
|
||||
void Reset();
|
||||
|
||||
void ExtractSuffixes();
|
||||
|
||||
void ExtractPrefixes();
|
||||
|
||||
void ExtractWordCandidates();
|
||||
|
||||
void CalculateFrequency();
|
||||
|
||||
void CalculateCohesions();
|
||||
|
||||
void CalculateSuffixEntropy();
|
||||
|
||||
void CalculatePrefixEntropy();
|
||||
|
||||
void SelectWords();
|
||||
|
||||
static bool
|
||||
DefaultPreCalculationFilter(const PhraseExtract&,
|
||||
const PhraseExtract::UTF8StringSlice8Bit&);
|
||||
|
||||
static bool
|
||||
DefaultPostCalculationFilter(const PhraseExtract&,
|
||||
const PhraseExtract::UTF8StringSlice8Bit&);
|
||||
|
||||
private:
|
||||
class DictType;
|
||||
|
||||
// Pointwise Mutual Information
|
||||
double PMI(const UTF8StringSlice8Bit& wordCandidate,
|
||||
const UTF8StringSlice8Bit& part1,
|
||||
const UTF8StringSlice8Bit& part2) const;
|
||||
|
||||
double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const;
|
||||
|
||||
double CalculateEntropy(
|
||||
const std::unordered_map<UTF8StringSlice8Bit, size_t,
|
||||
UTF8StringSlice8Bit::Hasher>& choices) const;
|
||||
|
||||
LengthType wordMinLength;
|
||||
LengthType wordMaxLength;
|
||||
LengthType prefixSetLength;
|
||||
LengthType suffixSetLength;
|
||||
std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
|
||||
preCalculationFilter;
|
||||
std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
|
||||
postCalculationFilter;
|
||||
|
||||
bool prefixesExtracted;
|
||||
bool suffixesExtracted;
|
||||
bool frequenciesCalculated;
|
||||
bool wordCandidatesExtracted;
|
||||
bool cohesionsCalculated;
|
||||
bool prefixEntropiesCalculated;
|
||||
bool suffixEntropiesCalculated;
|
||||
bool wordsSelected;
|
||||
|
||||
UTF8StringSlice utf8FullText;
|
||||
size_t totalOccurrence;
|
||||
double logTotalOccurrence;
|
||||
std::vector<UTF8StringSlice8Bit> prefixes;
|
||||
std::vector<UTF8StringSlice8Bit> suffixes;
|
||||
std::vector<UTF8StringSlice8Bit> wordCandidates;
|
||||
std::vector<UTF8StringSlice8Bit> words;
|
||||
DictType* signals;
|
||||
|
||||
friend class PhraseExtractTest;
|
||||
};
|
||||
|
||||
} // namespace opencc
|
||||
32
opencc/x86_64/include/opencc/Segmentation.hpp
Normal file
32
opencc/x86_64/include/opencc/Segmentation.hpp
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Abstract segmentation
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Segmentation {
|
||||
public:
|
||||
virtual SegmentsPtr Segment(const std::string& text) const = 0;
|
||||
};
|
||||
} // namespace opencc
|
||||
114
opencc/x86_64/include/opencc/Segments.hpp
Normal file
114
opencc/x86_64/include/opencc/Segments.hpp
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "Common.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Segmented text
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT Segments {
|
||||
public:
|
||||
Segments() {}
|
||||
|
||||
Segments(std::initializer_list<const char*> initList) {
|
||||
for (const std::string& item : initList) {
|
||||
AddSegment(item);
|
||||
}
|
||||
}
|
||||
|
||||
Segments(std::initializer_list<std::string> initList) {
|
||||
for (const std::string& item : initList) {
|
||||
AddSegment(item);
|
||||
}
|
||||
}
|
||||
|
||||
void AddSegment(const char* unmanagedString) {
|
||||
indexes.push_back(std::make_pair(unmanaged.size(), false));
|
||||
unmanaged.push_back(unmanagedString);
|
||||
}
|
||||
|
||||
void AddSegment(const std::string& str) {
|
||||
indexes.push_back(std::make_pair(managed.size(), true));
|
||||
managed.push_back(str);
|
||||
}
|
||||
|
||||
class iterator : public std::iterator<std::input_iterator_tag, const char*> {
|
||||
public:
|
||||
iterator(const Segments* const _segments, size_t _cursor)
|
||||
: segments(_segments), cursor(_cursor) {}
|
||||
|
||||
iterator& operator++() {
|
||||
cursor++;
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool operator==(const iterator& that) const {
|
||||
return cursor == that.cursor && segments == that.segments;
|
||||
}
|
||||
|
||||
bool operator!=(const iterator& that) const {
|
||||
return !this->operator==(that);
|
||||
}
|
||||
|
||||
const char* operator*() const { return segments->At(cursor); }
|
||||
|
||||
private:
|
||||
const Segments* const segments;
|
||||
size_t cursor;
|
||||
};
|
||||
|
||||
const char* At(size_t cursor) const {
|
||||
const auto& index = indexes[cursor];
|
||||
if (index.second) {
|
||||
return managed[index.first].c_str();
|
||||
} else {
|
||||
return unmanaged[index.first];
|
||||
}
|
||||
}
|
||||
|
||||
size_t Length() const { return indexes.size(); }
|
||||
|
||||
iterator begin() const { return iterator(this, 0); }
|
||||
|
||||
iterator end() const { return iterator(this, indexes.size()); }
|
||||
|
||||
std::string ToString() const {
|
||||
// TODO implement a nested structure to reduce concatenation,
|
||||
// like a purely functional differential list
|
||||
std::ostringstream buffer;
|
||||
for (const char* segment : *this) {
|
||||
buffer << segment;
|
||||
}
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
private:
|
||||
Segments(const Segments&) {}
|
||||
|
||||
std::vector<const char*> unmanaged;
|
||||
std::vector<std::string> managed;
|
||||
// index, managed
|
||||
std::vector<std::pair<size_t, bool>> indexes;
|
||||
};
|
||||
} // namespace opencc
|
||||
77
opencc/x86_64/include/opencc/SerializableDict.hpp
Normal file
77
opencc/x86_64/include/opencc/SerializableDict.hpp
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Dict.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Serializable dictionary interface
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT SerializableDict {
|
||||
public:
|
||||
/**
|
||||
* Serializes the dictionary and writes in to a file.
|
||||
*/
|
||||
virtual void SerializeToFile(FILE* fp) const = 0;
|
||||
|
||||
/**
|
||||
* Serializes the dictionary and writes in to a file.
|
||||
*/
|
||||
virtual void SerializeToFile(const std::string& fileName) const {
|
||||
FILE* fp = fopen(fileName.c_str(), "wb");
|
||||
if (fp == NULL) {
|
||||
throw FileNotWritable(fileName);
|
||||
}
|
||||
SerializeToFile(fp);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
template <typename DICT>
|
||||
static bool TryLoadFromFile(const std::string& fileName,
|
||||
std::shared_ptr<DICT>* dict) {
|
||||
FILE* fp =
|
||||
#ifdef _MSC_VER
|
||||
// well, the 'GetPlatformString' shall return a 'wstring'
|
||||
_wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb")
|
||||
#else
|
||||
fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb")
|
||||
#endif // _MSC_VER
|
||||
;
|
||||
|
||||
if (fp == NULL) {
|
||||
return false;
|
||||
}
|
||||
std::shared_ptr<DICT> loadedDict = DICT::NewFromFile(fp);
|
||||
fclose(fp);
|
||||
*dict = loadedDict;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename DICT>
|
||||
static std::shared_ptr<DICT> NewFromFile(const std::string& fileName) {
|
||||
std::shared_ptr<DICT> dict;
|
||||
if (!TryLoadFromFile<DICT>(fileName, &dict)) {
|
||||
throw FileNotFound(fileName);
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
};
|
||||
} // namespace opencc
|
||||
50
opencc/x86_64/include/opencc/SerializedValues.hpp
Normal file
50
opencc/x86_64/include/opencc/SerializedValues.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "SerializableDict.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Binary format for dictionary values serialization.
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT SerializedValues : public SerializableDict {
|
||||
public:
|
||||
SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {}
|
||||
|
||||
virtual ~SerializedValues() {}
|
||||
|
||||
virtual void SerializeToFile(FILE* fp) const;
|
||||
|
||||
static std::shared_ptr<SerializedValues> NewFromFile(FILE* fp);
|
||||
|
||||
const LexiconPtr& GetLexicon() const { return lexicon; }
|
||||
|
||||
size_t KeyMaxLength() const;
|
||||
|
||||
private:
|
||||
LexiconPtr lexicon;
|
||||
|
||||
void ConstructBuffer(std::string* valueBuffer,
|
||||
std::vector<uint16_t>* valueBytes,
|
||||
uint32_t* valueTotalLength) const;
|
||||
};
|
||||
} // namespace opencc
|
||||
93
opencc/x86_64/include/opencc/SimpleConverter.hpp
Normal file
93
opencc/x86_64/include/opencc/SimpleConverter.hpp
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "Export.hpp"
|
||||
#include <string>
|
||||
|
||||
#ifndef __OPENCC_SIMPLECONVERTER_HPP_
|
||||
#define __OPENCC_SIMPLECONVERTER_HPP_
|
||||
|
||||
/**
|
||||
* @defgroup opencc_simple_api OpenCC C++ Simple API
|
||||
*
|
||||
* Simple API in C++ language
|
||||
*/
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* A high level converter
|
||||
* This interface does not require C++11 to compile.
|
||||
* @ingroup opencc_simple_api
|
||||
*/
|
||||
class OPENCC_EXPORT SimpleConverter {
|
||||
public:
|
||||
/**
|
||||
* Constructor of SimpleConverter
|
||||
* @param configFileName File name of configuration.
|
||||
*/
|
||||
SimpleConverter(const std::string& configFileName);
|
||||
|
||||
~SimpleConverter();
|
||||
|
||||
/**
|
||||
* Converts a text
|
||||
* @param input Text to be converted.
|
||||
*/
|
||||
std::string Convert(const std::string& input) const;
|
||||
|
||||
/**
|
||||
* Converts a text
|
||||
* @param input A C-Style std::string (terminated by '\0') to be converted.
|
||||
*/
|
||||
std::string Convert(const char* input) const;
|
||||
|
||||
/**
|
||||
* Converts a text
|
||||
* @param input A C-Style std::string limited by a given length to be
|
||||
* converted.
|
||||
* @param length Maximal length in byte of the input std::string.
|
||||
*/
|
||||
std::string Convert(const char* input, size_t length) const;
|
||||
|
||||
/**
|
||||
* Converts a text and writes to an allocated buffer
|
||||
* Please make sure the buffer has sufficient space.
|
||||
* @param input A C-Style std::string (terminated by '\0') to be converted.
|
||||
* @param output Buffer to write the converted text.
|
||||
* @return Length of converted text.
|
||||
*/
|
||||
size_t Convert(const char* input, char* output) const;
|
||||
|
||||
/**
|
||||
* Converts a text and writes to an allocated buffer
|
||||
* Please make sure the buffer has sufficient space.
|
||||
* @param input A C-Style std::string limited by a given length to be
|
||||
* converted.
|
||||
* @param length Maximal length in byte of the input std::string.
|
||||
* @param output Buffer to write the converted text.
|
||||
* @return Length of converted text.
|
||||
*/
|
||||
size_t Convert(const char* input, size_t length, char* output) const;
|
||||
|
||||
private:
|
||||
const void* internalData;
|
||||
};
|
||||
|
||||
} // namespace opencc
|
||||
|
||||
#endif
|
||||
60
opencc/x86_64/include/opencc/TextDict.hpp
Normal file
60
opencc/x86_64/include/opencc/TextDict.hpp
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "SerializableDict.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* Text dictionary
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT TextDict : public Dict, public SerializableDict {
|
||||
public:
|
||||
/**
|
||||
* Constructor of TextDict.
|
||||
* _lexicon must be sorted.
|
||||
*/
|
||||
TextDict(const LexiconPtr& _lexicon);
|
||||
|
||||
virtual ~TextDict();
|
||||
|
||||
virtual size_t KeyMaxLength() const;
|
||||
|
||||
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
|
||||
|
||||
virtual LexiconPtr GetLexicon() const;
|
||||
|
||||
virtual void SerializeToFile(FILE* fp) const;
|
||||
|
||||
/**
|
||||
* Constructs a TextDict from another dictionary.
|
||||
*/
|
||||
static TextDictPtr NewFromDict(const Dict& dict);
|
||||
|
||||
static TextDictPtr NewFromFile(FILE* fp);
|
||||
|
||||
static TextDictPtr NewFromSortedFile(FILE* fp);
|
||||
|
||||
private:
|
||||
const size_t maxLength;
|
||||
const LexiconPtr lexicon;
|
||||
};
|
||||
} // namespace opencc
|
||||
246
opencc/x86_64/include/opencc/UTF8StringSlice.hpp
Normal file
246
opencc/x86_64/include/opencc/UTF8StringSlice.hpp
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "UTF8Util.hpp"
|
||||
|
||||
namespace opencc {
|
||||
|
||||
namespace internal {
|
||||
|
||||
inline size_t FNVHash(const char* text, const size_t byteLength,
|
||||
const size_t FNV_prime, const size_t FNV_offset_basis) {
|
||||
size_t hash = FNV_offset_basis;
|
||||
for (const char* pstr = text; pstr < text + byteLength; pstr++) {
|
||||
hash ^= *pstr;
|
||||
hash *= FNV_prime;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
template <int> size_t FNVHash(const char* text, const size_t byteLength);
|
||||
|
||||
template <>
|
||||
inline size_t FNVHash<4>(const char* text, const size_t byteLength) {
|
||||
return FNVHash(text, byteLength, 16777619UL, 2166136261UL);
|
||||
}
|
||||
|
||||
#if SIZE_MAX == 0xffffffffffffffff
|
||||
template <>
|
||||
inline size_t FNVHash<8>(const char* text, const size_t byteLength) {
|
||||
return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace internal
|
||||
|
||||
template <typename LENGTH_TYPE> class UTF8StringSliceBase {
|
||||
public:
|
||||
typedef LENGTH_TYPE LengthType;
|
||||
|
||||
UTF8StringSliceBase(const char* _str)
|
||||
: str(_str), utf8Length(static_cast<LengthType>(UTF8Util::Length(_str))),
|
||||
byteLength(static_cast<LengthType>(strlen(_str))) {}
|
||||
|
||||
UTF8StringSliceBase(const char* _str, const LengthType _utf8Length)
|
||||
: str(_str), utf8Length(_utf8Length) {
|
||||
CalculateByteLength();
|
||||
}
|
||||
|
||||
UTF8StringSliceBase(const char* _str, const LengthType _utf8Length,
|
||||
const LengthType _byteLength)
|
||||
: str(_str), utf8Length(_utf8Length), byteLength(_byteLength) {
|
||||
CalculateByteLength();
|
||||
}
|
||||
|
||||
LengthType UTF8Length() const { return utf8Length; }
|
||||
|
||||
LengthType ByteLength() const { return byteLength; }
|
||||
|
||||
UTF8StringSliceBase Left(const LengthType numberOfCharacters) const {
|
||||
if (numberOfCharacters == UTF8Length()) {
|
||||
return *this;
|
||||
} else {
|
||||
return UTF8StringSliceBase(str, numberOfCharacters);
|
||||
}
|
||||
}
|
||||
|
||||
UTF8StringSliceBase Right(const LengthType numberOfCharacters) const {
|
||||
if (numberOfCharacters == UTF8Length()) {
|
||||
return *this;
|
||||
} else {
|
||||
const char* pstr = str + byteLength;
|
||||
for (size_t i = 0; i < numberOfCharacters; i++) {
|
||||
pstr = UTF8Util::PrevChar(pstr);
|
||||
}
|
||||
return UTF8StringSliceBase(pstr, numberOfCharacters);
|
||||
}
|
||||
}
|
||||
|
||||
UTF8StringSliceBase SubString(const LengthType offset,
|
||||
const LengthType numberOfCharacters) const {
|
||||
if (offset == 0) {
|
||||
return Left(numberOfCharacters);
|
||||
} else {
|
||||
const char* pstr = str;
|
||||
for (size_t i = 0; i < offset; i++) {
|
||||
pstr = UTF8Util::NextChar(pstr);
|
||||
}
|
||||
return UTF8StringSliceBase(pstr, numberOfCharacters);
|
||||
}
|
||||
}
|
||||
|
||||
std::string ToString() const { return std::string(str, str + byteLength); }
|
||||
|
||||
const char* CString() const { return str; }
|
||||
|
||||
LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const {
|
||||
if (str == that.str) {
|
||||
return (std::min)(utf8Length, that.utf8Length);
|
||||
} else {
|
||||
const char* pstr1 = str;
|
||||
const char* pstr2 = that.str;
|
||||
for (size_t length = 0; length < utf8Length && length < that.utf8Length;
|
||||
length++) {
|
||||
size_t charLen1 = UTF8Util::NextCharLength(pstr1);
|
||||
size_t charLen2 = UTF8Util::NextCharLength(pstr2);
|
||||
if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) {
|
||||
return length;
|
||||
}
|
||||
pstr1 += charLen1;
|
||||
pstr2 += charLen2;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void MoveRight() {
|
||||
if (utf8Length > 0) {
|
||||
const size_t charLen = UTF8Util::NextCharLength(str);
|
||||
str += charLen;
|
||||
utf8Length--;
|
||||
byteLength -= charLen;
|
||||
}
|
||||
}
|
||||
|
||||
void MoveLeft() {
|
||||
if (utf8Length > 0) {
|
||||
const size_t charLen = UTF8Util::PrevCharLength(str + byteLength);
|
||||
utf8Length--;
|
||||
byteLength -= charLen;
|
||||
}
|
||||
}
|
||||
|
||||
int ReverseCompare(const UTF8StringSliceBase& that) const {
|
||||
const char* pstr1 = str + byteLength;
|
||||
const char* pstr2 = that.str + that.byteLength;
|
||||
const size_t length = (std::min)(utf8Length, that.utf8Length);
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
const size_t charLen1 = UTF8Util::PrevCharLength(pstr1);
|
||||
const size_t charLen2 = UTF8Util::PrevCharLength(pstr2);
|
||||
pstr1 -= charLen1;
|
||||
pstr2 -= charLen2;
|
||||
const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2));
|
||||
if (cmp < 0) {
|
||||
return -1;
|
||||
} else if (cmp > 0) {
|
||||
return 1;
|
||||
} else if (charLen1 < charLen2) {
|
||||
return -1;
|
||||
} else if (charLen1 > charLen2) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (utf8Length < that.utf8Length) {
|
||||
return -1;
|
||||
} else if (utf8Length > that.utf8Length) {
|
||||
return 1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const {
|
||||
return static_cast<LengthType>(
|
||||
ToString().find(pattern.str, 0, pattern.byteLength));
|
||||
}
|
||||
|
||||
bool operator<(const UTF8StringSliceBase& that) const {
|
||||
return Compare(that) < 0;
|
||||
}
|
||||
|
||||
bool operator>(const UTF8StringSliceBase& that) const {
|
||||
return Compare(that) > 0;
|
||||
}
|
||||
|
||||
bool operator==(const UTF8StringSliceBase& that) const {
|
||||
return (str == that.str && utf8Length == that.utf8Length) ||
|
||||
Compare(that) == 0;
|
||||
}
|
||||
|
||||
bool operator!=(const UTF8StringSliceBase& that) const {
|
||||
return !this->operator==(that);
|
||||
}
|
||||
|
||||
class Hasher {
|
||||
public:
|
||||
size_t operator()(const UTF8StringSliceBase& text) const {
|
||||
return internal::FNVHash<sizeof(size_t)>(text.CString(),
|
||||
text.ByteLength());
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
inline int Compare(const UTF8StringSliceBase& that) const {
|
||||
int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength));
|
||||
if (cmp == 0) {
|
||||
if (utf8Length < that.utf8Length) {
|
||||
cmp = -1;
|
||||
} else if (utf8Length > that.utf8Length) {
|
||||
cmp = 1;
|
||||
} else {
|
||||
cmp = 0;
|
||||
}
|
||||
}
|
||||
return cmp;
|
||||
}
|
||||
|
||||
void CalculateByteLength() {
|
||||
const char* pstr = str;
|
||||
for (size_t i = 0; i < utf8Length; i++) {
|
||||
pstr = UTF8Util::NextChar(pstr);
|
||||
}
|
||||
byteLength = static_cast<LengthType>(pstr - str);
|
||||
}
|
||||
|
||||
const char* str;
|
||||
LengthType utf8Length;
|
||||
LengthType byteLength;
|
||||
};
|
||||
|
||||
typedef UTF8StringSliceBase<size_t> UTF8StringSlice;
|
||||
|
||||
template <typename LENGTH_TYPE>
|
||||
std::ostream& operator<<(::std::ostream& os,
|
||||
const UTF8StringSliceBase<LENGTH_TYPE>& str) {
|
||||
return os << str.ToString();
|
||||
}
|
||||
|
||||
} // namespace opencc
|
||||
290
opencc/x86_64/include/opencc/UTF8Util.hpp
Normal file
290
opencc/x86_64/include/opencc/UTF8Util.hpp
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2013 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define NOMINMAX
|
||||
#include <Windows.h>
|
||||
#undef NOMINMAX
|
||||
#endif // _MSC_VER
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "Common.hpp"
|
||||
#include "Exception.hpp"
|
||||
|
||||
namespace opencc {
|
||||
/**
|
||||
* UTF8 std::string utilities
|
||||
* @ingroup opencc_cpp_api
|
||||
*/
|
||||
class OPENCC_EXPORT UTF8Util {
|
||||
public:
|
||||
/**
|
||||
* Detect UTF8 BOM and skip it.
|
||||
*/
|
||||
static void SkipUtf8Bom(FILE* fp);
|
||||
|
||||
/**
|
||||
* Returns the length in byte for the next UTF8 character.
|
||||
* On error returns 0.
|
||||
*/
|
||||
static size_t NextCharLengthNoException(const char* str) {
|
||||
char ch = *str;
|
||||
if ((ch & 0xF0) == 0xE0) {
|
||||
return 3;
|
||||
} else if ((ch & 0x80) == 0x00) {
|
||||
return 1;
|
||||
} else if ((ch & 0xE0) == 0xC0) {
|
||||
return 2;
|
||||
} else if ((ch & 0xF8) == 0xF0) {
|
||||
return 4;
|
||||
} else if ((ch & 0xFC) == 0xF8) {
|
||||
return 5;
|
||||
} else if ((ch & 0xFE) == 0xFC) {
|
||||
return 6;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length in byte for the next UTF8 character.
|
||||
*/
|
||||
static size_t NextCharLength(const char* str) {
|
||||
size_t length = NextCharLengthNoException(str);
|
||||
if (length == 0) {
|
||||
throw InvalidUTF8(str);
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length in byte for the previous UTF8 character.
|
||||
*/
|
||||
static size_t PrevCharLength(const char* str) {
|
||||
{
|
||||
const size_t length = NextCharLengthNoException(str - 3);
|
||||
if (length == 3) {
|
||||
return length;
|
||||
}
|
||||
}
|
||||
{
|
||||
const size_t length = NextCharLengthNoException(str - 1);
|
||||
if (length == 1) {
|
||||
return length;
|
||||
}
|
||||
}
|
||||
{
|
||||
const size_t length = NextCharLengthNoException(str - 2);
|
||||
if (length == 2) {
|
||||
return length;
|
||||
}
|
||||
}
|
||||
for (size_t i = 4; i <= 6; i++) {
|
||||
const size_t length = NextCharLengthNoException(str - i);
|
||||
if (length == i) {
|
||||
return length;
|
||||
}
|
||||
}
|
||||
throw InvalidUTF8(str);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the char* pointer over the next UTF8 character.
|
||||
*/
|
||||
static const char* NextChar(const char* str) {
|
||||
return str + NextCharLength(str);
|
||||
}
|
||||
|
||||
/**
|
||||
* Move the char* pointer before the previous UTF8 character.
|
||||
*/
|
||||
static const char* PrevChar(const char* str) {
|
||||
return str - PrevCharLength(str);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the UTF8 length of a valid UTF8 std::string.
|
||||
*/
|
||||
static size_t Length(const char* str) {
|
||||
size_t length = 0;
|
||||
while (*str != '\0') {
|
||||
str = NextChar(str);
|
||||
length++;
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds a character in the same line.
|
||||
* @param str The text to be searched in.
|
||||
* @param ch The character to find.
|
||||
* @return The pointer that points to the found chacter in str or EOL/EOF.
|
||||
*/
|
||||
static const char* FindNextInline(const char* str, const char ch) {
|
||||
while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
|
||||
str = NextChar(str);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the character is a line ending or end of file.
|
||||
*/
|
||||
static bool IsLineEndingOrFileEnding(const char ch) {
|
||||
return ch == '\0' || ch == '\n' || ch == '\r';
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a substring with given length to a new string.
|
||||
*/
|
||||
static std::string FromSubstr(const char* str, size_t length) {
|
||||
std::string newStr;
|
||||
newStr.resize(length);
|
||||
strncpy(const_cast<char*>(newStr.c_str()), str, length);
|
||||
return newStr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given std::string is longer or as long as the given
|
||||
* length.
|
||||
*/
|
||||
static bool NotShorterThan(const char* str, size_t byteLength) {
|
||||
while (byteLength > 0) {
|
||||
if (*str == '\0') {
|
||||
return false;
|
||||
}
|
||||
byteLength--;
|
||||
str++;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates a std::string with a maximal length in byte.
|
||||
* No UTF8 character will be broken.
|
||||
*/
|
||||
static std::string TruncateUTF8(const char* str, size_t maxByteLength) {
|
||||
std::string wordTrunc;
|
||||
if (NotShorterThan(str, maxByteLength)) {
|
||||
size_t len = 0;
|
||||
const char* pStr = str;
|
||||
for (;;) {
|
||||
const size_t charLength = NextCharLength(pStr);
|
||||
if (len + charLength > maxByteLength) {
|
||||
break;
|
||||
}
|
||||
pStr += charLength;
|
||||
len += charLength;
|
||||
}
|
||||
wordTrunc = FromSubstr(str, len);
|
||||
} else {
|
||||
wordTrunc = str;
|
||||
}
|
||||
return wordTrunc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces all patterns in a std::string in place.
|
||||
*/
|
||||
static void ReplaceAll(std::string& str, const char* from, const char* to) {
|
||||
std::string::size_type pos = 0;
|
||||
std::string::size_type fromLen = strlen(from);
|
||||
std::string::size_type toLen = strlen(to);
|
||||
while ((pos = str.find(from, pos)) != std::string::npos) {
|
||||
str.replace(pos, fromLen, to);
|
||||
pos += toLen;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins a std::string vector in to a std::string with a separator.
|
||||
*/
|
||||
static std::string Join(const std::vector<std::string>& strings,
|
||||
const std::string& separator) {
|
||||
std::ostringstream buffer;
|
||||
bool first = true;
|
||||
for (const auto& str : strings) {
|
||||
if (!first) {
|
||||
buffer << separator;
|
||||
}
|
||||
buffer << str;
|
||||
first = false;
|
||||
}
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins a std::string vector in to a std::string.
|
||||
*/
|
||||
static std::string Join(const std::vector<std::string>& strings) {
|
||||
std::ostringstream buffer;
|
||||
for (const auto& str : strings) {
|
||||
buffer << str;
|
||||
}
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
static void GetByteMap(const char* str, const size_t utf8Length,
|
||||
std::vector<size_t>* byteMap) {
|
||||
if (byteMap->size() < utf8Length) {
|
||||
byteMap->resize(utf8Length);
|
||||
}
|
||||
const char* pstr = str;
|
||||
for (size_t i = 0; i < utf8Length; i++) {
|
||||
(*byteMap)[i] = pstr - str;
|
||||
pstr = NextChar(pstr);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _MSC_VER
|
||||
static std::wstring GetPlatformString(const std::string& str) {
|
||||
return U8ToU16(str);
|
||||
}
|
||||
#else
|
||||
static std::string GetPlatformString(const std::string& str) { return str; }
|
||||
#endif // _MSC_VER
|
||||
|
||||
#ifdef _MSC_VER
|
||||
static std::string U16ToU8(const std::wstring& wstr) {
|
||||
std::string ret;
|
||||
int length = static_cast<int>(wstr.length());
|
||||
int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
|
||||
NULL, NULL);
|
||||
if (convcnt > 0) {
|
||||
ret.resize(convcnt);
|
||||
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
|
||||
NULL, NULL);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static std::wstring U8ToU16(const std::string& str) {
|
||||
std::wstring ret;
|
||||
int length = static_cast<int>(str.length());
|
||||
int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
|
||||
if (convcnt > 0) {
|
||||
ret.resize(convcnt);
|
||||
MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
#endif // _MSC_VER
|
||||
};
|
||||
} // namespace opencc
|
||||
161
opencc/x86_64/include/opencc/opencc.h
Normal file
161
opencc/x86_64/include/opencc/opencc.h
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef __OPENCC_H_
|
||||
#define __OPENCC_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include "Export.hpp"
|
||||
#include "SimpleConverter.hpp"
|
||||
#include <string>
|
||||
|
||||
extern "C" {
|
||||
#else
|
||||
#include <stddef.h>
|
||||
#endif
|
||||
|
||||
#ifndef OPENCC_EXPORT
|
||||
#define OPENCC_EXPORT
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @defgroup opencc_c_api OpenCC C API
|
||||
*
|
||||
* API in C language
|
||||
*/
|
||||
|
||||
/**
|
||||
* Filename of default Simplified to Traditional configuration
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json"
|
||||
|
||||
/**
|
||||
* Filename of default Traditional to Simplified configuration
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json"
|
||||
|
||||
/**
|
||||
* Type of opencc descriptor
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
typedef void* opencc_t;
|
||||
|
||||
/**
|
||||
* Makes an instance of opencc
|
||||
*
|
||||
* @param configFileName Location of configuration file. If this is set to NULL,
|
||||
* OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
|
||||
* @return A description pointer of the newly allocated instance of
|
||||
* opencc. On error the return value will be (opencc_t) -1.
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT opencc_t opencc_open(const char* configFileName);
|
||||
#ifdef _MSC_VER
|
||||
/**
|
||||
* Makes an instance of opencc (wide char / Unicode)
|
||||
*
|
||||
* @param configFileName Location of configuration file. If this is set to NULL,
|
||||
* OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
|
||||
* @return A description pointer of the newly allocated instance of
|
||||
* opencc. On error the return value will be (opencc_t) -1.
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName);
|
||||
#endif /* _MSC_VER */
|
||||
|
||||
/**
|
||||
* Destroys an instance of opencc
|
||||
*
|
||||
* @param opencc The description pointer.
|
||||
* @return 0 on success or non-zero number on failure.
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT int opencc_close(opencc_t opencc);
|
||||
|
||||
/**
|
||||
* Converts UTF-8 std::string
|
||||
*
|
||||
* @param opencc The opencc description pointer.
|
||||
* @param input The UTF-8 encoded std::string.
|
||||
* @param length The maximum length in byte to convert. If length is (size_t)-1,
|
||||
* the whole std::string (terminated by '\0') will be converted.
|
||||
* @param output The buffer to store converted text. You MUST make sure this
|
||||
* buffer has sufficient space.
|
||||
*
|
||||
* @return The length of converted std::string or (size_t)-1 on error.
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc,
|
||||
const char* input,
|
||||
size_t length, char* output);
|
||||
|
||||
/**
|
||||
* Converts UTF-8 std::string
|
||||
* This function returns an allocated C-Style std::string, which stores
|
||||
* the converted std::string.
|
||||
* You MUST call opencc_convert_utf8_free() to release allocated memory.
|
||||
*
|
||||
* @param opencc The opencc description pointer.
|
||||
* @param input The UTF-8 encoded std::string.
|
||||
* @param length The maximum length in byte to convert. If length is (size_t)-1,
|
||||
* the whole std::string (terminated by '\0') will be converted.
|
||||
*
|
||||
* @return The newly allocated UTF-8 std::string that stores text
|
||||
* converted, or NULL on error.
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input,
|
||||
size_t length);
|
||||
|
||||
/**
|
||||
* Releases allocated buffer by opencc_convert_utf8
|
||||
*
|
||||
* @param str Pointer to the allocated std::string buffer by
|
||||
* opencc_convert_utf8.
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT void opencc_convert_utf8_free(char* str);
|
||||
|
||||
/**
|
||||
* Returns the last error message
|
||||
*
|
||||
* Note that this function is the only one which is NOT thread-safe.
|
||||
*
|
||||
* @ingroup opencc_c_api
|
||||
*/
|
||||
OPENCC_EXPORT const char* opencc_error(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @defgroup opencc_cpp_api OpenCC C++ Comprehensive API
|
||||
*
|
||||
* Comprehensive API in C++ language
|
||||
*/
|
||||
|
||||
#endif
|
||||
21
opencc/x86_64/include/opencc/opencc_config.h
Normal file
21
opencc/x86_64/include/opencc/opencc_config.h
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Open Chinese Convert
|
||||
*
|
||||
* Copyright 2021 Carbo Kuo <byvoid@byvoid.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
/* #undef OPENCC_ENABLE_DARTS */
|
||||
Loading…
Add table
Add a link
Reference in a new issue