Auto update

This commit is contained in:
Jenkins for Fcitx 5 Android 2023-02-18 12:42:03 +00:00
parent 31c40e70b1
commit 95021a13ed
164 changed files with 10625 additions and 0 deletions

View file

@ -0,0 +1,82 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
// Microsoft Visual C++ specific
#if defined(_MSC_VER) && (_MSC_VER >= 1020)
#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820)
#endif
#include <cstddef>
#include <memory>
#include <string>
#include <vector>
#include "Export.hpp"
#include "Optional.hpp"
#include "opencc_config.h"
// Forward decalarations and alias
namespace opencc {
class Config;
class Conversion;
class ConversionChain;
class Converter;
class Dict;
class DictEntry;
class DictGroup;
class Lexicon;
class MarisaDict;
class MultiValueDictEntry;
class NoValueDictEntry;
class Segmentation;
class Segments;
class SerializableDict;
class SingleValueDictEntry;
class TextDict;
typedef std::shared_ptr<Conversion> ConversionPtr;
typedef std::shared_ptr<ConversionChain> ConversionChainPtr;
typedef std::shared_ptr<Converter> ConverterPtr;
typedef std::shared_ptr<Dict> DictPtr;
typedef std::shared_ptr<DictGroup> DictGroupPtr;
typedef std::shared_ptr<Lexicon> LexiconPtr;
typedef std::shared_ptr<MarisaDict> MarisaDictPtr;
typedef std::shared_ptr<Segmentation> SegmentationPtr;
typedef std::shared_ptr<Segments> SegmentsPtr;
typedef std::shared_ptr<SerializableDict> SerializableDictPtr;
typedef std::shared_ptr<TextDict> TextDictPtr;
#ifdef OPENCC_ENABLE_DARTS
class BinaryDict;
class DartsDict;
typedef std::shared_ptr<BinaryDict> BinaryDictPtr;
typedef std::shared_ptr<DartsDict> DartsDictPtr;
#endif
} // namespace opencc
#ifndef PKGDATADIR
const std::string PACKAGE_DATA_DIRECTORY = "";
#else // ifndef PKGDATADIR
const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/";
#endif // ifndef PKGDATADIR
#ifndef VERSION
#define VERSION "1.0.*"
#endif // ifndef VERSION

View file

@ -0,0 +1,42 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
namespace opencc {
/**
* Configuration loader
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Config {
public:
Config();
virtual ~Config();
ConverterPtr NewFromString(const std::string& json,
const std::string& configDirectory);
ConverterPtr NewFromFile(const std::string& fileName);
private:
void* internal;
};
} // namespace opencc

View file

@ -0,0 +1,47 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "Segmentation.hpp"
namespace opencc {
/**
* Conversion interface
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Conversion {
public:
Conversion(DictPtr _dict) : dict(_dict) {}
// Convert single phrase
std::string Convert(const std::string& phrase) const;
// Convert single phrase
std::string Convert(const char* phrase) const;
// Convert segmented text
SegmentsPtr Convert(const SegmentsPtr& input) const;
const DictPtr GetDict() const { return dict; }
private:
const DictPtr dict;
};
} // namespace opencc

View file

@ -0,0 +1,43 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <list>
#include "Common.hpp"
#include "Conversion.hpp"
namespace opencc {
/**
* Chain of conversions
* Consists of a list of conversions. Converts input in sequence.
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT ConversionChain {
public:
ConversionChain(const std::list<ConversionPtr> _conversions);
SegmentsPtr Convert(const SegmentsPtr& input) const;
const std::list<ConversionPtr> GetConversions() const { return conversions; }
private:
const std::list<ConversionPtr> conversions;
};
} // namespace opencc

View file

@ -0,0 +1,51 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "Segmentation.hpp"
namespace opencc {
/**
* Controller of segmentation and conversion
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Converter {
public:
Converter(const std::string& _name, SegmentationPtr _segmentation,
ConversionChainPtr _conversionChain)
: name(_name), segmentation(_segmentation),
conversionChain(_conversionChain) {}
std::string Convert(const std::string& text) const;
size_t Convert(const char* input, char* output) const;
const SegmentationPtr GetSegmentation() const { return segmentation; }
const ConversionChainPtr GetConversionChain() const {
return conversionChain;
}
private:
const std::string name;
const SegmentationPtr segmentation;
const ConversionChainPtr conversionChain;
};
} // namespace opencc

View file

@ -0,0 +1,92 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "DictEntry.hpp"
namespace opencc {
/**
* Abstract class of dictionary
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Dict {
public:
/**
* Matches a word exactly and returns the DictEntry or Optional::Null().
*/
virtual Optional<const DictEntry*> Match(const char* word,
size_t len) const = 0;
/**
* Matches a word exactly and returns the DictEntry or Optional::Null().
*/
Optional<const DictEntry*> Match(const std::string& word) const {
return Match(word.c_str(), word.length());
}
/**
* Matches the longest matched prefix of a word.
* For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
* the longest prefix of "banana" matched is "bana".
*/
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
size_t len) const;
/**
* Matches the longest matched prefix of a word.
*/
Optional<const DictEntry*> MatchPrefix(const char* word) const {
return MatchPrefix(word, KeyMaxLength());
}
/**
* Matches the longest matched prefix of a word.
*/
Optional<const DictEntry*> MatchPrefix(const std::string& word) const {
return MatchPrefix(word.c_str(), word.length());
}
/**
* Returns all matched prefixes of a word, sorted by the length (desc).
* For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
* all the matched prefixes of "banana" are "bana", "ban", "ba", "b".
*/
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
size_t len) const;
/**
* Returns all matched prefixes of a word, sorted by the length (desc).
*/
std::vector<const DictEntry*>
MatchAllPrefixes(const std::string& word) const {
return MatchAllPrefixes(word.c_str(), word.length());
}
/**
* Returns the length of the longest key in the dictionary.
*/
virtual size_t KeyMaxLength() const = 0;
/**
* Returns all entries in the dictionary.
*/
virtual LexiconPtr GetLexicon() const = 0;
};
} // namespace opencc

View file

@ -0,0 +1,32 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2017 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
namespace opencc {
/**
* Converts a dictionary from a format to another.
* @ingroup opencc_cpp_api
*/
OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName,
const std::string& outputFileName,
const std::string& formatFrom,
const std::string& formatTo);
} // namespace opencc

View file

@ -0,0 +1,173 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "Segments.hpp"
#include "UTF8Util.hpp"
namespace opencc {
/**
* Key-values pair entry
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT DictEntry {
public:
virtual ~DictEntry() {}
virtual std::string Key() const = 0;
virtual std::vector<std::string> Values() const = 0;
virtual std::string GetDefault() const = 0;
virtual size_t NumValues() const = 0;
virtual std::string ToString() const = 0;
size_t KeyLength() const { return Key().length(); }
bool operator<(const DictEntry& that) const { return Key() < that.Key(); }
bool operator==(const DictEntry& that) const { return Key() == that.Key(); }
static bool UPtrLessThan(const std::unique_ptr<DictEntry>& a,
const std::unique_ptr<DictEntry>& b) {
return *a < *b;
}
};
class OPENCC_EXPORT NoValueDictEntry : public DictEntry {
public:
NoValueDictEntry(const std::string& _key) : key(_key) {}
virtual ~NoValueDictEntry() {}
virtual std::string Key() const { return key; }
virtual std::vector<std::string> Values() const {
return std::vector<std::string>();
}
virtual std::string GetDefault() const { return key; }
virtual size_t NumValues() const { return 0; }
virtual std::string ToString() const { return key; }
private:
std::string key;
};
class OPENCC_EXPORT SingleValueDictEntry : public DictEntry {
public:
virtual std::string Value() const = 0;
virtual std::vector<std::string> Values() const {
return std::vector<std::string>{Value()};
}
virtual std::string GetDefault() const { return Value(); }
virtual size_t NumValues() const { return 1; }
virtual std::string ToString() const {
return std::string(Key()) + "\t" + Value();
}
};
class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry {
public:
StrSingleValueDictEntry(const std::string& _key, const std::string& _value)
: key(_key), value(_value) {}
virtual ~StrSingleValueDictEntry() {}
virtual std::string Key() const { return key; }
virtual std::string Value() const { return value; }
private:
std::string key;
std::string value;
};
class OPENCC_EXPORT MultiValueDictEntry : public DictEntry {
public:
virtual std::string GetDefault() const {
if (NumValues() > 0) {
return Values().at(0);
} else {
return Key();
}
}
virtual std::string ToString() const;
};
class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry {
public:
StrMultiValueDictEntry(const std::string& _key,
const std::vector<std::string>& _values)
: key(_key), values(_values) {}
virtual ~StrMultiValueDictEntry() {}
virtual std::string Key() const { return key; }
size_t NumValues() const { return values.size(); }
std::vector<std::string> Values() const { return values; }
private:
std::string key;
std::vector<std::string> values;
};
class OPENCC_EXPORT DictEntryFactory {
public:
static DictEntry* New(const std::string& key) {
return new NoValueDictEntry(key);
}
static DictEntry* New(const std::string& key, const std::string& value) {
return new StrSingleValueDictEntry(key, value);
}
static DictEntry* New(const std::string& key,
const std::vector<std::string>& values) {
if (values.size() == 0) {
return New(key);
} else if (values.size() == 1) {
return New(key, values.front());
}
return new StrMultiValueDictEntry(key, values);
}
static DictEntry* New(const DictEntry* entry) {
if (entry->NumValues() == 0) {
return new NoValueDictEntry(entry->Key());
} else if (entry->NumValues() == 1) {
return new StrSingleValueDictEntry(entry->Key(), entry->Values().front());
} else {
return new StrMultiValueDictEntry(entry->Key(), entry->Values());
}
}
};
} // namespace opencc

View file

@ -0,0 +1,57 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <list>
#include "Common.hpp"
#include "Dict.hpp"
namespace opencc {
/**
* Group of dictionaries
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT DictGroup : public Dict {
public:
DictGroup(const std::list<DictPtr>& dicts);
static DictGroupPtr NewFromDict(const Dict& dict);
virtual ~DictGroup();
virtual size_t KeyMaxLength() const;
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
size_t len) const;
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
size_t len) const;
virtual LexiconPtr GetLexicon() const;
const std::list<DictPtr> GetDicts() const { return dicts; }
private:
const size_t keyMaxLength;
const std::list<DictPtr> dicts;
};
} // namespace opencc

View file

@ -0,0 +1,88 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <sstream>
#include <stdexcept>
#include <string>
#include "Export.hpp"
#if defined(_MSC_VER) && _MSC_VER < 1900
// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not
// supported
#define noexcept
#endif // ifdef _MSC_VER
namespace opencc {
class OPENCC_EXPORT Exception {
public:
Exception() {}
virtual ~Exception() throw() {}
Exception(const std::string& _message) : message(_message) {}
virtual const char* what() const noexcept { return message.c_str(); }
protected:
std::string message;
};
class OPENCC_EXPORT FileNotFound : public Exception {
public:
FileNotFound(const std::string& fileName)
: Exception(fileName + " not found or not accessible.") {}
};
class OPENCC_EXPORT FileNotWritable : public Exception {
public:
FileNotWritable(const std::string& fileName)
: Exception(fileName + " not writable.") {}
};
class OPENCC_EXPORT InvalidFormat : public Exception {
public:
InvalidFormat(const std::string& message)
: Exception("Invalid format: " + message) {}
};
class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat {
public:
InvalidTextDictionary(const std::string& _message, size_t lineNum)
: InvalidFormat("") {
std::ostringstream buffer;
buffer << "Invalid text dictionary at line " << lineNum << ": " << _message;
message = buffer.str();
}
};
class OPENCC_EXPORT InvalidUTF8 : public Exception {
public:
InvalidUTF8(const std::string& _message)
: Exception("Invalid UTF8: " + _message) {}
};
class OPENCC_EXPORT ShouldNotBeHere : public Exception {
public:
ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {}
};
} // namespace opencc

View file

@ -0,0 +1,40 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
#define OPENCC_EXPORT
#define OPENCC_NO_EXPORT
#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
#ifndef OPENCC_EXPORT
#ifdef libopencc_EXPORTS
/* We are building this library */
#define OPENCC_EXPORT __declspec(dllexport)
#else // ifdef libopencc_EXPORTS
/* We are using this library */
#define OPENCC_EXPORT __declspec(dllimport)
#endif // ifdef libopencc_EXPORTS
#endif // ifndef OPENCC_EXPORT
#ifndef OPENCC_NO_EXPORT
#define OPENCC_NO_EXPORT
#endif // ifndef OPENCC_NO_EXPORT
#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)

View file

@ -0,0 +1,68 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "DictEntry.hpp"
namespace opencc {
/**
* Storage of all entries
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Lexicon {
public:
Lexicon() {}
Lexicon(std::vector<std::unique_ptr<DictEntry>> entries_)
: entries(std::move(entries_)) {}
Lexicon(const Lexicon&) = delete;
Lexicon& operator=(const Lexicon&) = delete;
// Lexicon will take the ownership of the entry.
void Add(DictEntry* entry) { entries.emplace_back(entry); }
void Add(std::unique_ptr<DictEntry> entry) {
entries.push_back(std::move(entry));
}
void Sort();
// Returns true if the lexicon is sorted by key.
bool IsSorted();
// Returns true if every key unique (after sorted).
// When dupkey is set, it is set to the duplicate key.
bool IsUnique(std::string* dupkey = nullptr);
const DictEntry* At(size_t index) const { return entries.at(index).get(); }
size_t Length() const { return entries.size(); }
std::vector<std::unique_ptr<DictEntry>>::const_iterator begin() const {
return entries.begin();
}
std::vector<std::unique_ptr<DictEntry>>::const_iterator end() const {
return entries.end();
}
private:
std::vector<std::unique_ptr<DictEntry>> entries;
};
} // namespace opencc

View file

@ -0,0 +1,63 @@
/*
* Open Chinese Convert
*
* Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "SerializableDict.hpp"
namespace opencc {
/**
* Darts dictionary
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict {
public:
virtual ~MarisaDict();
virtual size_t KeyMaxLength() const;
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
virtual Optional<const DictEntry*> MatchPrefix(const char* word,
size_t len) const;
virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
size_t len) const;
virtual LexiconPtr GetLexicon() const;
virtual void SerializeToFile(FILE* fp) const;
/**
* Constructs a MarisaDict from another dictionary.
*/
static MarisaDictPtr NewFromDict(const Dict& thatDict);
static MarisaDictPtr NewFromFile(FILE* fp);
private:
MarisaDict();
size_t maxLength;
LexiconPtr lexicon;
class MarisaInternal;
std::unique_ptr<MarisaInternal> internal;
};
} // namespace opencc

View file

@ -0,0 +1,43 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "DictGroup.hpp"
#include "Segmentation.hpp"
namespace opencc {
/**
* Implementation of maximal match segmentation
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation {
public:
MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {}
virtual ~MaxMatchSegmentation() {}
virtual SegmentsPtr Segment(const std::string& text) const;
const DictPtr GetDict() const { return dict; }
private:
const DictPtr dict;
};
} // namespace opencc

View file

@ -0,0 +1,76 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
namespace opencc {
/**
* A class that wraps type T into a nullable type.
* @ingroup opencc_cpp_api
*/
template <typename T> class Optional {
public:
/**
* The constructor of Optional.
*/
Optional(T actual) : isNull(false), data(actual) {}
/**
* Returns true if the instance is null.
*/
bool IsNull() const { return isNull; }
/**
* Returns the containing data of the instance.
*/
const T& Get() const { return data; }
/**
* Constructs a null instance.
*/
static Optional<T> Null() { return Optional(); }
private:
Optional() : isNull(true) {}
bool isNull;
T data;
};
/**
* Specialization of Optional for pointers.
*
* Reduce a bool.
*/
template <typename T> class Optional<T*> {
private:
Optional() : data(nullptr) {}
typedef T* TPtr;
TPtr data;
public:
Optional(TPtr actual) : data(actual) {}
bool IsNull() const { return data == nullptr; }
const TPtr& Get() const { return data; }
static Optional<TPtr> Null() { return Optional(); }
};
} // namespace opencc

View file

@ -0,0 +1,195 @@
/*
* Open Chinese Convert
*
* Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <functional>
#include <unordered_map>
#include "Common.hpp"
#include "UTF8StringSlice.hpp"
namespace opencc {
class OPENCC_EXPORT PhraseExtract {
public:
typedef UTF8StringSlice::LengthType LengthType;
typedef UTF8StringSliceBase<unsigned char> UTF8StringSlice8Bit;
PhraseExtract();
virtual ~PhraseExtract();
void Extract(const std::string& text) {
SetFullText(text);
ExtractSuffixes();
CalculateFrequency();
CalculateSuffixEntropy();
ReleaseSuffixes();
ExtractPrefixes();
CalculatePrefixEntropy();
ReleasePrefixes();
ExtractWordCandidates();
CalculateCohesions();
SelectWords();
}
void SetFullText(const std::string& fullText) {
utf8FullText = UTF8StringSlice(fullText.c_str());
}
void SetFullText(const char* fullText) {
utf8FullText = UTF8StringSlice(fullText);
}
void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; }
void SetWordMinLength(const LengthType _wordMinLength) {
wordMinLength = _wordMinLength;
}
void SetWordMaxLength(const LengthType _wordMaxLength) {
wordMaxLength = _wordMaxLength;
}
void SetPrefixSetLength(const LengthType _prefixSetLength) {
prefixSetLength = _prefixSetLength;
}
void SetSuffixSetLength(const LengthType _suffixSetLength) {
suffixSetLength = _suffixSetLength;
}
// PreCalculationFilter is called after frequencies statistics.
void SetPreCalculationFilter(
const std::function<bool(const PhraseExtract&,
const UTF8StringSlice8Bit&)>& filter) {
preCalculationFilter = filter;
}
void SetPostCalculationFilter(
const std::function<bool(const PhraseExtract&,
const UTF8StringSlice8Bit&)>& filter) {
postCalculationFilter = filter;
}
void ReleaseSuffixes() { std::vector<UTF8StringSlice8Bit>().swap(suffixes); }
void ReleasePrefixes() { std::vector<UTF8StringSlice8Bit>().swap(prefixes); }
const std::vector<UTF8StringSlice8Bit>& Words() const { return words; }
const std::vector<UTF8StringSlice8Bit>& WordCandidates() const {
return wordCandidates;
}
struct Signals {
size_t frequency;
double cohesion;
double suffixEntropy;
double prefixEntropy;
};
const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const;
double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const;
double Entropy(const UTF8StringSlice8Bit& wordCandidate) const;
double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
size_t Frequency(const UTF8StringSlice8Bit& word) const;
double Probability(const UTF8StringSlice8Bit& word) const;
double LogProbability(const UTF8StringSlice8Bit& word) const;
void Reset();
void ExtractSuffixes();
void ExtractPrefixes();
void ExtractWordCandidates();
void CalculateFrequency();
void CalculateCohesions();
void CalculateSuffixEntropy();
void CalculatePrefixEntropy();
void SelectWords();
static bool
DefaultPreCalculationFilter(const PhraseExtract&,
const PhraseExtract::UTF8StringSlice8Bit&);
static bool
DefaultPostCalculationFilter(const PhraseExtract&,
const PhraseExtract::UTF8StringSlice8Bit&);
private:
class DictType;
// Pointwise Mutual Information
double PMI(const UTF8StringSlice8Bit& wordCandidate,
const UTF8StringSlice8Bit& part1,
const UTF8StringSlice8Bit& part2) const;
double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const;
double CalculateEntropy(
const std::unordered_map<UTF8StringSlice8Bit, size_t,
UTF8StringSlice8Bit::Hasher>& choices) const;
LengthType wordMinLength;
LengthType wordMaxLength;
LengthType prefixSetLength;
LengthType suffixSetLength;
std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
preCalculationFilter;
std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
postCalculationFilter;
bool prefixesExtracted;
bool suffixesExtracted;
bool frequenciesCalculated;
bool wordCandidatesExtracted;
bool cohesionsCalculated;
bool prefixEntropiesCalculated;
bool suffixEntropiesCalculated;
bool wordsSelected;
UTF8StringSlice utf8FullText;
size_t totalOccurrence;
double logTotalOccurrence;
std::vector<UTF8StringSlice8Bit> prefixes;
std::vector<UTF8StringSlice8Bit> suffixes;
std::vector<UTF8StringSlice8Bit> wordCandidates;
std::vector<UTF8StringSlice8Bit> words;
DictType* signals;
friend class PhraseExtractTest;
};
} // namespace opencc

View file

@ -0,0 +1,32 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
namespace opencc {
/**
* Abstract segmentation
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Segmentation {
public:
virtual SegmentsPtr Segment(const std::string& text) const = 0;
};
} // namespace opencc

View file

@ -0,0 +1,114 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <sstream>
#include "Common.hpp"
namespace opencc {
/**
* Segmented text
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT Segments {
public:
Segments() {}
Segments(std::initializer_list<const char*> initList) {
for (const std::string& item : initList) {
AddSegment(item);
}
}
Segments(std::initializer_list<std::string> initList) {
for (const std::string& item : initList) {
AddSegment(item);
}
}
void AddSegment(const char* unmanagedString) {
indexes.push_back(std::make_pair(unmanaged.size(), false));
unmanaged.push_back(unmanagedString);
}
void AddSegment(const std::string& str) {
indexes.push_back(std::make_pair(managed.size(), true));
managed.push_back(str);
}
class iterator : public std::iterator<std::input_iterator_tag, const char*> {
public:
iterator(const Segments* const _segments, size_t _cursor)
: segments(_segments), cursor(_cursor) {}
iterator& operator++() {
cursor++;
return *this;
}
bool operator==(const iterator& that) const {
return cursor == that.cursor && segments == that.segments;
}
bool operator!=(const iterator& that) const {
return !this->operator==(that);
}
const char* operator*() const { return segments->At(cursor); }
private:
const Segments* const segments;
size_t cursor;
};
const char* At(size_t cursor) const {
const auto& index = indexes[cursor];
if (index.second) {
return managed[index.first].c_str();
} else {
return unmanaged[index.first];
}
}
size_t Length() const { return indexes.size(); }
iterator begin() const { return iterator(this, 0); }
iterator end() const { return iterator(this, indexes.size()); }
std::string ToString() const {
// TODO implement a nested structure to reduce concatenation,
// like a purely functional differential list
std::ostringstream buffer;
for (const char* segment : *this) {
buffer << segment;
}
return buffer.str();
}
private:
Segments(const Segments&) {}
std::vector<const char*> unmanaged;
std::vector<std::string> managed;
// index, managed
std::vector<std::pair<size_t, bool>> indexes;
};
} // namespace opencc

View file

@ -0,0 +1,77 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Dict.hpp"
namespace opencc {
/**
* Serializable dictionary interface
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT SerializableDict {
public:
/**
* Serializes the dictionary and writes in to a file.
*/
virtual void SerializeToFile(FILE* fp) const = 0;
/**
* Serializes the dictionary and writes in to a file.
*/
virtual void SerializeToFile(const std::string& fileName) const {
FILE* fp = fopen(fileName.c_str(), "wb");
if (fp == NULL) {
throw FileNotWritable(fileName);
}
SerializeToFile(fp);
fclose(fp);
}
template <typename DICT>
static bool TryLoadFromFile(const std::string& fileName,
std::shared_ptr<DICT>* dict) {
FILE* fp =
#ifdef _MSC_VER
// well, the 'GetPlatformString' shall return a 'wstring'
_wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb")
#else
fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb")
#endif // _MSC_VER
;
if (fp == NULL) {
return false;
}
std::shared_ptr<DICT> loadedDict = DICT::NewFromFile(fp);
fclose(fp);
*dict = loadedDict;
return true;
}
template <typename DICT>
static std::shared_ptr<DICT> NewFromFile(const std::string& fileName) {
std::shared_ptr<DICT> dict;
if (!TryLoadFromFile<DICT>(fileName, &dict)) {
throw FileNotFound(fileName);
}
return dict;
}
};
} // namespace opencc

View file

@ -0,0 +1,50 @@
/*
* Open Chinese Convert
*
* Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "SerializableDict.hpp"
namespace opencc {
/**
* Binary format for dictionary values serialization.
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT SerializedValues : public SerializableDict {
public:
SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {}
virtual ~SerializedValues() {}
virtual void SerializeToFile(FILE* fp) const;
static std::shared_ptr<SerializedValues> NewFromFile(FILE* fp);
const LexiconPtr& GetLexicon() const { return lexicon; }
size_t KeyMaxLength() const;
private:
LexiconPtr lexicon;
void ConstructBuffer(std::string* valueBuffer,
std::vector<uint16_t>* valueBytes,
uint32_t* valueTotalLength) const;
};
} // namespace opencc

View file

@ -0,0 +1,93 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Export.hpp"
#include <string>
#ifndef __OPENCC_SIMPLECONVERTER_HPP_
#define __OPENCC_SIMPLECONVERTER_HPP_
/**
* @defgroup opencc_simple_api OpenCC C++ Simple API
*
* Simple API in C++ language
*/
namespace opencc {
/**
* A high level converter
* This interface does not require C++11 to compile.
* @ingroup opencc_simple_api
*/
class OPENCC_EXPORT SimpleConverter {
public:
/**
* Constructor of SimpleConverter
* @param configFileName File name of configuration.
*/
SimpleConverter(const std::string& configFileName);
~SimpleConverter();
/**
* Converts a text
* @param input Text to be converted.
*/
std::string Convert(const std::string& input) const;
/**
* Converts a text
* @param input A C-Style std::string (terminated by '\0') to be converted.
*/
std::string Convert(const char* input) const;
/**
* Converts a text
* @param input A C-Style std::string limited by a given length to be
* converted.
* @param length Maximal length in byte of the input std::string.
*/
std::string Convert(const char* input, size_t length) const;
/**
* Converts a text and writes to an allocated buffer
* Please make sure the buffer has sufficient space.
* @param input A C-Style std::string (terminated by '\0') to be converted.
* @param output Buffer to write the converted text.
* @return Length of converted text.
*/
size_t Convert(const char* input, char* output) const;
/**
* Converts a text and writes to an allocated buffer
* Please make sure the buffer has sufficient space.
* @param input A C-Style std::string limited by a given length to be
* converted.
* @param length Maximal length in byte of the input std::string.
* @param output Buffer to write the converted text.
* @return Length of converted text.
*/
size_t Convert(const char* input, size_t length, char* output) const;
private:
const void* internalData;
};
} // namespace opencc
#endif

View file

@ -0,0 +1,60 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "Common.hpp"
#include "SerializableDict.hpp"
namespace opencc {
/**
* Text dictionary
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT TextDict : public Dict, public SerializableDict {
public:
/**
* Constructor of TextDict.
* _lexicon must be sorted.
*/
TextDict(const LexiconPtr& _lexicon);
virtual ~TextDict();
virtual size_t KeyMaxLength() const;
virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
virtual LexiconPtr GetLexicon() const;
virtual void SerializeToFile(FILE* fp) const;
/**
* Constructs a TextDict from another dictionary.
*/
static TextDictPtr NewFromDict(const Dict& dict);
static TextDictPtr NewFromFile(FILE* fp);
static TextDictPtr NewFromSortedFile(FILE* fp);
private:
const size_t maxLength;
const LexiconPtr lexicon;
};
} // namespace opencc

View file

@ -0,0 +1,246 @@
/*
* Open Chinese Convert
*
* Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstring>
#include "Common.hpp"
#include "UTF8Util.hpp"
namespace opencc {
namespace internal {
inline size_t FNVHash(const char* text, const size_t byteLength,
const size_t FNV_prime, const size_t FNV_offset_basis) {
size_t hash = FNV_offset_basis;
for (const char* pstr = text; pstr < text + byteLength; pstr++) {
hash ^= *pstr;
hash *= FNV_prime;
}
return hash;
}
template <int> size_t FNVHash(const char* text, const size_t byteLength);
template <>
inline size_t FNVHash<4>(const char* text, const size_t byteLength) {
return FNVHash(text, byteLength, 16777619UL, 2166136261UL);
}
#if SIZE_MAX == 0xffffffffffffffff
template <>
inline size_t FNVHash<8>(const char* text, const size_t byteLength) {
return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL);
}
#endif
} // namespace internal
template <typename LENGTH_TYPE> class UTF8StringSliceBase {
public:
typedef LENGTH_TYPE LengthType;
UTF8StringSliceBase(const char* _str)
: str(_str), utf8Length(static_cast<LengthType>(UTF8Util::Length(_str))),
byteLength(static_cast<LengthType>(strlen(_str))) {}
UTF8StringSliceBase(const char* _str, const LengthType _utf8Length)
: str(_str), utf8Length(_utf8Length) {
CalculateByteLength();
}
UTF8StringSliceBase(const char* _str, const LengthType _utf8Length,
const LengthType _byteLength)
: str(_str), utf8Length(_utf8Length), byteLength(_byteLength) {
CalculateByteLength();
}
LengthType UTF8Length() const { return utf8Length; }
LengthType ByteLength() const { return byteLength; }
UTF8StringSliceBase Left(const LengthType numberOfCharacters) const {
if (numberOfCharacters == UTF8Length()) {
return *this;
} else {
return UTF8StringSliceBase(str, numberOfCharacters);
}
}
UTF8StringSliceBase Right(const LengthType numberOfCharacters) const {
if (numberOfCharacters == UTF8Length()) {
return *this;
} else {
const char* pstr = str + byteLength;
for (size_t i = 0; i < numberOfCharacters; i++) {
pstr = UTF8Util::PrevChar(pstr);
}
return UTF8StringSliceBase(pstr, numberOfCharacters);
}
}
UTF8StringSliceBase SubString(const LengthType offset,
const LengthType numberOfCharacters) const {
if (offset == 0) {
return Left(numberOfCharacters);
} else {
const char* pstr = str;
for (size_t i = 0; i < offset; i++) {
pstr = UTF8Util::NextChar(pstr);
}
return UTF8StringSliceBase(pstr, numberOfCharacters);
}
}
std::string ToString() const { return std::string(str, str + byteLength); }
const char* CString() const { return str; }
LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const {
if (str == that.str) {
return (std::min)(utf8Length, that.utf8Length);
} else {
const char* pstr1 = str;
const char* pstr2 = that.str;
for (size_t length = 0; length < utf8Length && length < that.utf8Length;
length++) {
size_t charLen1 = UTF8Util::NextCharLength(pstr1);
size_t charLen2 = UTF8Util::NextCharLength(pstr2);
if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) {
return length;
}
pstr1 += charLen1;
pstr2 += charLen2;
}
return 0;
}
}
void MoveRight() {
if (utf8Length > 0) {
const size_t charLen = UTF8Util::NextCharLength(str);
str += charLen;
utf8Length--;
byteLength -= charLen;
}
}
void MoveLeft() {
if (utf8Length > 0) {
const size_t charLen = UTF8Util::PrevCharLength(str + byteLength);
utf8Length--;
byteLength -= charLen;
}
}
int ReverseCompare(const UTF8StringSliceBase& that) const {
const char* pstr1 = str + byteLength;
const char* pstr2 = that.str + that.byteLength;
const size_t length = (std::min)(utf8Length, that.utf8Length);
for (size_t i = 0; i < length; i++) {
const size_t charLen1 = UTF8Util::PrevCharLength(pstr1);
const size_t charLen2 = UTF8Util::PrevCharLength(pstr2);
pstr1 -= charLen1;
pstr2 -= charLen2;
const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2));
if (cmp < 0) {
return -1;
} else if (cmp > 0) {
return 1;
} else if (charLen1 < charLen2) {
return -1;
} else if (charLen1 > charLen2) {
return 1;
}
}
if (utf8Length < that.utf8Length) {
return -1;
} else if (utf8Length > that.utf8Length) {
return 1;
} else {
return 0;
}
}
LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const {
return static_cast<LengthType>(
ToString().find(pattern.str, 0, pattern.byteLength));
}
bool operator<(const UTF8StringSliceBase& that) const {
return Compare(that) < 0;
}
bool operator>(const UTF8StringSliceBase& that) const {
return Compare(that) > 0;
}
bool operator==(const UTF8StringSliceBase& that) const {
return (str == that.str && utf8Length == that.utf8Length) ||
Compare(that) == 0;
}
bool operator!=(const UTF8StringSliceBase& that) const {
return !this->operator==(that);
}
class Hasher {
public:
size_t operator()(const UTF8StringSliceBase& text) const {
return internal::FNVHash<sizeof(size_t)>(text.CString(),
text.ByteLength());
}
};
private:
inline int Compare(const UTF8StringSliceBase& that) const {
int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength));
if (cmp == 0) {
if (utf8Length < that.utf8Length) {
cmp = -1;
} else if (utf8Length > that.utf8Length) {
cmp = 1;
} else {
cmp = 0;
}
}
return cmp;
}
void CalculateByteLength() {
const char* pstr = str;
for (size_t i = 0; i < utf8Length; i++) {
pstr = UTF8Util::NextChar(pstr);
}
byteLength = static_cast<LengthType>(pstr - str);
}
const char* str;
LengthType utf8Length;
LengthType byteLength;
};
typedef UTF8StringSliceBase<size_t> UTF8StringSlice;
template <typename LENGTH_TYPE>
std::ostream& operator<<(::std::ostream& os,
const UTF8StringSliceBase<LENGTH_TYPE>& str) {
return os << str.ToString();
}
} // namespace opencc

View file

@ -0,0 +1,290 @@
/*
* Open Chinese Convert
*
* Copyright 2013 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifdef _MSC_VER
#define NOMINMAX
#include <Windows.h>
#undef NOMINMAX
#endif // _MSC_VER
#include <cstring>
#include "Common.hpp"
#include "Exception.hpp"
namespace opencc {
/**
* UTF8 std::string utilities
* @ingroup opencc_cpp_api
*/
class OPENCC_EXPORT UTF8Util {
public:
/**
* Detect UTF8 BOM and skip it.
*/
static void SkipUtf8Bom(FILE* fp);
/**
* Returns the length in byte for the next UTF8 character.
* On error returns 0.
*/
static size_t NextCharLengthNoException(const char* str) {
char ch = *str;
if ((ch & 0xF0) == 0xE0) {
return 3;
} else if ((ch & 0x80) == 0x00) {
return 1;
} else if ((ch & 0xE0) == 0xC0) {
return 2;
} else if ((ch & 0xF8) == 0xF0) {
return 4;
} else if ((ch & 0xFC) == 0xF8) {
return 5;
} else if ((ch & 0xFE) == 0xFC) {
return 6;
}
return 0;
}
/**
* Returns the length in byte for the next UTF8 character.
*/
static size_t NextCharLength(const char* str) {
size_t length = NextCharLengthNoException(str);
if (length == 0) {
throw InvalidUTF8(str);
}
return length;
}
/**
* Returns the length in byte for the previous UTF8 character.
*/
static size_t PrevCharLength(const char* str) {
{
const size_t length = NextCharLengthNoException(str - 3);
if (length == 3) {
return length;
}
}
{
const size_t length = NextCharLengthNoException(str - 1);
if (length == 1) {
return length;
}
}
{
const size_t length = NextCharLengthNoException(str - 2);
if (length == 2) {
return length;
}
}
for (size_t i = 4; i <= 6; i++) {
const size_t length = NextCharLengthNoException(str - i);
if (length == i) {
return length;
}
}
throw InvalidUTF8(str);
}
/**
* Returns the char* pointer over the next UTF8 character.
*/
static const char* NextChar(const char* str) {
return str + NextCharLength(str);
}
/**
* Move the char* pointer before the previous UTF8 character.
*/
static const char* PrevChar(const char* str) {
return str - PrevCharLength(str);
}
/**
* Returns the UTF8 length of a valid UTF8 std::string.
*/
static size_t Length(const char* str) {
size_t length = 0;
while (*str != '\0') {
str = NextChar(str);
length++;
}
return length;
}
/**
* Finds a character in the same line.
* @param str The text to be searched in.
* @param ch The character to find.
* @return The pointer that points to the found chacter in str or EOL/EOF.
*/
static const char* FindNextInline(const char* str, const char ch) {
while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
str = NextChar(str);
}
return str;
}
/**
* Returns true if the character is a line ending or end of file.
*/
static bool IsLineEndingOrFileEnding(const char ch) {
return ch == '\0' || ch == '\n' || ch == '\r';
}
/**
* Copies a substring with given length to a new string.
*/
static std::string FromSubstr(const char* str, size_t length) {
std::string newStr;
newStr.resize(length);
strncpy(const_cast<char*>(newStr.c_str()), str, length);
return newStr;
}
/**
* Returns true if the given std::string is longer or as long as the given
* length.
*/
static bool NotShorterThan(const char* str, size_t byteLength) {
while (byteLength > 0) {
if (*str == '\0') {
return false;
}
byteLength--;
str++;
}
return true;
}
/**
* Truncates a std::string with a maximal length in byte.
* No UTF8 character will be broken.
*/
static std::string TruncateUTF8(const char* str, size_t maxByteLength) {
std::string wordTrunc;
if (NotShorterThan(str, maxByteLength)) {
size_t len = 0;
const char* pStr = str;
for (;;) {
const size_t charLength = NextCharLength(pStr);
if (len + charLength > maxByteLength) {
break;
}
pStr += charLength;
len += charLength;
}
wordTrunc = FromSubstr(str, len);
} else {
wordTrunc = str;
}
return wordTrunc;
}
/**
* Replaces all patterns in a std::string in place.
*/
static void ReplaceAll(std::string& str, const char* from, const char* to) {
std::string::size_type pos = 0;
std::string::size_type fromLen = strlen(from);
std::string::size_type toLen = strlen(to);
while ((pos = str.find(from, pos)) != std::string::npos) {
str.replace(pos, fromLen, to);
pos += toLen;
}
}
/**
* Joins a std::string vector in to a std::string with a separator.
*/
static std::string Join(const std::vector<std::string>& strings,
const std::string& separator) {
std::ostringstream buffer;
bool first = true;
for (const auto& str : strings) {
if (!first) {
buffer << separator;
}
buffer << str;
first = false;
}
return buffer.str();
}
/**
* Joins a std::string vector in to a std::string.
*/
static std::string Join(const std::vector<std::string>& strings) {
std::ostringstream buffer;
for (const auto& str : strings) {
buffer << str;
}
return buffer.str();
}
static void GetByteMap(const char* str, const size_t utf8Length,
std::vector<size_t>* byteMap) {
if (byteMap->size() < utf8Length) {
byteMap->resize(utf8Length);
}
const char* pstr = str;
for (size_t i = 0; i < utf8Length; i++) {
(*byteMap)[i] = pstr - str;
pstr = NextChar(pstr);
}
}
#ifdef _MSC_VER
static std::wstring GetPlatformString(const std::string& str) {
return U8ToU16(str);
}
#else
static std::string GetPlatformString(const std::string& str) { return str; }
#endif // _MSC_VER
#ifdef _MSC_VER
static std::string U16ToU8(const std::wstring& wstr) {
std::string ret;
int length = static_cast<int>(wstr.length());
int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
NULL, NULL);
if (convcnt > 0) {
ret.resize(convcnt);
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
NULL, NULL);
}
return ret;
}
static std::wstring U8ToU16(const std::string& str) {
std::wstring ret;
int length = static_cast<int>(str.length());
int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
if (convcnt > 0) {
ret.resize(convcnt);
MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
}
return ret;
}
#endif // _MSC_VER
};
} // namespace opencc

View file

@ -0,0 +1,161 @@
/*
* Open Chinese Convert
*
* Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __OPENCC_H_
#define __OPENCC_H_
#ifdef __cplusplus
#include "Export.hpp"
#include "SimpleConverter.hpp"
#include <string>
extern "C" {
#else
#include <stddef.h>
#endif
#ifndef OPENCC_EXPORT
#define OPENCC_EXPORT
#endif
/**
* @defgroup opencc_c_api OpenCC C API
*
* API in C language
*/
/**
* Filename of default Simplified to Traditional configuration
*
* @ingroup opencc_c_api
*/
#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json"
/**
* Filename of default Traditional to Simplified configuration
*
* @ingroup opencc_c_api
*/
#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json"
/**
* Type of opencc descriptor
*
* @ingroup opencc_c_api
*/
typedef void* opencc_t;
/**
* Makes an instance of opencc
*
* @param configFileName Location of configuration file. If this is set to NULL,
* OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
* @return A description pointer of the newly allocated instance of
* opencc. On error the return value will be (opencc_t) -1.
* @ingroup opencc_c_api
*/
OPENCC_EXPORT opencc_t opencc_open(const char* configFileName);
#ifdef _MSC_VER
/**
* Makes an instance of opencc (wide char / Unicode)
*
* @param configFileName Location of configuration file. If this is set to NULL,
* OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
* @return A description pointer of the newly allocated instance of
* opencc. On error the return value will be (opencc_t) -1.
* @ingroup opencc_c_api
*/
OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName);
#endif /* _MSC_VER */
/**
* Destroys an instance of opencc
*
* @param opencc The description pointer.
* @return 0 on success or non-zero number on failure.
* @ingroup opencc_c_api
*/
OPENCC_EXPORT int opencc_close(opencc_t opencc);
/**
* Converts UTF-8 std::string
*
* @param opencc The opencc description pointer.
* @param input The UTF-8 encoded std::string.
* @param length The maximum length in byte to convert. If length is (size_t)-1,
* the whole std::string (terminated by '\0') will be converted.
* @param output The buffer to store converted text. You MUST make sure this
* buffer has sufficient space.
*
* @return The length of converted std::string or (size_t)-1 on error.
*
* @ingroup opencc_c_api
*/
OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc,
const char* input,
size_t length, char* output);
/**
* Converts UTF-8 std::string
* This function returns an allocated C-Style std::string, which stores
* the converted std::string.
* You MUST call opencc_convert_utf8_free() to release allocated memory.
*
* @param opencc The opencc description pointer.
* @param input The UTF-8 encoded std::string.
* @param length The maximum length in byte to convert. If length is (size_t)-1,
* the whole std::string (terminated by '\0') will be converted.
*
* @return The newly allocated UTF-8 std::string that stores text
* converted, or NULL on error.
* @ingroup opencc_c_api
*/
OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input,
size_t length);
/**
* Releases allocated buffer by opencc_convert_utf8
*
* @param str Pointer to the allocated std::string buffer by
* opencc_convert_utf8.
*
* @ingroup opencc_c_api
*/
OPENCC_EXPORT void opencc_convert_utf8_free(char* str);
/**
* Returns the last error message
*
* Note that this function is the only one which is NOT thread-safe.
*
* @ingroup opencc_c_api
*/
OPENCC_EXPORT const char* opencc_error(void);
#ifdef __cplusplus
} // extern "C"
#endif
/**
* @defgroup opencc_cpp_api OpenCC C++ Comprehensive API
*
* Comprehensive API in C++ language
*/
#endif

View file

@ -0,0 +1,21 @@
/*
* Open Chinese Convert
*
* Copyright 2021 Carbo Kuo <byvoid@byvoid.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
/* #undef OPENCC_ENABLE_DARTS */