66 lines
2.3 KiB
Bash
Executable file
66 lines
2.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
# translated from https://github.com/fcitx/libime/blob/master/data/CMakeLists.txt
|
|
|
|
set -euxo pipefail
|
|
|
|
OUT_DIR="./libime"
|
|
if [ -f "$OUT_DIR" ]; then
|
|
echo "$OUT_DIR is a file"
|
|
exit 1
|
|
fi
|
|
if [ -d "$OUT_DIR" ]; then
|
|
rm -r "$OUT_DIR"
|
|
fi
|
|
mkdir -p "$OUT_DIR/data"
|
|
mkdir -p "$OUT_DIR/table"
|
|
|
|
LIBIME_REPO="https://raw.githubusercontent.com/fcitx/libime"
|
|
LIBIME_TAG="1.0.7"
|
|
|
|
curl -LO "$LIBIME_REPO/$LIBIME_TAG/data/convert_open_gram_arpa.py"
|
|
curl -LO "$LIBIME_REPO/$LIBIME_TAG/data/convert_open_gram_dict.py"
|
|
|
|
OPENGRAM_LM_TAR="lm_sc.3gm.arpa-20140820.tar.bz2"
|
|
OPENGRAM_LM_URL="https://download.fcitx-im.org/data/$OPENGRAM_LM_TAR"
|
|
OPENGRAM_LM_HASH="751bab7c55ea93a2cedfb0fbb7eb09f67d4da9c2c55496e5f31eb8580f1d1e2f"
|
|
|
|
curl -LO "$OPENGRAM_LM_URL"
|
|
echo "$OPENGRAM_LM_HASH $OPENGRAM_LM_TAR" | sha256sum --check --status
|
|
tar xf "$OPENGRAM_LM_TAR"
|
|
|
|
OPENGRAM_LM_SRC="./kenlm_sc.arpa"
|
|
OPENGRAM_LM_OUTPUT="$OUT_DIR/data/zh_CN.lm"
|
|
OPENGRAM_LM_PREDICT_OUTPUT="$OUT_DIR/data/zh_CN.lm.predict"
|
|
|
|
python ./convert_open_gram_arpa.py ./lm_sc.3gm.arpa > "$OPENGRAM_LM_SRC"
|
|
libime_slm_build_binary -s -a 22 -q 8 trie "$OPENGRAM_LM_SRC" "$OPENGRAM_LM_OUTPUT"
|
|
libime_prediction "$OPENGRAM_LM_OUTPUT" "$OPENGRAM_LM_SRC" "$OPENGRAM_LM_PREDICT_OUTPUT"
|
|
|
|
OPENGRAM_DICT_TAR="dict.utf8-20211021.tar.xz"
|
|
OPENGRAM_DICT_URL="https://download.fcitx-im.org/data/$OPENGRAM_DICT_TAR"
|
|
OPENGRAM_DICT_HASH="300597e6f7f79f788480fd665de8a07bfe90227048b5a7e39f40f43a62a981df"
|
|
|
|
curl -LO "$OPENGRAM_DICT_URL"
|
|
echo "$OPENGRAM_DICT_HASH $OPENGRAM_DICT_TAR" | sha256sum --check --status
|
|
tar xf "$OPENGRAM_DICT_TAR"
|
|
|
|
OPENGRAM_DICT_SRC="./dict.converted"
|
|
OPENGRAM_DICT_OUTPUT="$OUT_DIR/data/sc.dict"
|
|
|
|
python ./convert_open_gram_dict.py ./dict.utf8 > "$OPENGRAM_DICT_SRC"
|
|
libime_pinyindict "$OPENGRAM_DICT_SRC" "$OPENGRAM_DICT_OUTPUT"
|
|
|
|
TABLE_DICT_TAR="table.tar.gz"
|
|
TABLE_DICT_URL="https://download.fcitx-im.org/data/$TABLE_DICT_TAR"
|
|
TABLE_DICT_HASH="6196053c724125e3ae3d8bd6b2f9172d0c83b65b0d410d3cde63b7a8d6ab87b7"
|
|
TABLE_TXT_FILES=(db.txt erbi.txt qxm.txt wanfeng.txt wbpy.txt wbx.txt zrm.txt cj.txt)
|
|
|
|
curl -LO "$TABLE_DICT_URL"
|
|
echo "$TABLE_DICT_HASH $TABLE_DICT_TAR" | sha256sum --check --status
|
|
tar xf "$TABLE_DICT_TAR"
|
|
|
|
for TABLE_TXT_FILE in "${TABLE_TXT_FILES[@]}"; do
|
|
TABLE_DICT_FILE="${TABLE_TXT_FILE/.txt/.main.dict}"
|
|
libime_tabledict "$TABLE_TXT_FILE" "$OUT_DIR/table/$TABLE_DICT_FILE"
|
|
done
|