fcitx5-android-prebuilt/scripts/libime-data-gen.sh
2022-04-04 23:59:56 -04:00

66 lines
2.3 KiB
Bash
Executable file

#!/usr/bin/env bash
# translated from https://github.com/fcitx/libime/blob/master/data/CMakeLists.txt
set -euxo pipefail
OUT_DIR="./libime"
if [ -f "$OUT_DIR" ]; then
echo "$OUT_DIR is a file"
exit 1
fi
if [ -d "$OUT_DIR" ]; then
rm -r "$OUT_DIR"
fi
mkdir -p "$OUT_DIR/data"
mkdir -p "$OUT_DIR/table"
LIBIME_REPO="https://raw.githubusercontent.com/fcitx/libime"
LIBIME_TAG="1.0.7"
curl -LO "$LIBIME_REPO/$LIBIME_TAG/data/convert_open_gram_arpa.py"
curl -LO "$LIBIME_REPO/$LIBIME_TAG/data/convert_open_gram_dict.py"
OPENGRAM_LM_TAR="lm_sc.3gm.arpa-20140820.tar.bz2"
OPENGRAM_LM_URL="https://download.fcitx-im.org/data/$OPENGRAM_LM_TAR"
OPENGRAM_LM_HASH="751bab7c55ea93a2cedfb0fbb7eb09f67d4da9c2c55496e5f31eb8580f1d1e2f"
curl -LO "$OPENGRAM_LM_URL"
echo "$OPENGRAM_LM_HASH $OPENGRAM_LM_TAR" | sha256sum --check --status
tar xf "$OPENGRAM_LM_TAR"
OPENGRAM_LM_SRC="./kenlm_sc.arpa"
OPENGRAM_LM_OUTPUT="$OUT_DIR/data/zh_CN.lm"
OPENGRAM_LM_PREDICT_OUTPUT="$OUT_DIR/data/zh_CN.lm.predict"
python ./convert_open_gram_arpa.py ./lm_sc.3gm.arpa > "$OPENGRAM_LM_SRC"
libime_slm_build_binary -s -a 22 -q 8 trie "$OPENGRAM_LM_SRC" "$OPENGRAM_LM_OUTPUT"
libime_prediction "$OPENGRAM_LM_OUTPUT" "$OPENGRAM_LM_SRC" "$OPENGRAM_LM_PREDICT_OUTPUT"
OPENGRAM_DICT_TAR="dict.utf8-20211021.tar.xz"
OPENGRAM_DICT_URL="https://download.fcitx-im.org/data/$OPENGRAM_DICT_TAR"
OPENGRAM_DICT_HASH="300597e6f7f79f788480fd665de8a07bfe90227048b5a7e39f40f43a62a981df"
curl -LO "$OPENGRAM_DICT_URL"
echo "$OPENGRAM_DICT_HASH $OPENGRAM_DICT_TAR" | sha256sum --check --status
tar xf "$OPENGRAM_DICT_TAR"
OPENGRAM_DICT_SRC="./dict.converted"
OPENGRAM_DICT_OUTPUT="$OUT_DIR/data/sc.dict"
python ./convert_open_gram_dict.py ./dict.utf8 > "$OPENGRAM_DICT_SRC"
libime_pinyindict "$OPENGRAM_DICT_SRC" "$OPENGRAM_DICT_OUTPUT"
TABLE_DICT_TAR="table.tar.gz"
TABLE_DICT_URL="https://download.fcitx-im.org/data/$TABLE_DICT_TAR"
TABLE_DICT_HASH="6196053c724125e3ae3d8bd6b2f9172d0c83b65b0d410d3cde63b7a8d6ab87b7"
TABLE_TXT_FILES=(db.txt erbi.txt qxm.txt wanfeng.txt wbpy.txt wbx.txt zrm.txt cj.txt)
curl -LO "$TABLE_DICT_URL"
echo "$TABLE_DICT_HASH $TABLE_DICT_TAR" | sha256sum --check --status
tar xf "$TABLE_DICT_TAR"
for TABLE_TXT_FILE in "${TABLE_TXT_FILES[@]}"; do
TABLE_DICT_FILE="${TABLE_TXT_FILE/.txt/.main.dict}"
libime_tabledict "$TABLE_TXT_FILE" "$OUT_DIR/table/$TABLE_DICT_FILE"
done