feat(desktop): built-in Han/Eng layout auto-switching, terminal/Vim support, and Unicode input
This commit is contained in:
parent
1da04b9e46
commit
a3ac44da2b
109 changed files with 8234 additions and 242 deletions
78
apps/desktop/tests/eval/analysis/hybrid_eval.py
Normal file
78
apps/desktop/tests/eval/analysis/hybrid_eval.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Hybrid: LLM first word vs personal n-gram guess on identical cases.
|
||||
|
||||
Reports exact first-word hits and 'useful' hits (guess is a prefix of the true first word: Korean stem + ending).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
|
||||
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
|
||||
texts += [r[0] for r in db.execute(
|
||||
"select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")]
|
||||
cases = json.load(open(cases_path, encoding='utf-8'))
|
||||
out = open(out_path, 'w', encoding='utf-8')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,"\')]+$')
|
||||
|
||||
|
||||
def words(text):
|
||||
return [TRAIL.sub('', w) for w in re.sub(r'\s+', ' ', text or '').strip().split(' ') if w]
|
||||
|
||||
|
||||
def ngram_guess(prefix_words, exclude):
|
||||
bi, tri = defaultdict(Counter), defaultdict(Counter)
|
||||
for text in texts:
|
||||
if exclude in re.sub(r'\s+', ' ', text):
|
||||
continue
|
||||
ws = words(text)
|
||||
for i in range(1, len(ws)):
|
||||
bi[ws[i - 1]][ws[i]] += 1
|
||||
if i >= 2:
|
||||
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
|
||||
if len(prefix_words) >= 2 and tri.get((prefix_words[-2], prefix_words[-1])):
|
||||
word, n = tri[(prefix_words[-2], prefix_words[-1])].most_common(1)[0]
|
||||
return word, n, 'tri'
|
||||
if prefix_words and bi.get(prefix_words[-1]):
|
||||
word, n = bi[prefix_words[-1]].most_common(1)[0]
|
||||
return word, n, 'bi'
|
||||
return None, 0, None
|
||||
|
||||
|
||||
stats = Counter()
|
||||
for c in cases:
|
||||
truth = words(c['truth'])
|
||||
if not truth:
|
||||
continue
|
||||
t0 = truth[0]
|
||||
full = re.sub(r'\s+', ' ', c['prefix'] + c['truth']).strip()
|
||||
g, n, kind = ngram_guess(words(c['prefix']), full)
|
||||
llm = words(c['suggestion'])
|
||||
l0 = llm[0] if llm else None
|
||||
stats['cases'] += 1
|
||||
for name, guess in (('llm', l0), ('ngram', g)):
|
||||
if guess:
|
||||
stats[f'{name}_shown'] += 1
|
||||
stats[f'{name}_exact'] += guess == t0
|
||||
stats[f'{name}_useful'] += len(guess) >= 1 and t0.startswith(guess)
|
||||
if g and l0:
|
||||
stats['both'] += 1
|
||||
if g == l0 or l0.startswith(g):
|
||||
stats['agree'] += 1
|
||||
stats['agree_exact'] += g == t0
|
||||
stats['agree_useful'] += t0.startswith(g)
|
||||
# hybrid policy: ngram when it has a guess, otherwise LLM
|
||||
h = g or l0
|
||||
if h:
|
||||
stats['hyb_shown'] += 1
|
||||
stats['hyb_useful'] += t0.startswith(h)
|
||||
stats['hyb_exact'] += h == t0
|
||||
|
||||
N = stats['cases']
|
||||
p = lambda k, d: f"{stats[k]}/{stats[d]} = {stats[k]/stats[d]:.3f}" if stats[d] else '-'
|
||||
print(f'cases {N}', file=out)
|
||||
for name in ('llm', 'ngram', 'hyb'):
|
||||
print(f"{name:6} shown {stats[name + '_shown']/N:.3f} exact|shown {p(name + '_exact', name + '_shown')} useful|shown {p(name + '_useful', name + '_shown')}", file=out)
|
||||
print(f"agree (ngram==LLM first word) {p('agree', 'both')} exact {p('agree_exact', 'agree')} useful {p('agree_useful', 'agree')}", file=out)
|
||||
63
apps/desktop/tests/eval/analysis/ngram_eval.py
Normal file
63
apps/desktop/tests/eval/analysis/ngram_eval.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""Personal n-gram next-word predictor on the same replay cases (leave-one-out).
|
||||
|
||||
Compares with the LLM on identical cases: does the user's own history predict the next word better?
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
|
||||
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
|
||||
texts += [r[0] for r in db.execute("select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")] if db.execute("select count(*) from sqlite_master where name='history'").fetchone()[0] else []
|
||||
cases = json.load(open(cases_path, encoding='utf-8'))
|
||||
out = open(out_path, 'w', encoding='utf-8')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
|
||||
|
||||
|
||||
def words(text):
|
||||
return [w for w in re.sub(r'\s+', ' ', text).strip().split(' ') if w]
|
||||
|
||||
|
||||
def build(exclude):
|
||||
bi, tri = defaultdict(Counter), defaultdict(Counter)
|
||||
for text in texts:
|
||||
if exclude and exclude in re.sub(r'\s+', ' ', text):
|
||||
continue
|
||||
ws = [TRAIL.sub('', w) for w in words(text)]
|
||||
for i in range(1, len(ws)):
|
||||
bi[ws[i - 1]][ws[i]] += 1
|
||||
if i >= 2:
|
||||
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
|
||||
return bi, tri
|
||||
|
||||
|
||||
shown = hits = 0
|
||||
min_count = int(sys.argv[4]) if len(sys.argv) > 4 else 2
|
||||
examples = []
|
||||
for c in cases:
|
||||
full = re.sub(r'\s+', ' ', (c['prefix'] + c['truth'])).strip()
|
||||
bi, tri = build(full)
|
||||
pw = [TRAIL.sub('', w) for w in words(c['prefix'])]
|
||||
truth_first = TRAIL.sub('', words(c['truth'])[0]) if words(c['truth']) else ''
|
||||
guess = None
|
||||
if len(pw) >= 2 and tri.get((pw[-2], pw[-1])):
|
||||
word, n = tri[(pw[-2], pw[-1])].most_common(1)[0]
|
||||
if n >= min_count:
|
||||
guess = word
|
||||
if guess is None and pw and bi.get(pw[-1]):
|
||||
word, n = bi[pw[-1]].most_common(1)[0]
|
||||
if n >= min_count:
|
||||
guess = word
|
||||
if guess:
|
||||
shown += 1
|
||||
hit = guess == truth_first
|
||||
hits += hit
|
||||
if len(examples) < 20:
|
||||
examples.append((c['prefix'][-25:], guess, truth_first, hit))
|
||||
print(f'corpus texts {len(texts)}; cases {len(cases)}; min_count {min_count}', file=out)
|
||||
print(f'ngram coverage {shown/len(cases):.3f} hit|shown {hits/shown if shown else 0:.3f} hits {hits}', file=out)
|
||||
for e in examples:
|
||||
print(' ', e, file=out)
|
||||
74
apps/desktop/tests/eval/analysis/simulate_gating.py
Normal file
74
apps/desktop/tests/eval/analysis/simulate_gating.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
|
||||
cases = json.load(open(sys.argv[1], encoding='utf-8'))
|
||||
out = open(sys.argv[2], 'w', encoding='utf-8')
|
||||
END = re.compile(r'[.!?。!?…\n]')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
|
||||
|
||||
|
||||
def norm(text):
|
||||
return re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
|
||||
def truncate(tokens, tau):
|
||||
"""Keep tokens while each logprob >= tau; stop at sentence end. Drop a trailing partial word."""
|
||||
text = ''
|
||||
first_word_lp = 0.0
|
||||
first_word_done = False
|
||||
started = False
|
||||
for tok in tokens:
|
||||
t, lp = tok['token'], tok['logprob']
|
||||
if '<' in t:
|
||||
break
|
||||
if lp < tau:
|
||||
# partial word cut: if this token continues a word (no leading space), drop the tail of the last word
|
||||
if text and not t.startswith(' ') and not text.endswith(' '):
|
||||
text = text[: text.rfind(' ') + 1] if ' ' in text.strip() else ''
|
||||
break
|
||||
text += t
|
||||
if text.strip():
|
||||
started = True
|
||||
if started and not first_word_done:
|
||||
first_word_lp += lp
|
||||
if ' ' in text.strip():
|
||||
first_word_done = True
|
||||
if END.search(t):
|
||||
break
|
||||
return norm(text), first_word_lp
|
||||
|
||||
|
||||
def score(sugg, truth):
|
||||
truth = norm(truth)
|
||||
sw, tw = sugg.split(' ') if sugg else [], truth.split(' ')
|
||||
m = 0
|
||||
while m < min(len(sw), len(tw)) and TRAIL.sub('', sw[m]) == TRAIL.sub('', tw[m]):
|
||||
m += 1
|
||||
bare = TRAIL.sub('', sugg)
|
||||
full = bool(bare) and truth.startswith(bare)
|
||||
acc = bare if full else ' '.join(sw[:m])
|
||||
return m, full, max(0, len(acc) - 1) if acc else 0
|
||||
|
||||
|
||||
total_truth = sum(len(norm(c['truth'])) for c in cases)
|
||||
print('cases', len(cases), file=out)
|
||||
print('tau gate coverage hit|shown full|shown savedRate meanLen', file=out)
|
||||
for tau in (-0.5, -1.0, -1.5, -2.0, -3.0, -99):
|
||||
for gate in (-0.1, -0.3, -0.5, -1.0, -99):
|
||||
shown = hits = full_n = saved = length = 0
|
||||
for c in cases:
|
||||
if not c['tokens']:
|
||||
continue
|
||||
sugg, fw = truncate(c['tokens'], tau)
|
||||
if not sugg or fw < gate:
|
||||
continue
|
||||
shown += 1
|
||||
m, full, s = score(sugg, c['truth'])
|
||||
hits += m >= 1
|
||||
full_n += full
|
||||
saved += s
|
||||
length += len(sugg)
|
||||
if shown:
|
||||
print(f'{tau:5} {gate:5} {shown/len(cases):.3f} {hits/shown:.3f} {full_n/shown:.3f} {saved/total_truth:.4f} {length/shown:.1f}', file=out)
|
||||
Loading…
Add table
Add a link
Reference in a new issue