78 lines
3.1 KiB
Python
78 lines
3.1 KiB
Python
"""Hybrid: LLM first word vs personal n-gram guess on identical cases.
|
|
|
|
Reports exact first-word hits and 'useful' hits (guess is a prefix of the true first word: Korean stem + ending).
|
|
"""
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
|
|
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
|
|
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
|
|
texts += [r[0] for r in db.execute(
|
|
"select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")]
|
|
cases = json.load(open(cases_path, encoding='utf-8'))
|
|
out = open(out_path, 'w', encoding='utf-8')
|
|
TRAIL = re.compile(r'[\s.!?。!?…~,"\')]+$')
|
|
|
|
|
|
def words(text):
|
|
return [TRAIL.sub('', w) for w in re.sub(r'\s+', ' ', text or '').strip().split(' ') if w]
|
|
|
|
|
|
def ngram_guess(prefix_words, exclude):
|
|
bi, tri = defaultdict(Counter), defaultdict(Counter)
|
|
for text in texts:
|
|
if exclude in re.sub(r'\s+', ' ', text):
|
|
continue
|
|
ws = words(text)
|
|
for i in range(1, len(ws)):
|
|
bi[ws[i - 1]][ws[i]] += 1
|
|
if i >= 2:
|
|
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
|
|
if len(prefix_words) >= 2 and tri.get((prefix_words[-2], prefix_words[-1])):
|
|
word, n = tri[(prefix_words[-2], prefix_words[-1])].most_common(1)[0]
|
|
return word, n, 'tri'
|
|
if prefix_words and bi.get(prefix_words[-1]):
|
|
word, n = bi[prefix_words[-1]].most_common(1)[0]
|
|
return word, n, 'bi'
|
|
return None, 0, None
|
|
|
|
|
|
stats = Counter()
|
|
for c in cases:
|
|
truth = words(c['truth'])
|
|
if not truth:
|
|
continue
|
|
t0 = truth[0]
|
|
full = re.sub(r'\s+', ' ', c['prefix'] + c['truth']).strip()
|
|
g, n, kind = ngram_guess(words(c['prefix']), full)
|
|
llm = words(c['suggestion'])
|
|
l0 = llm[0] if llm else None
|
|
stats['cases'] += 1
|
|
for name, guess in (('llm', l0), ('ngram', g)):
|
|
if guess:
|
|
stats[f'{name}_shown'] += 1
|
|
stats[f'{name}_exact'] += guess == t0
|
|
stats[f'{name}_useful'] += len(guess) >= 1 and t0.startswith(guess)
|
|
if g and l0:
|
|
stats['both'] += 1
|
|
if g == l0 or l0.startswith(g):
|
|
stats['agree'] += 1
|
|
stats['agree_exact'] += g == t0
|
|
stats['agree_useful'] += t0.startswith(g)
|
|
# hybrid policy: ngram when it has a guess, otherwise LLM
|
|
h = g or l0
|
|
if h:
|
|
stats['hyb_shown'] += 1
|
|
stats['hyb_useful'] += t0.startswith(h)
|
|
stats['hyb_exact'] += h == t0
|
|
|
|
N = stats['cases']
|
|
p = lambda k, d: f"{stats[k]}/{stats[d]} = {stats[k]/stats[d]:.3f}" if stats[d] else '-'
|
|
print(f'cases {N}', file=out)
|
|
for name in ('llm', 'ngram', 'hyb'):
|
|
print(f"{name:6} shown {stats[name + '_shown']/N:.3f} exact|shown {p(name + '_exact', name + '_shown')} useful|shown {p(name + '_useful', name + '_shown')}", file=out)
|
|
print(f"agree (ngram==LLM first word) {p('agree', 'both')} exact {p('agree_exact', 'agree')} useful {p('agree_useful', 'agree')}", file=out)
|