d3ro-voice/apps/desktop/tests/eval/analysis/ngram_eval.py

63 lines
2.5 KiB
Python

"""Personal n-gram next-word predictor on the same replay cases (leave-one-out).
Compares with the LLM on identical cases: does the user's own history predict the next word better?
"""
import json
import re
import sqlite3
import sys
from collections import Counter, defaultdict
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
texts += [r[0] for r in db.execute("select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")] if db.execute("select count(*) from sqlite_master where name='history'").fetchone()[0] else []
cases = json.load(open(cases_path, encoding='utf-8'))
out = open(out_path, 'w', encoding='utf-8')
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
def words(text):
return [w for w in re.sub(r'\s+', ' ', text).strip().split(' ') if w]
def build(exclude):
bi, tri = defaultdict(Counter), defaultdict(Counter)
for text in texts:
if exclude and exclude in re.sub(r'\s+', ' ', text):
continue
ws = [TRAIL.sub('', w) for w in words(text)]
for i in range(1, len(ws)):
bi[ws[i - 1]][ws[i]] += 1
if i >= 2:
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
return bi, tri
shown = hits = 0
min_count = int(sys.argv[4]) if len(sys.argv) > 4 else 2
examples = []
for c in cases:
full = re.sub(r'\s+', ' ', (c['prefix'] + c['truth'])).strip()
bi, tri = build(full)
pw = [TRAIL.sub('', w) for w in words(c['prefix'])]
truth_first = TRAIL.sub('', words(c['truth'])[0]) if words(c['truth']) else ''
guess = None
if len(pw) >= 2 and tri.get((pw[-2], pw[-1])):
word, n = tri[(pw[-2], pw[-1])].most_common(1)[0]
if n >= min_count:
guess = word
if guess is None and pw and bi.get(pw[-1]):
word, n = bi[pw[-1]].most_common(1)[0]
if n >= min_count:
guess = word
if guess:
shown += 1
hit = guess == truth_first
hits += hit
if len(examples) < 20:
examples.append((c['prefix'][-25:], guess, truth_first, hit))
print(f'corpus texts {len(texts)}; cases {len(cases)}; min_count {min_count}', file=out)
print(f'ngram coverage {shown/len(cases):.3f} hit|shown {hits/shown if shown else 0:.3f} hits {hits}', file=out)
for e in examples:
print(' ', e, file=out)