import json import math import re import sys cases = json.load(open(sys.argv[1], encoding='utf-8')) out = open(sys.argv[2], 'w', encoding='utf-8') END = re.compile(r'[.!?。!?…\n]') TRAIL = re.compile(r'[\s.!?。!?…~,]+$') def norm(text): return re.sub(r'\s+', ' ', text).strip() def truncate(tokens, tau): """Keep tokens while each logprob >= tau; stop at sentence end. Drop a trailing partial word.""" text = '' first_word_lp = 0.0 first_word_done = False started = False for tok in tokens: t, lp = tok['token'], tok['logprob'] if '<' in t: break if lp < tau: # partial word cut: if this token continues a word (no leading space), drop the tail of the last word if text and not t.startswith(' ') and not text.endswith(' '): text = text[: text.rfind(' ') + 1] if ' ' in text.strip() else '' break text += t if text.strip(): started = True if started and not first_word_done: first_word_lp += lp if ' ' in text.strip(): first_word_done = True if END.search(t): break return norm(text), first_word_lp def score(sugg, truth): truth = norm(truth) sw, tw = sugg.split(' ') if sugg else [], truth.split(' ') m = 0 while m < min(len(sw), len(tw)) and TRAIL.sub('', sw[m]) == TRAIL.sub('', tw[m]): m += 1 bare = TRAIL.sub('', sugg) full = bool(bare) and truth.startswith(bare) acc = bare if full else ' '.join(sw[:m]) return m, full, max(0, len(acc) - 1) if acc else 0 total_truth = sum(len(norm(c['truth'])) for c in cases) print('cases', len(cases), file=out) print('tau gate coverage hit|shown full|shown savedRate meanLen', file=out) for tau in (-0.5, -1.0, -1.5, -2.0, -3.0, -99): for gate in (-0.1, -0.3, -0.5, -1.0, -99): shown = hits = full_n = saved = length = 0 for c in cases: if not c['tokens']: continue sugg, fw = truncate(c['tokens'], tau) if not sugg or fw < gate: continue shown += 1 m, full, s = score(sugg, c['truth']) hits += m >= 1 full_n += full saved += s length += len(sugg) if shown: print(f'{tau:5} {gate:5} {shown/len(cases):.3f} {hits/shown:.3f} {full_n/shown:.3f} {saved/total_truth:.4f} {length/shown:.1f}', file=out)