74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
|
|
cases = json.load(open(sys.argv[1], encoding='utf-8'))
|
|
out = open(sys.argv[2], 'w', encoding='utf-8')
|
|
END = re.compile(r'[.!?。!?…\n]')
|
|
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
|
|
|
|
|
|
def norm(text):
|
|
return re.sub(r'\s+', ' ', text).strip()
|
|
|
|
|
|
def truncate(tokens, tau):
|
|
"""Keep tokens while each logprob >= tau; stop at sentence end. Drop a trailing partial word."""
|
|
text = ''
|
|
first_word_lp = 0.0
|
|
first_word_done = False
|
|
started = False
|
|
for tok in tokens:
|
|
t, lp = tok['token'], tok['logprob']
|
|
if '<' in t:
|
|
break
|
|
if lp < tau:
|
|
# partial word cut: if this token continues a word (no leading space), drop the tail of the last word
|
|
if text and not t.startswith(' ') and not text.endswith(' '):
|
|
text = text[: text.rfind(' ') + 1] if ' ' in text.strip() else ''
|
|
break
|
|
text += t
|
|
if text.strip():
|
|
started = True
|
|
if started and not first_word_done:
|
|
first_word_lp += lp
|
|
if ' ' in text.strip():
|
|
first_word_done = True
|
|
if END.search(t):
|
|
break
|
|
return norm(text), first_word_lp
|
|
|
|
|
|
def score(sugg, truth):
|
|
truth = norm(truth)
|
|
sw, tw = sugg.split(' ') if sugg else [], truth.split(' ')
|
|
m = 0
|
|
while m < min(len(sw), len(tw)) and TRAIL.sub('', sw[m]) == TRAIL.sub('', tw[m]):
|
|
m += 1
|
|
bare = TRAIL.sub('', sugg)
|
|
full = bool(bare) and truth.startswith(bare)
|
|
acc = bare if full else ' '.join(sw[:m])
|
|
return m, full, max(0, len(acc) - 1) if acc else 0
|
|
|
|
|
|
total_truth = sum(len(norm(c['truth'])) for c in cases)
|
|
print('cases', len(cases), file=out)
|
|
print('tau gate coverage hit|shown full|shown savedRate meanLen', file=out)
|
|
for tau in (-0.5, -1.0, -1.5, -2.0, -3.0, -99):
|
|
for gate in (-0.1, -0.3, -0.5, -1.0, -99):
|
|
shown = hits = full_n = saved = length = 0
|
|
for c in cases:
|
|
if not c['tokens']:
|
|
continue
|
|
sugg, fw = truncate(c['tokens'], tau)
|
|
if not sugg or fw < gate:
|
|
continue
|
|
shown += 1
|
|
m, full, s = score(sugg, c['truth'])
|
|
hits += m >= 1
|
|
full_n += full
|
|
saved += s
|
|
length += len(sugg)
|
|
if shown:
|
|
print(f'{tau:5} {gate:5} {shown/len(cases):.3f} {hits/shown:.3f} {full_n/shown:.3f} {saved/total_truth:.4f} {length/shown:.1f}', file=out)
|