feat(desktop): built-in Han/Eng layout auto-switching, terminal/Vim support, and Unicode input
This commit is contained in:
parent
1da04b9e46
commit
a3ac44da2b
109 changed files with 8234 additions and 242 deletions
78
apps/desktop/tests/eval/analysis/hybrid_eval.py
Normal file
78
apps/desktop/tests/eval/analysis/hybrid_eval.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Hybrid: LLM first word vs personal n-gram guess on identical cases.
|
||||
|
||||
Reports exact first-word hits and 'useful' hits (guess is a prefix of the true first word: Korean stem + ending).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
|
||||
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
|
||||
texts += [r[0] for r in db.execute(
|
||||
"select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")]
|
||||
cases = json.load(open(cases_path, encoding='utf-8'))
|
||||
out = open(out_path, 'w', encoding='utf-8')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,"\')]+$')
|
||||
|
||||
|
||||
def words(text):
|
||||
return [TRAIL.sub('', w) for w in re.sub(r'\s+', ' ', text or '').strip().split(' ') if w]
|
||||
|
||||
|
||||
def ngram_guess(prefix_words, exclude):
|
||||
bi, tri = defaultdict(Counter), defaultdict(Counter)
|
||||
for text in texts:
|
||||
if exclude in re.sub(r'\s+', ' ', text):
|
||||
continue
|
||||
ws = words(text)
|
||||
for i in range(1, len(ws)):
|
||||
bi[ws[i - 1]][ws[i]] += 1
|
||||
if i >= 2:
|
||||
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
|
||||
if len(prefix_words) >= 2 and tri.get((prefix_words[-2], prefix_words[-1])):
|
||||
word, n = tri[(prefix_words[-2], prefix_words[-1])].most_common(1)[0]
|
||||
return word, n, 'tri'
|
||||
if prefix_words and bi.get(prefix_words[-1]):
|
||||
word, n = bi[prefix_words[-1]].most_common(1)[0]
|
||||
return word, n, 'bi'
|
||||
return None, 0, None
|
||||
|
||||
|
||||
stats = Counter()
|
||||
for c in cases:
|
||||
truth = words(c['truth'])
|
||||
if not truth:
|
||||
continue
|
||||
t0 = truth[0]
|
||||
full = re.sub(r'\s+', ' ', c['prefix'] + c['truth']).strip()
|
||||
g, n, kind = ngram_guess(words(c['prefix']), full)
|
||||
llm = words(c['suggestion'])
|
||||
l0 = llm[0] if llm else None
|
||||
stats['cases'] += 1
|
||||
for name, guess in (('llm', l0), ('ngram', g)):
|
||||
if guess:
|
||||
stats[f'{name}_shown'] += 1
|
||||
stats[f'{name}_exact'] += guess == t0
|
||||
stats[f'{name}_useful'] += len(guess) >= 1 and t0.startswith(guess)
|
||||
if g and l0:
|
||||
stats['both'] += 1
|
||||
if g == l0 or l0.startswith(g):
|
||||
stats['agree'] += 1
|
||||
stats['agree_exact'] += g == t0
|
||||
stats['agree_useful'] += t0.startswith(g)
|
||||
# hybrid policy: ngram when it has a guess, otherwise LLM
|
||||
h = g or l0
|
||||
if h:
|
||||
stats['hyb_shown'] += 1
|
||||
stats['hyb_useful'] += t0.startswith(h)
|
||||
stats['hyb_exact'] += h == t0
|
||||
|
||||
N = stats['cases']
|
||||
p = lambda k, d: f"{stats[k]}/{stats[d]} = {stats[k]/stats[d]:.3f}" if stats[d] else '-'
|
||||
print(f'cases {N}', file=out)
|
||||
for name in ('llm', 'ngram', 'hyb'):
|
||||
print(f"{name:6} shown {stats[name + '_shown']/N:.3f} exact|shown {p(name + '_exact', name + '_shown')} useful|shown {p(name + '_useful', name + '_shown')}", file=out)
|
||||
print(f"agree (ngram==LLM first word) {p('agree', 'both')} exact {p('agree_exact', 'agree')} useful {p('agree_useful', 'agree')}", file=out)
|
||||
63
apps/desktop/tests/eval/analysis/ngram_eval.py
Normal file
63
apps/desktop/tests/eval/analysis/ngram_eval.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""Personal n-gram next-word predictor on the same replay cases (leave-one-out).
|
||||
|
||||
Compares with the LLM on identical cases: does the user's own history predict the next word better?
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
|
||||
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
|
||||
texts += [r[0] for r in db.execute("select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")] if db.execute("select count(*) from sqlite_master where name='history'").fetchone()[0] else []
|
||||
cases = json.load(open(cases_path, encoding='utf-8'))
|
||||
out = open(out_path, 'w', encoding='utf-8')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
|
||||
|
||||
|
||||
def words(text):
|
||||
return [w for w in re.sub(r'\s+', ' ', text).strip().split(' ') if w]
|
||||
|
||||
|
||||
def build(exclude):
|
||||
bi, tri = defaultdict(Counter), defaultdict(Counter)
|
||||
for text in texts:
|
||||
if exclude and exclude in re.sub(r'\s+', ' ', text):
|
||||
continue
|
||||
ws = [TRAIL.sub('', w) for w in words(text)]
|
||||
for i in range(1, len(ws)):
|
||||
bi[ws[i - 1]][ws[i]] += 1
|
||||
if i >= 2:
|
||||
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
|
||||
return bi, tri
|
||||
|
||||
|
||||
shown = hits = 0
|
||||
min_count = int(sys.argv[4]) if len(sys.argv) > 4 else 2
|
||||
examples = []
|
||||
for c in cases:
|
||||
full = re.sub(r'\s+', ' ', (c['prefix'] + c['truth'])).strip()
|
||||
bi, tri = build(full)
|
||||
pw = [TRAIL.sub('', w) for w in words(c['prefix'])]
|
||||
truth_first = TRAIL.sub('', words(c['truth'])[0]) if words(c['truth']) else ''
|
||||
guess = None
|
||||
if len(pw) >= 2 and tri.get((pw[-2], pw[-1])):
|
||||
word, n = tri[(pw[-2], pw[-1])].most_common(1)[0]
|
||||
if n >= min_count:
|
||||
guess = word
|
||||
if guess is None and pw and bi.get(pw[-1]):
|
||||
word, n = bi[pw[-1]].most_common(1)[0]
|
||||
if n >= min_count:
|
||||
guess = word
|
||||
if guess:
|
||||
shown += 1
|
||||
hit = guess == truth_first
|
||||
hits += hit
|
||||
if len(examples) < 20:
|
||||
examples.append((c['prefix'][-25:], guess, truth_first, hit))
|
||||
print(f'corpus texts {len(texts)}; cases {len(cases)}; min_count {min_count}', file=out)
|
||||
print(f'ngram coverage {shown/len(cases):.3f} hit|shown {hits/shown if shown else 0:.3f} hits {hits}', file=out)
|
||||
for e in examples:
|
||||
print(' ', e, file=out)
|
||||
74
apps/desktop/tests/eval/analysis/simulate_gating.py
Normal file
74
apps/desktop/tests/eval/analysis/simulate_gating.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
|
||||
cases = json.load(open(sys.argv[1], encoding='utf-8'))
|
||||
out = open(sys.argv[2], 'w', encoding='utf-8')
|
||||
END = re.compile(r'[.!?。!?…\n]')
|
||||
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
|
||||
|
||||
|
||||
def norm(text):
|
||||
return re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
|
||||
def truncate(tokens, tau):
|
||||
"""Keep tokens while each logprob >= tau; stop at sentence end. Drop a trailing partial word."""
|
||||
text = ''
|
||||
first_word_lp = 0.0
|
||||
first_word_done = False
|
||||
started = False
|
||||
for tok in tokens:
|
||||
t, lp = tok['token'], tok['logprob']
|
||||
if '<' in t:
|
||||
break
|
||||
if lp < tau:
|
||||
# partial word cut: if this token continues a word (no leading space), drop the tail of the last word
|
||||
if text and not t.startswith(' ') and not text.endswith(' '):
|
||||
text = text[: text.rfind(' ') + 1] if ' ' in text.strip() else ''
|
||||
break
|
||||
text += t
|
||||
if text.strip():
|
||||
started = True
|
||||
if started and not first_word_done:
|
||||
first_word_lp += lp
|
||||
if ' ' in text.strip():
|
||||
first_word_done = True
|
||||
if END.search(t):
|
||||
break
|
||||
return norm(text), first_word_lp
|
||||
|
||||
|
||||
def score(sugg, truth):
|
||||
truth = norm(truth)
|
||||
sw, tw = sugg.split(' ') if sugg else [], truth.split(' ')
|
||||
m = 0
|
||||
while m < min(len(sw), len(tw)) and TRAIL.sub('', sw[m]) == TRAIL.sub('', tw[m]):
|
||||
m += 1
|
||||
bare = TRAIL.sub('', sugg)
|
||||
full = bool(bare) and truth.startswith(bare)
|
||||
acc = bare if full else ' '.join(sw[:m])
|
||||
return m, full, max(0, len(acc) - 1) if acc else 0
|
||||
|
||||
|
||||
total_truth = sum(len(norm(c['truth'])) for c in cases)
|
||||
print('cases', len(cases), file=out)
|
||||
print('tau gate coverage hit|shown full|shown savedRate meanLen', file=out)
|
||||
for tau in (-0.5, -1.0, -1.5, -2.0, -3.0, -99):
|
||||
for gate in (-0.1, -0.3, -0.5, -1.0, -99):
|
||||
shown = hits = full_n = saved = length = 0
|
||||
for c in cases:
|
||||
if not c['tokens']:
|
||||
continue
|
||||
sugg, fw = truncate(c['tokens'], tau)
|
||||
if not sugg or fw < gate:
|
||||
continue
|
||||
shown += 1
|
||||
m, full, s = score(sugg, c['truth'])
|
||||
hits += m >= 1
|
||||
full_n += full
|
||||
saved += s
|
||||
length += len(sugg)
|
||||
if shown:
|
||||
print(f'{tau:5} {gate:5} {shown/len(cases):.3f} {hits/shown:.3f} {full_n/shown:.3f} {saved/total_truth:.4f} {length/shown:.1f}', file=out)
|
||||
46
apps/desktop/tests/eval/replay-metrics.test.ts
Normal file
46
apps/desktop/tests/eval/replay-metrics.test.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { scoreReplayCase, summarizeReplay, type ReplayCaseInput } from './replay-metrics'
|
||||
|
||||
function input(suggestion: string | null, truth: string, logprobs: number[] = [-0.1]): ReplayCaseInput {
|
||||
return {
|
||||
appName: 'Discord.exe',
|
||||
prefix: '오늘 회의는 ',
|
||||
truth,
|
||||
raw: suggestion ?? '',
|
||||
suggestion,
|
||||
tokens: logprobs.map((logprob, index) => ({ token: `t${index}`, logprob })),
|
||||
latencyMs: 300,
|
||||
promptTokens: 100,
|
||||
cachedPromptTokens: 80
|
||||
}
|
||||
}
|
||||
|
||||
describe('replay-metrics', () => {
|
||||
it('맞은 어절까지 수락했다고 보고 아낀 글자를 센다', () => {
|
||||
const result = scoreReplayCase(input('내일 오후 세 시에 해요', '내일 오후 네 시로 옮겼어'))
|
||||
expect(result).toMatchObject({ shown: true, matchedWords: 2, firstWordHit: true, fullAccept: false })
|
||||
expect(result.savedChars).toBe('내일 오후'.length - 1)
|
||||
})
|
||||
|
||||
it('제안 전체가 실제 글 앞부분이면 통째로 수락한 것으로 본다 (끝 문장부호 무시)', () => {
|
||||
const result = scoreReplayCase(input('내일 오후.', '내일 오후 세 시야'))
|
||||
expect(result.fullAccept).toBe(true)
|
||||
expect(result.savedChars).toBe('내일 오후'.length - 1)
|
||||
})
|
||||
|
||||
it('첫 어절부터 다르면 아낀 글자는 0', () => {
|
||||
const result = scoreReplayCase(input('다음 주에 다시 잡자', '내일 오후 세 시야'))
|
||||
expect(result).toMatchObject({ firstWordHit: false, savedChars: 0 })
|
||||
})
|
||||
|
||||
it('요약: 노출률·적중률·게이팅 곡선', () => {
|
||||
const summary = summarizeReplay([
|
||||
scoreReplayCase(input('내일 오후', '내일 오후 세 시야', [-0.05])),
|
||||
scoreReplayCase(input('다음 주에', '내일 오후 세 시야', [-2.5])),
|
||||
scoreReplayCase(input(null, '내일 오후 세 시야', []))
|
||||
])
|
||||
expect(summary).toMatchObject({ cases: 3, coverage: 0.667, firstWordHitAmongShown: 0.5, cachedPromptShare: 0.8 })
|
||||
const strict = summary.gatingCurve.find((point) => point.minMeanLogprob === -0.1)
|
||||
expect(strict).toMatchObject({ coverage: 0.333, firstWordHitAmongShown: 1 })
|
||||
})
|
||||
})
|
||||
166
apps/desktop/tests/eval/replay-metrics.ts
Normal file
166
apps/desktop/tests/eval/replay-metrics.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
// tests/eval/replay-metrics.ts
|
||||
// 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증).
|
||||
//
|
||||
// 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고):
|
||||
// - coverage: 제안을 보여 준 비율
|
||||
// - firstWordHit: 첫 어절이 정확히 맞음
|
||||
// - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음)
|
||||
// - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회
|
||||
// - savedRate: Σ savedChars / Σ 정답 글자 수
|
||||
|
||||
export interface ReplayToken {
|
||||
token: string
|
||||
logprob: number
|
||||
}
|
||||
|
||||
export interface ReplayCaseInput {
|
||||
appName: string | null
|
||||
prefix: string
|
||||
truth: string
|
||||
raw: string
|
||||
suggestion: string | null
|
||||
tokens: ReplayToken[]
|
||||
latencyMs: number
|
||||
promptTokens: number | null
|
||||
cachedPromptTokens: number | null
|
||||
}
|
||||
|
||||
export interface ReplayCaseResult extends ReplayCaseInput {
|
||||
shown: boolean
|
||||
matchedWords: number
|
||||
commonPrefixChars: number
|
||||
firstWordHit: boolean
|
||||
fullAccept: boolean
|
||||
savedChars: number
|
||||
truthChars: number
|
||||
firstTokenLogprob: number | null
|
||||
meanLogprob: number | null
|
||||
minLogprob: number | null
|
||||
}
|
||||
|
||||
const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u
|
||||
|
||||
function normalize(text: string): string {
|
||||
return text.replace(/\s+/gu, ' ').trim()
|
||||
}
|
||||
|
||||
function words(text: string): string[] {
|
||||
const normalized = normalize(text)
|
||||
return normalized ? normalized.split(' ') : []
|
||||
}
|
||||
|
||||
export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult {
|
||||
const truth = normalize(input.truth)
|
||||
const suggestion = input.suggestion ? normalize(input.suggestion) : ''
|
||||
const shown = suggestion.length > 0
|
||||
|
||||
let commonPrefixChars = 0
|
||||
while (
|
||||
commonPrefixChars < Math.min(truth.length, suggestion.length) &&
|
||||
truth[commonPrefixChars] === suggestion[commonPrefixChars]
|
||||
) {
|
||||
commonPrefixChars += 1
|
||||
}
|
||||
|
||||
const truthWords = words(truth)
|
||||
const suggestionWords = words(suggestion)
|
||||
let matchedWords = 0
|
||||
while (
|
||||
matchedWords < Math.min(truthWords.length, suggestionWords.length) &&
|
||||
truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ===
|
||||
suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '')
|
||||
) {
|
||||
matchedWords += 1
|
||||
}
|
||||
|
||||
const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '')
|
||||
const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion)
|
||||
const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ')
|
||||
const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0
|
||||
|
||||
const logprobs = input.tokens.map((token) => token.logprob)
|
||||
return {
|
||||
...input,
|
||||
shown,
|
||||
matchedWords,
|
||||
commonPrefixChars,
|
||||
firstWordHit: matchedWords >= 1,
|
||||
fullAccept,
|
||||
savedChars,
|
||||
truthChars: truth.length,
|
||||
firstTokenLogprob: logprobs[0] ?? null,
|
||||
meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null,
|
||||
minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null
|
||||
}
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number | null {
|
||||
if (values.length === 0) return null
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]
|
||||
}
|
||||
|
||||
function rate(part: number, whole: number): number {
|
||||
return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000
|
||||
}
|
||||
|
||||
export interface ReplaySummary {
|
||||
cases: number
|
||||
coverage: number
|
||||
firstWordHitRate: number
|
||||
firstWordHitAmongShown: number
|
||||
fullAcceptRate: number
|
||||
meanCommonPrefixChars: number
|
||||
savedRate: number
|
||||
latencyP50: number | null
|
||||
latencyP90: number | null
|
||||
cachedPromptShare: number | null
|
||||
/** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */
|
||||
gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }>
|
||||
}
|
||||
|
||||
export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary {
|
||||
const shown = results.filter((result) => result.shown)
|
||||
const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0)
|
||||
const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0)
|
||||
const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0)
|
||||
|
||||
const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => {
|
||||
const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold)
|
||||
return {
|
||||
minMeanLogprob: threshold,
|
||||
coverage: rate(kept.length, results.length),
|
||||
firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length),
|
||||
savedRate: rate(
|
||||
kept.reduce((sum, result) => sum + result.savedChars, 0),
|
||||
truthChars
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
cases: results.length,
|
||||
coverage: rate(shown.length, results.length),
|
||||
firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length),
|
||||
firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length),
|
||||
fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length),
|
||||
meanCommonPrefixChars:
|
||||
shown.length === 0
|
||||
? 0
|
||||
: Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100,
|
||||
savedRate: rate(
|
||||
results.reduce((sum, result) => sum + result.savedChars, 0),
|
||||
truthChars
|
||||
),
|
||||
latencyP50: percentile(
|
||||
results.map((result) => result.latencyMs),
|
||||
50
|
||||
),
|
||||
latencyP90: percentile(
|
||||
results.map((result) => result.latencyMs),
|
||||
90
|
||||
),
|
||||
cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens),
|
||||
gatingCurve
|
||||
}
|
||||
}
|
||||
125
apps/desktop/tests/eval/suggestion-latency.eval.test.ts
Normal file
125
apps/desktop/tests/eval/suggestion-latency.eval.test.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
// tests/eval/suggestion-latency.eval.test.ts
|
||||
//
|
||||
// 첫 후보 지연 분해 — 실제 타이핑처럼 문장을 어절 단위로 늘려 가며 앱과 같은 프롬프트로 연달아 요청하고,
|
||||
// 시간이 어디에 쓰이는지(프롬프트 읽기 / 캐시 재사용 / 생성)를 잰다. 프롬프트 순서를 바꿀 가치가 있는지 판단용.
|
||||
//
|
||||
// 실행: D3RO_EVAL_DB=<d3ro.db> D3RO_EVAL_OUT=<폴더> [D3RO_EVAL_SENTENCES=15]
|
||||
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval/suggestion-latency.eval.test.ts
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import Database from 'better-sqlite3'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
|
||||
|
||||
vi.mock('../../src/main/services/LoggerService', () => ({
|
||||
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
|
||||
}))
|
||||
|
||||
const DB_PATH = process.env.D3RO_EVAL_DB
|
||||
const OUT_DIR = process.env.D3RO_EVAL_OUT
|
||||
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
|
||||
const SENTENCES = Number(process.env.D3RO_EVAL_SENTENCES ?? 15)
|
||||
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
|
||||
|
||||
interface Timing {
|
||||
sentence: number
|
||||
step: number
|
||||
promptTokens: number
|
||||
cachedTokens: number
|
||||
promptMs: number
|
||||
evalMs: number
|
||||
totalMs: number
|
||||
wallMs: number
|
||||
}
|
||||
|
||||
const ns = (value: number | undefined) => Math.round((value ?? 0) / 1e6)
|
||||
|
||||
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion latency breakdown', () => {
|
||||
it(
|
||||
'replays word-by-word typing',
|
||||
async () => {
|
||||
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
|
||||
const db = new Database(DB_PATH as string, { readonly: true, fileMustExist: true })
|
||||
const rows = db
|
||||
.prepare("SELECT text FROM typing_samples WHERE source = 'typed' ORDER BY created_at DESC LIMIT 400")
|
||||
.all() as Array<{ text: string }>
|
||||
db.close()
|
||||
const sentences = rows
|
||||
.map((row) => row.text.trim())
|
||||
.filter((text) => text.split(/\s+/u).length >= 6 && /[가-힣]/u.test(text) && !/https?:|[/\\=&]/u.test(text))
|
||||
.slice(0, SENTENCES)
|
||||
|
||||
const timings: Timing[] = []
|
||||
for (const [sentenceIndex, text] of sentences.entries()) {
|
||||
const words = text.split(/\s+/u)
|
||||
for (let step = 2; step < words.length; step += 1) {
|
||||
const prefix = `${words.slice(0, step).join(' ')} `
|
||||
const prompt = buildSuggestionPrompt({
|
||||
prefix,
|
||||
documentText: prefix,
|
||||
suffix: null,
|
||||
appName: 'Discord.exe',
|
||||
windowTitle: null,
|
||||
recentWrites: [],
|
||||
candidates: 1,
|
||||
maxChars: 160,
|
||||
prefill: true
|
||||
})
|
||||
const raw = renderRawChatPrompt(MODEL, {
|
||||
system: prompt.systemPrompt,
|
||||
user: prompt.text,
|
||||
assistantPrefix: prompt.assistantPrefix ?? ''
|
||||
})
|
||||
const startedAt = Date.now()
|
||||
const response = await fetch(`${OLLAMA}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
prompt: raw,
|
||||
raw: true,
|
||||
stream: false,
|
||||
keep_alive: '10m',
|
||||
options: { temperature: 0.3, num_predict: 64, stop: rawChatStopSequences(MODEL) }
|
||||
})
|
||||
})
|
||||
const data = (await response.json()) as {
|
||||
prompt_eval_count?: number
|
||||
prompt_eval_cached_count?: number
|
||||
prompt_eval_duration?: number
|
||||
eval_duration?: number
|
||||
total_duration?: number
|
||||
}
|
||||
timings.push({
|
||||
sentence: sentenceIndex,
|
||||
step,
|
||||
promptTokens: data.prompt_eval_count ?? 0,
|
||||
cachedTokens: data.prompt_eval_cached_count ?? 0,
|
||||
promptMs: ns(data.prompt_eval_duration),
|
||||
evalMs: ns(data.eval_duration),
|
||||
totalMs: ns(data.total_duration),
|
||||
wallMs: Date.now() - startedAt
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const avg = (pick: (t: Timing) => number) =>
|
||||
Math.round(timings.reduce((sum, t) => sum + pick(t), 0) / Math.max(1, timings.length))
|
||||
const summary = {
|
||||
requests: timings.length,
|
||||
avgPromptTokens: avg((t) => t.promptTokens),
|
||||
avgCachedTokens: avg((t) => t.cachedTokens),
|
||||
cachedShare: Math.round((avg((t) => t.cachedTokens) / Math.max(1, avg((t) => t.promptTokens))) * 1000) / 1000,
|
||||
avgPromptMs: avg((t) => t.promptMs),
|
||||
avgEvalMs: avg((t) => t.evalMs),
|
||||
avgTotalMs: avg((t) => t.totalMs),
|
||||
avgWallMs: avg((t) => t.wallMs)
|
||||
}
|
||||
mkdirSync(OUT_DIR as string, { recursive: true })
|
||||
writeFileSync(join(OUT_DIR as string, 'latency.json'), JSON.stringify({ summary, timings }, null, 2))
|
||||
expect(timings.length).toBeGreaterThan(0)
|
||||
},
|
||||
30 * 60_000
|
||||
)
|
||||
})
|
||||
287
apps/desktop/tests/eval/suggestion-replay.eval.test.ts
Normal file
287
apps/desktop/tests/eval/suggestion-replay.eval.test.ts
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
// tests/eval/suggestion-replay.eval.test.ts
|
||||
//
|
||||
// 문장 예측 재생 평가 — 사용자가 실제로 친 문장을 어절 경계에서 잘라 앞부분으로 제안을 만들고,
|
||||
// 뒷부분(실제로 이어 친 글)과 비교한다. 앱과 같은 경로(buildSuggestionPrompt → renderRawChatPrompt
|
||||
// → Ollama raw → extractContinuation → 사실 검사)를 쓴다. 개인 힌트(문구·그래프)는 아직 넣지 않는다.
|
||||
//
|
||||
// 평소 테스트에서는 건너뛴다. 실행(앱의 Ollama 가 떠 있어야 한다):
|
||||
// D3RO_EVAL_DB=<d3ro.db 경로> D3RO_EVAL_OUT=<결과 폴더> [D3RO_EVAL_CASES=120] [D3RO_EVAL_MODEL=gemma4:e4b]
|
||||
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval
|
||||
//
|
||||
// 결과: cases.json(케이스별 원문·제안·토큰 확률) + summary.json. 토큰 확률을 남겨 두면 게이팅·절단
|
||||
// 기준값을 모델을 다시 돌리지 않고 오프라인으로 고를 수 있다.
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import Database from 'better-sqlite3'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { extractContinuation } from '@d3ro/core/input-intelligence'
|
||||
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
|
||||
import { splitSentences } from '@d3ro/core/personal-graph'
|
||||
import { findUnsupportedFacts } from '@d3ro/core/writing-context'
|
||||
import { PersonalNgramModel, predictPersonalContinuation } from '@d3ro/core/personal-ngram'
|
||||
import { copiesContext, cutByConfidence } from '@d3ro/core/suggestion-confidence'
|
||||
import { scoreReplayCase, summarizeReplay, type ReplayCaseResult } from './replay-metrics'
|
||||
|
||||
vi.mock('../../src/main/services/LoggerService', () => ({
|
||||
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
|
||||
}))
|
||||
|
||||
const DB_PATH = process.env.D3RO_EVAL_DB
|
||||
const OUT_DIR = process.env.D3RO_EVAL_OUT
|
||||
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
|
||||
const CASES = Number(process.env.D3RO_EVAL_CASES ?? 120)
|
||||
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
|
||||
/**
|
||||
* app — 앱과 같은 지시형 prefill 프롬프트, 문맥은 지금 문장만
|
||||
* recent — app + 같은 앱에서 15분 안에 먼저 쓴 글(앱의 recentWrites 와 같은 자리)
|
||||
* plain — 지시문·템플릿 없이 앞 글 + 지금 문장을 그대로 이어 쓰게 한다(순수 언어 모델 이어쓰기)
|
||||
* ngram — 개인 n-gram 만 (모델 호출 없음). 정답 문장이 든 기록은 빼고 만든다(leave-one-out)
|
||||
* hybrid — ngram 이 내면 그것, 아니면 recent
|
||||
*/
|
||||
const VARIANT = (process.env.D3RO_EVAL_VARIANT ?? 'app') as 'app' | 'recent' | 'plain' | 'ngram' | 'hybrid'
|
||||
/** word — 어절 경계(단어를 끝내고 멈춤), mid — 단어 중간(치다가 멈춤) */
|
||||
const CUT = (process.env.D3RO_EVAL_CUT ?? 'word') as 'word' | 'mid'
|
||||
/** 1 이면 앱처럼 모델 출력을 토큰 확신으로 거르고 자른다 (core cutByConfidence) */
|
||||
const GATE = process.env.D3RO_EVAL_GATE === '1'
|
||||
const RECENT_WINDOW_MS = 15 * 60_000
|
||||
/** 앱 기본값과 같다 (SUGGESTION_DEFAULTS.maxOutputTokens) */
|
||||
const MAX_TOKENS = 64
|
||||
|
||||
interface ReplaySource {
|
||||
text: string
|
||||
appName: string | null
|
||||
createdAt: number
|
||||
}
|
||||
|
||||
interface GenerateResponse {
|
||||
response: string
|
||||
total_duration?: number
|
||||
prompt_eval_count?: number
|
||||
prompt_eval_cached_count?: number
|
||||
logprobs?: Array<{ token: string; logprob: number }>
|
||||
}
|
||||
|
||||
/** 결정적 의사 난수 — 실행마다 같은 케이스를 고른다(변경 전후 비교). */
|
||||
function seeded(seed: number): () => number {
|
||||
let state = seed
|
||||
return () => {
|
||||
state = (state * 1_103_515_245 + 12_345) % 2_147_483_648
|
||||
return state / 2_147_483_648
|
||||
}
|
||||
}
|
||||
|
||||
/** 평가할 문장 — 직접 친 글 중 어절 4개 이상, 주소·명령 같은 것은 뺀다. */
|
||||
function loadSources(dbPath: string): ReplaySource[] {
|
||||
const db = new Database(dbPath, { readonly: true, fileMustExist: true })
|
||||
try {
|
||||
const rows = db
|
||||
.prepare(
|
||||
"SELECT text, app_name AS appName, created_at AS createdAt FROM typing_samples WHERE source = 'typed' ORDER BY created_at"
|
||||
)
|
||||
.all() as ReplaySource[]
|
||||
const seen = new Set<string>()
|
||||
return rows.filter((row) => {
|
||||
const text = row.text.trim()
|
||||
if (seen.has(text)) return false
|
||||
seen.add(text)
|
||||
const words = text.split(/\s+/u)
|
||||
return words.length >= 4 && text.length >= 15 && !/https?:|www\.|[/\\=&]/u.test(text) && /[가-힣]/u.test(text)
|
||||
})
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
/** n-gram 이 배울 기록 전부 — 앱과 같이 출처를 가리지 않는다. */
|
||||
function loadCorpus(dbPath: string): string[] {
|
||||
const db = new Database(dbPath, { readonly: true, fileMustExist: true })
|
||||
try {
|
||||
return (db.prepare('SELECT text FROM typing_samples ORDER BY created_at').all() as Array<{ text: string }>).map(
|
||||
(row) => row.text
|
||||
)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
function squash(text: string): string {
|
||||
return text.replace(/\s+/gu, ' ').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* 끊을 자리 하나를 골라 (앞부분, 정답) 으로 자른다. 정답이 최소 2어절 남게 한다.
|
||||
* word 는 공백 바로 뒤, mid 는 두 글자 이상 어절의 첫 글자 뒤(그 어절의 나머지가 정답 첫머리).
|
||||
*/
|
||||
function cut(text: string, random: () => number): { prefix: string; truth: string } | null {
|
||||
const boundaries: number[] = []
|
||||
for (let index = 1; index < text.length; index += 1) {
|
||||
if (CUT === 'word') {
|
||||
if (text[index] === ' ' && text[index - 1] !== ' ') boundaries.push(index + 1)
|
||||
} else if (text[index - 1] === ' ' && index + 1 < text.length && /\S/u.test(text[index]) && /\S/u.test(text[index + 1])) {
|
||||
boundaries.push(index + 1)
|
||||
}
|
||||
}
|
||||
const usable = boundaries.filter((at) => at >= 6 && text.slice(at).trim().split(/\s+/u).length >= 2)
|
||||
if (usable.length === 0) return null
|
||||
const at = usable[Math.floor(random() * usable.length)]
|
||||
return { prefix: text.slice(0, at), truth: text.slice(at) }
|
||||
}
|
||||
|
||||
async function generate(prompt: string, stop: string[]): Promise<{ data: GenerateResponse; ms: number }> {
|
||||
const startedAt = Date.now()
|
||||
const response = await fetch(`${OLLAMA}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
prompt,
|
||||
raw: true,
|
||||
stream: false,
|
||||
logprobs: true,
|
||||
top_logprobs: 0,
|
||||
keep_alive: '10m',
|
||||
// 앱은 0.3 으로 뽑는다 — 비교 가능하게 하려고 평가는 결정적으로(0) 돌린다.
|
||||
options: { temperature: 0, num_predict: MAX_TOKENS, stop }
|
||||
})
|
||||
})
|
||||
if (!response.ok) throw new Error(`ollama ${response.status}: ${await response.text()}`)
|
||||
return { data: (await response.json()) as GenerateResponse, ms: Date.now() - startedAt }
|
||||
}
|
||||
|
||||
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion replay eval', () => {
|
||||
it(
|
||||
'replays typed sentences through the suggestion pipeline',
|
||||
async () => {
|
||||
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
|
||||
const sources = loadSources(DB_PATH as string)
|
||||
const random = seeded(20261024)
|
||||
const picked: Array<ReplaySource & { prefix: string; truth: string }> = []
|
||||
const pool = [...sources]
|
||||
while (picked.length < CASES && pool.length > 0) {
|
||||
const source = pool.splice(Math.floor(random() * pool.length), 1)[0]
|
||||
const split = cut(source.text, random)
|
||||
if (split) picked.push({ ...source, ...split })
|
||||
}
|
||||
|
||||
const corpus = VARIANT === 'ngram' || VARIANT === 'hybrid' ? loadCorpus(DB_PATH as string) : []
|
||||
const results: ReplayCaseResult[] = []
|
||||
for (const item of picked) {
|
||||
if (VARIANT === 'ngram' || VARIANT === 'hybrid') {
|
||||
// 정답 문장이 든 기록은 빼고 배운다 — 그대로 두면 답을 외워서 맞힌다.
|
||||
const sentence = squash(item.text)
|
||||
const ngram = new PersonalNgramModel()
|
||||
for (const text of corpus) if (!squash(text).includes(sentence)) ngram.addText(text)
|
||||
const startedAt = Date.now()
|
||||
// 기준값을 주지 않으면 앱과 같은 값(predictPersonalContinuation)을 쓴다 — 격자 탐색 때만 덮는다.
|
||||
const prediction = process.env.D3RO_EVAL_NGRAM_SUPPORT
|
||||
? ngram.predict(item.prefix, {
|
||||
minSupport: Number(process.env.D3RO_EVAL_NGRAM_SUPPORT),
|
||||
minConfidence: Number(process.env.D3RO_EVAL_NGRAM_CONFIDENCE ?? 0.25),
|
||||
maxWords: Number(process.env.D3RO_EVAL_NGRAM_WORDS ?? 3)
|
||||
})
|
||||
: predictPersonalContinuation(ngram, item.prefix)
|
||||
if (prediction || VARIANT === 'ngram') {
|
||||
results.push(
|
||||
scoreReplayCase({
|
||||
appName: item.appName,
|
||||
prefix: item.prefix,
|
||||
truth: item.truth,
|
||||
raw: prediction?.text ?? '',
|
||||
suggestion: prediction?.text ?? null,
|
||||
tokens: [],
|
||||
latencyMs: Date.now() - startedAt,
|
||||
promptTokens: null,
|
||||
cachedPromptTokens: null
|
||||
})
|
||||
)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// 앞서 쓴 글 — 같은 앱, 15분 안, 오래된 것부터 최대 4개 (자기 자신·뒤에 쓴 글은 넣지 않는다)
|
||||
const recent =
|
||||
VARIANT === 'app'
|
||||
? []
|
||||
: sources
|
||||
.filter(
|
||||
(other) =>
|
||||
other.appName === item.appName &&
|
||||
other.createdAt < item.createdAt &&
|
||||
item.createdAt - other.createdAt <= RECENT_WINDOW_MS
|
||||
)
|
||||
.slice(-4)
|
||||
const prompt = buildSuggestionPrompt({
|
||||
prefix: item.prefix,
|
||||
documentText: item.prefix,
|
||||
suffix: null,
|
||||
appName: item.appName,
|
||||
windowTitle: null,
|
||||
recentWrites: recent.map((other) => ({ text: other.text, ageMs: item.createdAt - other.createdAt })),
|
||||
candidates: 1,
|
||||
maxChars: 160,
|
||||
prefill: true
|
||||
})
|
||||
const rawPrompt =
|
||||
VARIANT === 'plain'
|
||||
? [...recent.map((other) => other.text), item.prefix.trimEnd()].join('\n')
|
||||
: renderRawChatPrompt(MODEL, {
|
||||
system: prompt.systemPrompt,
|
||||
user: prompt.text,
|
||||
assistantPrefix: prompt.assistantPrefix ?? ''
|
||||
})
|
||||
if (rawPrompt === null) throw new Error(`model ${MODEL} has no raw template`)
|
||||
const stop = VARIANT === 'plain' ? ['\n'] : rawChatStopSequences(MODEL)
|
||||
const { data, ms } = await generate(rawPrompt, stop)
|
||||
// 앱(SuggestionService keepConfidentPart)과 같은 거르기 — 첫 어절이 자신 없으면 숨긴다.
|
||||
const tokens = data.logprobs ?? []
|
||||
const cutResult = GATE && tokens.length > 0 ? cutByConfidence(tokens) : undefined
|
||||
const joined = tokens.map((entry) => entry.token).join('')
|
||||
const confidentRaw =
|
||||
cutResult === undefined
|
||||
? data.response
|
||||
: cutResult === null
|
||||
? ''
|
||||
: data.response.startsWith(joined.slice(0, cutResult.keepChars))
|
||||
? joined.slice(0, cutResult.keepChars)
|
||||
: data.response
|
||||
const continuation = extractContinuation(confidentRaw, {
|
||||
mode: prompt.writing.mode,
|
||||
prefill: prompt.assistantPrefix ?? '',
|
||||
writtenSentences: splitSentences(item.prefix),
|
||||
wordFinished: prompt.wordFinished,
|
||||
allowEmoji: prompt.allowEmoji,
|
||||
maxChars: 160
|
||||
})
|
||||
// 앱과 같은 거부 규칙: 지어낸 사실 · 앞 글 베끼기
|
||||
const contextText = [...recent.map((other) => other.text), item.prefix, prompt.factSource].join(' ')
|
||||
const rejected = !continuation
|
||||
? null
|
||||
: findUnsupportedFacts(continuation.text, prompt.factSource).length > 0
|
||||
? 'unsupported-facts'
|
||||
: copiesContext(continuation.text, contextText)
|
||||
? 'copied-context'
|
||||
: null
|
||||
results.push(
|
||||
scoreReplayCase({
|
||||
appName: item.appName,
|
||||
prefix: item.prefix,
|
||||
truth: item.truth,
|
||||
raw: data.response,
|
||||
suggestion: rejected ? null : continuation?.text ?? null,
|
||||
tokens: (data.logprobs ?? []).map((entry) => ({ token: entry.token, logprob: entry.logprob })),
|
||||
latencyMs: ms,
|
||||
promptTokens: data.prompt_eval_count ?? null,
|
||||
cachedPromptTokens: data.prompt_eval_cached_count ?? null
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
mkdirSync(OUT_DIR as string, { recursive: true })
|
||||
writeFileSync(join(OUT_DIR as string, 'cases.json'), JSON.stringify(results, null, 2))
|
||||
const summary = { model: MODEL, variant: VARIANT, cut: CUT, gate: GATE, sources: sources.length, ...summarizeReplay(results) }
|
||||
writeFileSync(join(OUT_DIR as string, 'summary.json'), JSON.stringify(summary, null, 2))
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
},
|
||||
30 * 60_000
|
||||
)
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue