feat(desktop): built-in Han/Eng layout auto-switching, terminal/Vim support, and Unicode input

This commit is contained in:
Yun Chan 2026-10-27 10:48:07 +09:00
parent 1da04b9e46
commit a3ac44da2b
109 changed files with 8234 additions and 242 deletions

View file

@ -0,0 +1,78 @@
"""Hybrid: LLM first word vs personal n-gram guess on identical cases.
Reports exact first-word hits and 'useful' hits (guess is a prefix of the true first word: Korean stem + ending).
"""
import json
import re
import sqlite3
import sys
from collections import Counter, defaultdict
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
texts += [r[0] for r in db.execute(
"select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")]
cases = json.load(open(cases_path, encoding='utf-8'))
out = open(out_path, 'w', encoding='utf-8')
TRAIL = re.compile(r'[\s.!?。!?…~,"\')]+$')
def words(text):
return [TRAIL.sub('', w) for w in re.sub(r'\s+', ' ', text or '').strip().split(' ') if w]
def ngram_guess(prefix_words, exclude):
bi, tri = defaultdict(Counter), defaultdict(Counter)
for text in texts:
if exclude in re.sub(r'\s+', ' ', text):
continue
ws = words(text)
for i in range(1, len(ws)):
bi[ws[i - 1]][ws[i]] += 1
if i >= 2:
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
if len(prefix_words) >= 2 and tri.get((prefix_words[-2], prefix_words[-1])):
word, n = tri[(prefix_words[-2], prefix_words[-1])].most_common(1)[0]
return word, n, 'tri'
if prefix_words and bi.get(prefix_words[-1]):
word, n = bi[prefix_words[-1]].most_common(1)[0]
return word, n, 'bi'
return None, 0, None
stats = Counter()
for c in cases:
truth = words(c['truth'])
if not truth:
continue
t0 = truth[0]
full = re.sub(r'\s+', ' ', c['prefix'] + c['truth']).strip()
g, n, kind = ngram_guess(words(c['prefix']), full)
llm = words(c['suggestion'])
l0 = llm[0] if llm else None
stats['cases'] += 1
for name, guess in (('llm', l0), ('ngram', g)):
if guess:
stats[f'{name}_shown'] += 1
stats[f'{name}_exact'] += guess == t0
stats[f'{name}_useful'] += len(guess) >= 1 and t0.startswith(guess)
if g and l0:
stats['both'] += 1
if g == l0 or l0.startswith(g):
stats['agree'] += 1
stats['agree_exact'] += g == t0
stats['agree_useful'] += t0.startswith(g)
# hybrid policy: ngram when it has a guess, otherwise LLM
h = g or l0
if h:
stats['hyb_shown'] += 1
stats['hyb_useful'] += t0.startswith(h)
stats['hyb_exact'] += h == t0
N = stats['cases']
p = lambda k, d: f"{stats[k]}/{stats[d]} = {stats[k]/stats[d]:.3f}" if stats[d] else '-'
print(f'cases {N}', file=out)
for name in ('llm', 'ngram', 'hyb'):
print(f"{name:6} shown {stats[name + '_shown']/N:.3f} exact|shown {p(name + '_exact', name + '_shown')} useful|shown {p(name + '_useful', name + '_shown')}", file=out)
print(f"agree (ngram==LLM first word) {p('agree', 'both')} exact {p('agree_exact', 'agree')} useful {p('agree_useful', 'agree')}", file=out)

View file

@ -0,0 +1,63 @@
"""Personal n-gram next-word predictor on the same replay cases (leave-one-out).
Compares with the LLM on identical cases: does the user's own history predict the next word better?
"""
import json
import re
import sqlite3
import sys
from collections import Counter, defaultdict
db_path, cases_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
db = sqlite3.connect('file:' + db_path + '?mode=ro', uri=True)
texts = [r[0] for r in db.execute("select text from typing_samples where source in ('typed','voice')")]
texts += [r[0] for r in db.execute("select coalesce(polished_text, original_text) from history where coalesce(polished_text, original_text) is not null")] if db.execute("select count(*) from sqlite_master where name='history'").fetchone()[0] else []
cases = json.load(open(cases_path, encoding='utf-8'))
out = open(out_path, 'w', encoding='utf-8')
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
def words(text):
return [w for w in re.sub(r'\s+', ' ', text).strip().split(' ') if w]
def build(exclude):
bi, tri = defaultdict(Counter), defaultdict(Counter)
for text in texts:
if exclude and exclude in re.sub(r'\s+', ' ', text):
continue
ws = [TRAIL.sub('', w) for w in words(text)]
for i in range(1, len(ws)):
bi[ws[i - 1]][ws[i]] += 1
if i >= 2:
tri[(ws[i - 2], ws[i - 1])][ws[i]] += 1
return bi, tri
shown = hits = 0
min_count = int(sys.argv[4]) if len(sys.argv) > 4 else 2
examples = []
for c in cases:
full = re.sub(r'\s+', ' ', (c['prefix'] + c['truth'])).strip()
bi, tri = build(full)
pw = [TRAIL.sub('', w) for w in words(c['prefix'])]
truth_first = TRAIL.sub('', words(c['truth'])[0]) if words(c['truth']) else ''
guess = None
if len(pw) >= 2 and tri.get((pw[-2], pw[-1])):
word, n = tri[(pw[-2], pw[-1])].most_common(1)[0]
if n >= min_count:
guess = word
if guess is None and pw and bi.get(pw[-1]):
word, n = bi[pw[-1]].most_common(1)[0]
if n >= min_count:
guess = word
if guess:
shown += 1
hit = guess == truth_first
hits += hit
if len(examples) < 20:
examples.append((c['prefix'][-25:], guess, truth_first, hit))
print(f'corpus texts {len(texts)}; cases {len(cases)}; min_count {min_count}', file=out)
print(f'ngram coverage {shown/len(cases):.3f} hit|shown {hits/shown if shown else 0:.3f} hits {hits}', file=out)
for e in examples:
print(' ', e, file=out)

View file

@ -0,0 +1,74 @@
import json
import math
import re
import sys
cases = json.load(open(sys.argv[1], encoding='utf-8'))
out = open(sys.argv[2], 'w', encoding='utf-8')
END = re.compile(r'[.!?。!?…\n]')
TRAIL = re.compile(r'[\s.!?。!?…~,]+$')
def norm(text):
return re.sub(r'\s+', ' ', text).strip()
def truncate(tokens, tau):
"""Keep tokens while each logprob >= tau; stop at sentence end. Drop a trailing partial word."""
text = ''
first_word_lp = 0.0
first_word_done = False
started = False
for tok in tokens:
t, lp = tok['token'], tok['logprob']
if '<' in t:
break
if lp < tau:
# partial word cut: if this token continues a word (no leading space), drop the tail of the last word
if text and not t.startswith(' ') and not text.endswith(' '):
text = text[: text.rfind(' ') + 1] if ' ' in text.strip() else ''
break
text += t
if text.strip():
started = True
if started and not first_word_done:
first_word_lp += lp
if ' ' in text.strip():
first_word_done = True
if END.search(t):
break
return norm(text), first_word_lp
def score(sugg, truth):
truth = norm(truth)
sw, tw = sugg.split(' ') if sugg else [], truth.split(' ')
m = 0
while m < min(len(sw), len(tw)) and TRAIL.sub('', sw[m]) == TRAIL.sub('', tw[m]):
m += 1
bare = TRAIL.sub('', sugg)
full = bool(bare) and truth.startswith(bare)
acc = bare if full else ' '.join(sw[:m])
return m, full, max(0, len(acc) - 1) if acc else 0
total_truth = sum(len(norm(c['truth'])) for c in cases)
print('cases', len(cases), file=out)
print('tau gate coverage hit|shown full|shown savedRate meanLen', file=out)
for tau in (-0.5, -1.0, -1.5, -2.0, -3.0, -99):
for gate in (-0.1, -0.3, -0.5, -1.0, -99):
shown = hits = full_n = saved = length = 0
for c in cases:
if not c['tokens']:
continue
sugg, fw = truncate(c['tokens'], tau)
if not sugg or fw < gate:
continue
shown += 1
m, full, s = score(sugg, c['truth'])
hits += m >= 1
full_n += full
saved += s
length += len(sugg)
if shown:
print(f'{tau:5} {gate:5} {shown/len(cases):.3f} {hits/shown:.3f} {full_n/shown:.3f} {saved/total_truth:.4f} {length/shown:.1f}', file=out)

View file

@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest'
import { scoreReplayCase, summarizeReplay, type ReplayCaseInput } from './replay-metrics'
function input(suggestion: string | null, truth: string, logprobs: number[] = [-0.1]): ReplayCaseInput {
return {
appName: 'Discord.exe',
prefix: '오늘 회의는 ',
truth,
raw: suggestion ?? '',
suggestion,
tokens: logprobs.map((logprob, index) => ({ token: `t${index}`, logprob })),
latencyMs: 300,
promptTokens: 100,
cachedPromptTokens: 80
}
}
describe('replay-metrics', () => {
it('맞은 어절까지 수락했다고 보고 아낀 글자를 센다', () => {
const result = scoreReplayCase(input('내일 오후 세 시에 해요', '내일 오후 네 시로 옮겼어'))
expect(result).toMatchObject({ shown: true, matchedWords: 2, firstWordHit: true, fullAccept: false })
expect(result.savedChars).toBe('내일 오후'.length - 1)
})
it('제안 전체가 실제 글 앞부분이면 통째로 수락한 것으로 본다 (끝 문장부호 무시)', () => {
const result = scoreReplayCase(input('내일 오후.', '내일 오후 세 시야'))
expect(result.fullAccept).toBe(true)
expect(result.savedChars).toBe('내일 오후'.length - 1)
})
it('첫 어절부터 다르면 아낀 글자는 0', () => {
const result = scoreReplayCase(input('다음 주에 다시 잡자', '내일 오후 세 시야'))
expect(result).toMatchObject({ firstWordHit: false, savedChars: 0 })
})
it('요약: 노출률·적중률·게이팅 곡선', () => {
const summary = summarizeReplay([
scoreReplayCase(input('내일 오후', '내일 오후 세 시야', [-0.05])),
scoreReplayCase(input('다음 주에', '내일 오후 세 시야', [-2.5])),
scoreReplayCase(input(null, '내일 오후 세 시야', []))
])
expect(summary).toMatchObject({ cases: 3, coverage: 0.667, firstWordHitAmongShown: 0.5, cachedPromptShare: 0.8 })
const strict = summary.gatingCurve.find((point) => point.minMeanLogprob === -0.1)
expect(strict).toMatchObject({ coverage: 0.333, firstWordHitAmongShown: 1 })
})
})

View file

@ -0,0 +1,166 @@
// tests/eval/replay-metrics.ts
// 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증).
//
// 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고):
// - coverage: 제안을 보여 준 비율
// - firstWordHit: 첫 어절이 정확히 맞음
// - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음)
// - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회
// - savedRate: Σ savedChars / Σ 정답 글자 수
export interface ReplayToken {
token: string
logprob: number
}
export interface ReplayCaseInput {
appName: string | null
prefix: string
truth: string
raw: string
suggestion: string | null
tokens: ReplayToken[]
latencyMs: number
promptTokens: number | null
cachedPromptTokens: number | null
}
export interface ReplayCaseResult extends ReplayCaseInput {
shown: boolean
matchedWords: number
commonPrefixChars: number
firstWordHit: boolean
fullAccept: boolean
savedChars: number
truthChars: number
firstTokenLogprob: number | null
meanLogprob: number | null
minLogprob: number | null
}
const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u
function normalize(text: string): string {
return text.replace(/\s+/gu, ' ').trim()
}
function words(text: string): string[] {
const normalized = normalize(text)
return normalized ? normalized.split(' ') : []
}
export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult {
const truth = normalize(input.truth)
const suggestion = input.suggestion ? normalize(input.suggestion) : ''
const shown = suggestion.length > 0
let commonPrefixChars = 0
while (
commonPrefixChars < Math.min(truth.length, suggestion.length) &&
truth[commonPrefixChars] === suggestion[commonPrefixChars]
) {
commonPrefixChars += 1
}
const truthWords = words(truth)
const suggestionWords = words(suggestion)
let matchedWords = 0
while (
matchedWords < Math.min(truthWords.length, suggestionWords.length) &&
truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ===
suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '')
) {
matchedWords += 1
}
const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '')
const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion)
const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ')
const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0
const logprobs = input.tokens.map((token) => token.logprob)
return {
...input,
shown,
matchedWords,
commonPrefixChars,
firstWordHit: matchedWords >= 1,
fullAccept,
savedChars,
truthChars: truth.length,
firstTokenLogprob: logprobs[0] ?? null,
meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null,
minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null
}
}
function percentile(values: number[], p: number): number | null {
if (values.length === 0) return null
const sorted = [...values].sort((a, b) => a - b)
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]
}
function rate(part: number, whole: number): number {
return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000
}
export interface ReplaySummary {
cases: number
coverage: number
firstWordHitRate: number
firstWordHitAmongShown: number
fullAcceptRate: number
meanCommonPrefixChars: number
savedRate: number
latencyP50: number | null
latencyP90: number | null
cachedPromptShare: number | null
/** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */
gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }>
}
export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary {
const shown = results.filter((result) => result.shown)
const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0)
const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0)
const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0)
const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => {
const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold)
return {
minMeanLogprob: threshold,
coverage: rate(kept.length, results.length),
firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length),
savedRate: rate(
kept.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
)
}
})
return {
cases: results.length,
coverage: rate(shown.length, results.length),
firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length),
firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length),
fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length),
meanCommonPrefixChars:
shown.length === 0
? 0
: Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100,
savedRate: rate(
results.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
),
latencyP50: percentile(
results.map((result) => result.latencyMs),
50
),
latencyP90: percentile(
results.map((result) => result.latencyMs),
90
),
cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens),
gatingCurve
}
}

View file

@ -0,0 +1,125 @@
// tests/eval/suggestion-latency.eval.test.ts
//
// 첫 후보 지연 분해 — 실제 타이핑처럼 문장을 어절 단위로 늘려 가며 앱과 같은 프롬프트로 연달아 요청하고,
// 시간이 어디에 쓰이는지(프롬프트 읽기 / 캐시 재사용 / 생성)를 잰다. 프롬프트 순서를 바꿀 가치가 있는지 판단용.
//
// 실행: D3RO_EVAL_DB=<d3ro.db> D3RO_EVAL_OUT=<폴더> [D3RO_EVAL_SENTENCES=15]
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval/suggestion-latency.eval.test.ts
import { mkdirSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import Database from 'better-sqlite3'
import { describe, expect, it, vi } from 'vitest'
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
vi.mock('../../src/main/services/LoggerService', () => ({
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
}))
const DB_PATH = process.env.D3RO_EVAL_DB
const OUT_DIR = process.env.D3RO_EVAL_OUT
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
const SENTENCES = Number(process.env.D3RO_EVAL_SENTENCES ?? 15)
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
interface Timing {
sentence: number
step: number
promptTokens: number
cachedTokens: number
promptMs: number
evalMs: number
totalMs: number
wallMs: number
}
const ns = (value: number | undefined) => Math.round((value ?? 0) / 1e6)
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion latency breakdown', () => {
it(
'replays word-by-word typing',
async () => {
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
const db = new Database(DB_PATH as string, { readonly: true, fileMustExist: true })
const rows = db
.prepare("SELECT text FROM typing_samples WHERE source = 'typed' ORDER BY created_at DESC LIMIT 400")
.all() as Array<{ text: string }>
db.close()
const sentences = rows
.map((row) => row.text.trim())
.filter((text) => text.split(/\s+/u).length >= 6 && /[가-힣]/u.test(text) && !/https?:|[/\\=&]/u.test(text))
.slice(0, SENTENCES)
const timings: Timing[] = []
for (const [sentenceIndex, text] of sentences.entries()) {
const words = text.split(/\s+/u)
for (let step = 2; step < words.length; step += 1) {
const prefix = `${words.slice(0, step).join(' ')} `
const prompt = buildSuggestionPrompt({
prefix,
documentText: prefix,
suffix: null,
appName: 'Discord.exe',
windowTitle: null,
recentWrites: [],
candidates: 1,
maxChars: 160,
prefill: true
})
const raw = renderRawChatPrompt(MODEL, {
system: prompt.systemPrompt,
user: prompt.text,
assistantPrefix: prompt.assistantPrefix ?? ''
})
const startedAt = Date.now()
const response = await fetch(`${OLLAMA}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: MODEL,
prompt: raw,
raw: true,
stream: false,
keep_alive: '10m',
options: { temperature: 0.3, num_predict: 64, stop: rawChatStopSequences(MODEL) }
})
})
const data = (await response.json()) as {
prompt_eval_count?: number
prompt_eval_cached_count?: number
prompt_eval_duration?: number
eval_duration?: number
total_duration?: number
}
timings.push({
sentence: sentenceIndex,
step,
promptTokens: data.prompt_eval_count ?? 0,
cachedTokens: data.prompt_eval_cached_count ?? 0,
promptMs: ns(data.prompt_eval_duration),
evalMs: ns(data.eval_duration),
totalMs: ns(data.total_duration),
wallMs: Date.now() - startedAt
})
}
}
const avg = (pick: (t: Timing) => number) =>
Math.round(timings.reduce((sum, t) => sum + pick(t), 0) / Math.max(1, timings.length))
const summary = {
requests: timings.length,
avgPromptTokens: avg((t) => t.promptTokens),
avgCachedTokens: avg((t) => t.cachedTokens),
cachedShare: Math.round((avg((t) => t.cachedTokens) / Math.max(1, avg((t) => t.promptTokens))) * 1000) / 1000,
avgPromptMs: avg((t) => t.promptMs),
avgEvalMs: avg((t) => t.evalMs),
avgTotalMs: avg((t) => t.totalMs),
avgWallMs: avg((t) => t.wallMs)
}
mkdirSync(OUT_DIR as string, { recursive: true })
writeFileSync(join(OUT_DIR as string, 'latency.json'), JSON.stringify({ summary, timings }, null, 2))
expect(timings.length).toBeGreaterThan(0)
},
30 * 60_000
)
})

View file

@ -0,0 +1,287 @@
// tests/eval/suggestion-replay.eval.test.ts
//
// 문장 예측 재생 평가 — 사용자가 실제로 친 문장을 어절 경계에서 잘라 앞부분으로 제안을 만들고,
// 뒷부분(실제로 이어 친 글)과 비교한다. 앱과 같은 경로(buildSuggestionPrompt → renderRawChatPrompt
// → Ollama raw → extractContinuation → 사실 검사)를 쓴다. 개인 힌트(문구·그래프)는 아직 넣지 않는다.
//
// 평소 테스트에서는 건너뛴다. 실행(앱의 Ollama 가 떠 있어야 한다):
// D3RO_EVAL_DB=<d3ro.db 경로> D3RO_EVAL_OUT=<결과 폴더> [D3RO_EVAL_CASES=120] [D3RO_EVAL_MODEL=gemma4:e4b]
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval
//
// 결과: cases.json(케이스별 원문·제안·토큰 확률) + summary.json. 토큰 확률을 남겨 두면 게이팅·절단
// 기준값을 모델을 다시 돌리지 않고 오프라인으로 고를 수 있다.
import { mkdirSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import Database from 'better-sqlite3'
import { describe, expect, it, vi } from 'vitest'
import { extractContinuation } from '@d3ro/core/input-intelligence'
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
import { splitSentences } from '@d3ro/core/personal-graph'
import { findUnsupportedFacts } from '@d3ro/core/writing-context'
import { PersonalNgramModel, predictPersonalContinuation } from '@d3ro/core/personal-ngram'
import { copiesContext, cutByConfidence } from '@d3ro/core/suggestion-confidence'
import { scoreReplayCase, summarizeReplay, type ReplayCaseResult } from './replay-metrics'
vi.mock('../../src/main/services/LoggerService', () => ({
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
}))
const DB_PATH = process.env.D3RO_EVAL_DB
const OUT_DIR = process.env.D3RO_EVAL_OUT
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
const CASES = Number(process.env.D3RO_EVAL_CASES ?? 120)
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
/**
* app — 앱과 같은 지시형 prefill 프롬프트, 문맥은 지금 문장만
* recent — app + 같은 앱에서 15분 안에 먼저 쓴 글(앱의 recentWrites 와 같은 자리)
* plain — 지시문·템플릿 없이 앞 글 + 지금 문장을 그대로 이어 쓰게 한다(순수 언어 모델 이어쓰기)
* ngram — 개인 n-gram 만 (모델 호출 없음). 정답 문장이 든 기록은 빼고 만든다(leave-one-out)
* hybrid — ngram 이 내면 그것, 아니면 recent
*/
const VARIANT = (process.env.D3RO_EVAL_VARIANT ?? 'app') as 'app' | 'recent' | 'plain' | 'ngram' | 'hybrid'
/** word — 어절 경계(단어를 끝내고 멈춤), mid — 단어 중간(치다가 멈춤) */
const CUT = (process.env.D3RO_EVAL_CUT ?? 'word') as 'word' | 'mid'
/** 1 이면 앱처럼 모델 출력을 토큰 확신으로 거르고 자른다 (core cutByConfidence) */
const GATE = process.env.D3RO_EVAL_GATE === '1'
const RECENT_WINDOW_MS = 15 * 60_000
/** 앱 기본값과 같다 (SUGGESTION_DEFAULTS.maxOutputTokens) */
const MAX_TOKENS = 64
interface ReplaySource {
text: string
appName: string | null
createdAt: number
}
interface GenerateResponse {
response: string
total_duration?: number
prompt_eval_count?: number
prompt_eval_cached_count?: number
logprobs?: Array<{ token: string; logprob: number }>
}
/** 결정적 의사 난수 — 실행마다 같은 케이스를 고른다(변경 전후 비교). */
function seeded(seed: number): () => number {
let state = seed
return () => {
state = (state * 1_103_515_245 + 12_345) % 2_147_483_648
return state / 2_147_483_648
}
}
/** 평가할 문장 — 직접 친 글 중 어절 4개 이상, 주소·명령 같은 것은 뺀다. */
function loadSources(dbPath: string): ReplaySource[] {
const db = new Database(dbPath, { readonly: true, fileMustExist: true })
try {
const rows = db
.prepare(
"SELECT text, app_name AS appName, created_at AS createdAt FROM typing_samples WHERE source = 'typed' ORDER BY created_at"
)
.all() as ReplaySource[]
const seen = new Set<string>()
return rows.filter((row) => {
const text = row.text.trim()
if (seen.has(text)) return false
seen.add(text)
const words = text.split(/\s+/u)
return words.length >= 4 && text.length >= 15 && !/https?:|www\.|[/\\=&]/u.test(text) && /[가-힣]/u.test(text)
})
} finally {
db.close()
}
}
/** n-gram 이 배울 기록 전부 — 앱과 같이 출처를 가리지 않는다. */
function loadCorpus(dbPath: string): string[] {
const db = new Database(dbPath, { readonly: true, fileMustExist: true })
try {
return (db.prepare('SELECT text FROM typing_samples ORDER BY created_at').all() as Array<{ text: string }>).map(
(row) => row.text
)
} finally {
db.close()
}
}
function squash(text: string): string {
return text.replace(/\s+/gu, ' ').trim()
}
/**
* 끊을 자리 하나를 골라 (앞부분, 정답) 으로 자른다. 정답이 최소 2어절 남게 한다.
* word 는 공백 바로 뒤, mid 는 두 글자 이상 어절의 첫 글자 뒤(그 어절의 나머지가 정답 첫머리).
*/
function cut(text: string, random: () => number): { prefix: string; truth: string } | null {
const boundaries: number[] = []
for (let index = 1; index < text.length; index += 1) {
if (CUT === 'word') {
if (text[index] === ' ' && text[index - 1] !== ' ') boundaries.push(index + 1)
} else if (text[index - 1] === ' ' && index + 1 < text.length && /\S/u.test(text[index]) && /\S/u.test(text[index + 1])) {
boundaries.push(index + 1)
}
}
const usable = boundaries.filter((at) => at >= 6 && text.slice(at).trim().split(/\s+/u).length >= 2)
if (usable.length === 0) return null
const at = usable[Math.floor(random() * usable.length)]
return { prefix: text.slice(0, at), truth: text.slice(at) }
}
async function generate(prompt: string, stop: string[]): Promise<{ data: GenerateResponse; ms: number }> {
const startedAt = Date.now()
const response = await fetch(`${OLLAMA}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: MODEL,
prompt,
raw: true,
stream: false,
logprobs: true,
top_logprobs: 0,
keep_alive: '10m',
// 앱은 0.3 으로 뽑는다 — 비교 가능하게 하려고 평가는 결정적으로(0) 돌린다.
options: { temperature: 0, num_predict: MAX_TOKENS, stop }
})
})
if (!response.ok) throw new Error(`ollama ${response.status}: ${await response.text()}`)
return { data: (await response.json()) as GenerateResponse, ms: Date.now() - startedAt }
}
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion replay eval', () => {
it(
'replays typed sentences through the suggestion pipeline',
async () => {
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
const sources = loadSources(DB_PATH as string)
const random = seeded(20261024)
const picked: Array<ReplaySource & { prefix: string; truth: string }> = []
const pool = [...sources]
while (picked.length < CASES && pool.length > 0) {
const source = pool.splice(Math.floor(random() * pool.length), 1)[0]
const split = cut(source.text, random)
if (split) picked.push({ ...source, ...split })
}
const corpus = VARIANT === 'ngram' || VARIANT === 'hybrid' ? loadCorpus(DB_PATH as string) : []
const results: ReplayCaseResult[] = []
for (const item of picked) {
if (VARIANT === 'ngram' || VARIANT === 'hybrid') {
// 정답 문장이 든 기록은 빼고 배운다 — 그대로 두면 답을 외워서 맞힌다.
const sentence = squash(item.text)
const ngram = new PersonalNgramModel()
for (const text of corpus) if (!squash(text).includes(sentence)) ngram.addText(text)
const startedAt = Date.now()
// 기준값을 주지 않으면 앱과 같은 값(predictPersonalContinuation)을 쓴다 — 격자 탐색 때만 덮는다.
const prediction = process.env.D3RO_EVAL_NGRAM_SUPPORT
? ngram.predict(item.prefix, {
minSupport: Number(process.env.D3RO_EVAL_NGRAM_SUPPORT),
minConfidence: Number(process.env.D3RO_EVAL_NGRAM_CONFIDENCE ?? 0.25),
maxWords: Number(process.env.D3RO_EVAL_NGRAM_WORDS ?? 3)
})
: predictPersonalContinuation(ngram, item.prefix)
if (prediction || VARIANT === 'ngram') {
results.push(
scoreReplayCase({
appName: item.appName,
prefix: item.prefix,
truth: item.truth,
raw: prediction?.text ?? '',
suggestion: prediction?.text ?? null,
tokens: [],
latencyMs: Date.now() - startedAt,
promptTokens: null,
cachedPromptTokens: null
})
)
continue
}
}
// 앞서 쓴 글 — 같은 앱, 15분 안, 오래된 것부터 최대 4개 (자기 자신·뒤에 쓴 글은 넣지 않는다)
const recent =
VARIANT === 'app'
? []
: sources
.filter(
(other) =>
other.appName === item.appName &&
other.createdAt < item.createdAt &&
item.createdAt - other.createdAt <= RECENT_WINDOW_MS
)
.slice(-4)
const prompt = buildSuggestionPrompt({
prefix: item.prefix,
documentText: item.prefix,
suffix: null,
appName: item.appName,
windowTitle: null,
recentWrites: recent.map((other) => ({ text: other.text, ageMs: item.createdAt - other.createdAt })),
candidates: 1,
maxChars: 160,
prefill: true
})
const rawPrompt =
VARIANT === 'plain'
? [...recent.map((other) => other.text), item.prefix.trimEnd()].join('\n')
: renderRawChatPrompt(MODEL, {
system: prompt.systemPrompt,
user: prompt.text,
assistantPrefix: prompt.assistantPrefix ?? ''
})
if (rawPrompt === null) throw new Error(`model ${MODEL} has no raw template`)
const stop = VARIANT === 'plain' ? ['\n'] : rawChatStopSequences(MODEL)
const { data, ms } = await generate(rawPrompt, stop)
// 앱(SuggestionService keepConfidentPart)과 같은 거르기 — 첫 어절이 자신 없으면 숨긴다.
const tokens = data.logprobs ?? []
const cutResult = GATE && tokens.length > 0 ? cutByConfidence(tokens) : undefined
const joined = tokens.map((entry) => entry.token).join('')
const confidentRaw =
cutResult === undefined
? data.response
: cutResult === null
? ''
: data.response.startsWith(joined.slice(0, cutResult.keepChars))
? joined.slice(0, cutResult.keepChars)
: data.response
const continuation = extractContinuation(confidentRaw, {
mode: prompt.writing.mode,
prefill: prompt.assistantPrefix ?? '',
writtenSentences: splitSentences(item.prefix),
wordFinished: prompt.wordFinished,
allowEmoji: prompt.allowEmoji,
maxChars: 160
})
// 앱과 같은 거부 규칙: 지어낸 사실 · 앞 글 베끼기
const contextText = [...recent.map((other) => other.text), item.prefix, prompt.factSource].join(' ')
const rejected = !continuation
? null
: findUnsupportedFacts(continuation.text, prompt.factSource).length > 0
? 'unsupported-facts'
: copiesContext(continuation.text, contextText)
? 'copied-context'
: null
results.push(
scoreReplayCase({
appName: item.appName,
prefix: item.prefix,
truth: item.truth,
raw: data.response,
suggestion: rejected ? null : continuation?.text ?? null,
tokens: (data.logprobs ?? []).map((entry) => ({ token: entry.token, logprob: entry.logprob })),
latencyMs: ms,
promptTokens: data.prompt_eval_count ?? null,
cachedPromptTokens: data.prompt_eval_cached_count ?? null
})
)
}
mkdirSync(OUT_DIR as string, { recursive: true })
writeFileSync(join(OUT_DIR as string, 'cases.json'), JSON.stringify(results, null, 2))
const summary = { model: MODEL, variant: VARIANT, cut: CUT, gate: GATE, sources: sources.length, ...summarizeReplay(results) }
writeFileSync(join(OUT_DIR as string, 'summary.json'), JSON.stringify(summary, null, 2))
expect(results.length).toBeGreaterThan(0)
},
30 * 60_000
)
})