feat(desktop): built-in Han/Eng layout auto-switching, terminal/Vim support, and Unicode input

This commit is contained in:
Yun Chan 2026-10-27 10:48:07 +09:00
parent 1da04b9e46
commit a3ac44da2b
109 changed files with 8234 additions and 242 deletions

View file

@ -0,0 +1,166 @@
// tests/eval/replay-metrics.ts
// 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증).
//
// 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고):
// - coverage: 제안을 보여 준 비율
// - firstWordHit: 첫 어절이 정확히 맞음
// - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음)
// - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회
// - savedRate: Σ savedChars / Σ 정답 글자 수
export interface ReplayToken {
token: string
logprob: number
}
export interface ReplayCaseInput {
appName: string | null
prefix: string
truth: string
raw: string
suggestion: string | null
tokens: ReplayToken[]
latencyMs: number
promptTokens: number | null
cachedPromptTokens: number | null
}
export interface ReplayCaseResult extends ReplayCaseInput {
shown: boolean
matchedWords: number
commonPrefixChars: number
firstWordHit: boolean
fullAccept: boolean
savedChars: number
truthChars: number
firstTokenLogprob: number | null
meanLogprob: number | null
minLogprob: number | null
}
const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u
function normalize(text: string): string {
return text.replace(/\s+/gu, ' ').trim()
}
function words(text: string): string[] {
const normalized = normalize(text)
return normalized ? normalized.split(' ') : []
}
export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult {
const truth = normalize(input.truth)
const suggestion = input.suggestion ? normalize(input.suggestion) : ''
const shown = suggestion.length > 0
let commonPrefixChars = 0
while (
commonPrefixChars < Math.min(truth.length, suggestion.length) &&
truth[commonPrefixChars] === suggestion[commonPrefixChars]
) {
commonPrefixChars += 1
}
const truthWords = words(truth)
const suggestionWords = words(suggestion)
let matchedWords = 0
while (
matchedWords < Math.min(truthWords.length, suggestionWords.length) &&
truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ===
suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '')
) {
matchedWords += 1
}
const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '')
const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion)
const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ')
const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0
const logprobs = input.tokens.map((token) => token.logprob)
return {
...input,
shown,
matchedWords,
commonPrefixChars,
firstWordHit: matchedWords >= 1,
fullAccept,
savedChars,
truthChars: truth.length,
firstTokenLogprob: logprobs[0] ?? null,
meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null,
minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null
}
}
function percentile(values: number[], p: number): number | null {
if (values.length === 0) return null
const sorted = [...values].sort((a, b) => a - b)
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]
}
function rate(part: number, whole: number): number {
return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000
}
export interface ReplaySummary {
cases: number
coverage: number
firstWordHitRate: number
firstWordHitAmongShown: number
fullAcceptRate: number
meanCommonPrefixChars: number
savedRate: number
latencyP50: number | null
latencyP90: number | null
cachedPromptShare: number | null
/** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */
gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }>
}
export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary {
const shown = results.filter((result) => result.shown)
const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0)
const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0)
const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0)
const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => {
const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold)
return {
minMeanLogprob: threshold,
coverage: rate(kept.length, results.length),
firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length),
savedRate: rate(
kept.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
)
}
})
return {
cases: results.length,
coverage: rate(shown.length, results.length),
firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length),
firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length),
fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length),
meanCommonPrefixChars:
shown.length === 0
? 0
: Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100,
savedRate: rate(
results.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
),
latencyP50: percentile(
results.map((result) => result.latencyMs),
50
),
latencyP90: percentile(
results.map((result) => result.latencyMs),
90
),
cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens),
gatingCurve
}
}