feat(desktop): built-in Han/Eng layout auto-switching, terminal/Vim support, and Unicode input
This commit is contained in:
parent
1da04b9e46
commit
a3ac44da2b
109 changed files with 8234 additions and 242 deletions
166
apps/desktop/tests/eval/replay-metrics.ts
Normal file
166
apps/desktop/tests/eval/replay-metrics.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
// tests/eval/replay-metrics.ts
|
||||
// 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증).
|
||||
//
|
||||
// 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고):
|
||||
// - coverage: 제안을 보여 준 비율
|
||||
// - firstWordHit: 첫 어절이 정확히 맞음
|
||||
// - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음)
|
||||
// - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회
|
||||
// - savedRate: Σ savedChars / Σ 정답 글자 수
|
||||
|
||||
export interface ReplayToken {
|
||||
token: string
|
||||
logprob: number
|
||||
}
|
||||
|
||||
export interface ReplayCaseInput {
|
||||
appName: string | null
|
||||
prefix: string
|
||||
truth: string
|
||||
raw: string
|
||||
suggestion: string | null
|
||||
tokens: ReplayToken[]
|
||||
latencyMs: number
|
||||
promptTokens: number | null
|
||||
cachedPromptTokens: number | null
|
||||
}
|
||||
|
||||
export interface ReplayCaseResult extends ReplayCaseInput {
|
||||
shown: boolean
|
||||
matchedWords: number
|
||||
commonPrefixChars: number
|
||||
firstWordHit: boolean
|
||||
fullAccept: boolean
|
||||
savedChars: number
|
||||
truthChars: number
|
||||
firstTokenLogprob: number | null
|
||||
meanLogprob: number | null
|
||||
minLogprob: number | null
|
||||
}
|
||||
|
||||
const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u
|
||||
|
||||
function normalize(text: string): string {
|
||||
return text.replace(/\s+/gu, ' ').trim()
|
||||
}
|
||||
|
||||
function words(text: string): string[] {
|
||||
const normalized = normalize(text)
|
||||
return normalized ? normalized.split(' ') : []
|
||||
}
|
||||
|
||||
export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult {
|
||||
const truth = normalize(input.truth)
|
||||
const suggestion = input.suggestion ? normalize(input.suggestion) : ''
|
||||
const shown = suggestion.length > 0
|
||||
|
||||
let commonPrefixChars = 0
|
||||
while (
|
||||
commonPrefixChars < Math.min(truth.length, suggestion.length) &&
|
||||
truth[commonPrefixChars] === suggestion[commonPrefixChars]
|
||||
) {
|
||||
commonPrefixChars += 1
|
||||
}
|
||||
|
||||
const truthWords = words(truth)
|
||||
const suggestionWords = words(suggestion)
|
||||
let matchedWords = 0
|
||||
while (
|
||||
matchedWords < Math.min(truthWords.length, suggestionWords.length) &&
|
||||
truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ===
|
||||
suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '')
|
||||
) {
|
||||
matchedWords += 1
|
||||
}
|
||||
|
||||
const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '')
|
||||
const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion)
|
||||
const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ')
|
||||
const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0
|
||||
|
||||
const logprobs = input.tokens.map((token) => token.logprob)
|
||||
return {
|
||||
...input,
|
||||
shown,
|
||||
matchedWords,
|
||||
commonPrefixChars,
|
||||
firstWordHit: matchedWords >= 1,
|
||||
fullAccept,
|
||||
savedChars,
|
||||
truthChars: truth.length,
|
||||
firstTokenLogprob: logprobs[0] ?? null,
|
||||
meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null,
|
||||
minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null
|
||||
}
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number | null {
|
||||
if (values.length === 0) return null
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]
|
||||
}
|
||||
|
||||
function rate(part: number, whole: number): number {
|
||||
return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000
|
||||
}
|
||||
|
||||
export interface ReplaySummary {
|
||||
cases: number
|
||||
coverage: number
|
||||
firstWordHitRate: number
|
||||
firstWordHitAmongShown: number
|
||||
fullAcceptRate: number
|
||||
meanCommonPrefixChars: number
|
||||
savedRate: number
|
||||
latencyP50: number | null
|
||||
latencyP90: number | null
|
||||
cachedPromptShare: number | null
|
||||
/** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */
|
||||
gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }>
|
||||
}
|
||||
|
||||
export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary {
|
||||
const shown = results.filter((result) => result.shown)
|
||||
const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0)
|
||||
const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0)
|
||||
const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0)
|
||||
|
||||
const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => {
|
||||
const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold)
|
||||
return {
|
||||
minMeanLogprob: threshold,
|
||||
coverage: rate(kept.length, results.length),
|
||||
firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length),
|
||||
savedRate: rate(
|
||||
kept.reduce((sum, result) => sum + result.savedChars, 0),
|
||||
truthChars
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
cases: results.length,
|
||||
coverage: rate(shown.length, results.length),
|
||||
firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length),
|
||||
firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length),
|
||||
fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length),
|
||||
meanCommonPrefixChars:
|
||||
shown.length === 0
|
||||
? 0
|
||||
: Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100,
|
||||
savedRate: rate(
|
||||
results.reduce((sum, result) => sum + result.savedChars, 0),
|
||||
truthChars
|
||||
),
|
||||
latencyP50: percentile(
|
||||
results.map((result) => result.latencyMs),
|
||||
50
|
||||
),
|
||||
latencyP90: percentile(
|
||||
results.map((result) => result.latencyMs),
|
||||
90
|
||||
),
|
||||
cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens),
|
||||
gatingCurve
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue