// tests/eval/replay-metrics.ts // 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증). // // 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고): // - coverage: 제안을 보여 준 비율 // - firstWordHit: 첫 어절이 정확히 맞음 // - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음) // - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회 // - savedRate: Σ savedChars / Σ 정답 글자 수 export interface ReplayToken { token: string logprob: number } export interface ReplayCaseInput { appName: string | null prefix: string truth: string raw: string suggestion: string | null tokens: ReplayToken[] latencyMs: number promptTokens: number | null cachedPromptTokens: number | null } export interface ReplayCaseResult extends ReplayCaseInput { shown: boolean matchedWords: number commonPrefixChars: number firstWordHit: boolean fullAccept: boolean savedChars: number truthChars: number firstTokenLogprob: number | null meanLogprob: number | null minLogprob: number | null } const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u function normalize(text: string): string { return text.replace(/\s+/gu, ' ').trim() } function words(text: string): string[] { const normalized = normalize(text) return normalized ? normalized.split(' ') : [] } export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult { const truth = normalize(input.truth) const suggestion = input.suggestion ? normalize(input.suggestion) : '' const shown = suggestion.length > 0 let commonPrefixChars = 0 while ( commonPrefixChars < Math.min(truth.length, suggestion.length) && truth[commonPrefixChars] === suggestion[commonPrefixChars] ) { commonPrefixChars += 1 } const truthWords = words(truth) const suggestionWords = words(suggestion) let matchedWords = 0 while ( matchedWords < Math.min(truthWords.length, suggestionWords.length) && truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') === suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ) { matchedWords += 1 } const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '') const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion) const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ') const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0 const logprobs = input.tokens.map((token) => token.logprob) return { ...input, shown, matchedWords, commonPrefixChars, firstWordHit: matchedWords >= 1, fullAccept, savedChars, truthChars: truth.length, firstTokenLogprob: logprobs[0] ?? null, meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null, minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null } } function percentile(values: number[], p: number): number | null { if (values.length === 0) return null const sorted = [...values].sort((a, b) => a - b) return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))] } function rate(part: number, whole: number): number { return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000 } export interface ReplaySummary { cases: number coverage: number firstWordHitRate: number firstWordHitAmongShown: number fullAcceptRate: number meanCommonPrefixChars: number savedRate: number latencyP50: number | null latencyP90: number | null cachedPromptShare: number | null /** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */ gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }> } export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary { const shown = results.filter((result) => result.shown) const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0) const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0) const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0) const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => { const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold) return { minMeanLogprob: threshold, coverage: rate(kept.length, results.length), firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length), savedRate: rate( kept.reduce((sum, result) => sum + result.savedChars, 0), truthChars ) } }) return { cases: results.length, coverage: rate(shown.length, results.length), firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length), firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length), fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length), meanCommonPrefixChars: shown.length === 0 ? 0 : Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100, savedRate: rate( results.reduce((sum, result) => sum + result.savedChars, 0), truthChars ), latencyP50: percentile( results.map((result) => result.latencyMs), 50 ), latencyP90: percentile( results.map((result) => result.latencyMs), 90 ), cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens), gatingCurve } }