d3ro-voice/apps/desktop/tests/eval/replay-metrics.ts

166 lines
5.7 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// tests/eval/replay-metrics.ts
// 재생 평가 점수 — 제안과 실제로 이어 친 글을 비교한다 (순수 함수, replay-metrics.test.ts 가 검증).
//
// 지표 (ChaI-TeA 2024 · Smart Compose 2019 참고):
// - coverage: 제안을 보여 준 비율
// - firstWordHit: 첫 어절이 정확히 맞음
// - fullAccept: 제안 전체가 실제 글의 앞부분과 같음 (통째로 수락해도 고칠 게 없음)
// - savedChars: 맞은 어절까지 수락했다고 보고 아낀 글자 수 − 수락 키 1회
// - savedRate: Σ savedChars / Σ 정답 글자 수
export interface ReplayToken {
token: string
logprob: number
}
export interface ReplayCaseInput {
appName: string | null
prefix: string
truth: string
raw: string
suggestion: string | null
tokens: ReplayToken[]
latencyMs: number
promptTokens: number | null
cachedPromptTokens: number | null
}
export interface ReplayCaseResult extends ReplayCaseInput {
shown: boolean
matchedWords: number
commonPrefixChars: number
firstWordHit: boolean
fullAccept: boolean
savedChars: number
truthChars: number
firstTokenLogprob: number | null
meanLogprob: number | null
minLogprob: number | null
}
const TRAILING_PUNCTUATION = /[\s.!?。!?…~,]+$/u
function normalize(text: string): string {
return text.replace(/\s+/gu, ' ').trim()
}
function words(text: string): string[] {
const normalized = normalize(text)
return normalized ? normalized.split(' ') : []
}
export function scoreReplayCase(input: ReplayCaseInput): ReplayCaseResult {
const truth = normalize(input.truth)
const suggestion = input.suggestion ? normalize(input.suggestion) : ''
const shown = suggestion.length > 0
let commonPrefixChars = 0
while (
commonPrefixChars < Math.min(truth.length, suggestion.length) &&
truth[commonPrefixChars] === suggestion[commonPrefixChars]
) {
commonPrefixChars += 1
}
const truthWords = words(truth)
const suggestionWords = words(suggestion)
let matchedWords = 0
while (
matchedWords < Math.min(truthWords.length, suggestionWords.length) &&
truthWords[matchedWords].replace(TRAILING_PUNCTUATION, '') ===
suggestionWords[matchedWords].replace(TRAILING_PUNCTUATION, '')
) {
matchedWords += 1
}
const bareSuggestion = suggestion.replace(TRAILING_PUNCTUATION, '')
const fullAccept = shown && bareSuggestion.length > 0 && truth.startsWith(bareSuggestion)
const acceptedText = fullAccept ? bareSuggestion : suggestionWords.slice(0, matchedWords).join(' ')
const savedChars = acceptedText.length > 0 ? Math.max(0, acceptedText.length - 1) : 0
const logprobs = input.tokens.map((token) => token.logprob)
return {
...input,
shown,
matchedWords,
commonPrefixChars,
firstWordHit: matchedWords >= 1,
fullAccept,
savedChars,
truthChars: truth.length,
firstTokenLogprob: logprobs[0] ?? null,
meanLogprob: logprobs.length > 0 ? logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length : null,
minLogprob: logprobs.length > 0 ? Math.min(...logprobs) : null
}
}
function percentile(values: number[], p: number): number | null {
if (values.length === 0) return null
const sorted = [...values].sort((a, b) => a - b)
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]
}
function rate(part: number, whole: number): number {
return whole === 0 ? 0 : Math.round((part / whole) * 1000) / 1000
}
export interface ReplaySummary {
cases: number
coverage: number
firstWordHitRate: number
firstWordHitAmongShown: number
fullAcceptRate: number
meanCommonPrefixChars: number
savedRate: number
latencyP50: number | null
latencyP90: number | null
cachedPromptShare: number | null
/** 평균 logprob 기준값별 노출률·첫 어절 적중률 — 게이팅 기준을 고르는 곡선 */
gatingCurve: Array<{ minMeanLogprob: number; coverage: number; firstWordHitAmongShown: number; savedRate: number }>
}
export function summarizeReplay(results: ReplayCaseResult[]): ReplaySummary {
const shown = results.filter((result) => result.shown)
const truthChars = results.reduce((sum, result) => sum + result.truthChars, 0)
const promptTokens = results.reduce((sum, result) => sum + (result.promptTokens ?? 0), 0)
const cached = results.reduce((sum, result) => sum + (result.cachedPromptTokens ?? 0), 0)
const gatingCurve = [-0.1, -0.2, -0.3, -0.5, -0.75, -1, -1.5, -2, -Infinity].map((threshold) => {
const kept = shown.filter((result) => (result.meanLogprob ?? -Infinity) >= threshold)
return {
minMeanLogprob: threshold,
coverage: rate(kept.length, results.length),
firstWordHitAmongShown: rate(kept.filter((result) => result.firstWordHit).length, kept.length),
savedRate: rate(
kept.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
)
}
})
return {
cases: results.length,
coverage: rate(shown.length, results.length),
firstWordHitRate: rate(results.filter((result) => result.firstWordHit).length, results.length),
firstWordHitAmongShown: rate(shown.filter((result) => result.firstWordHit).length, shown.length),
fullAcceptRate: rate(results.filter((result) => result.fullAccept).length, results.length),
meanCommonPrefixChars:
shown.length === 0
? 0
: Math.round((shown.reduce((sum, result) => sum + result.commonPrefixChars, 0) / shown.length) * 100) / 100,
savedRate: rate(
results.reduce((sum, result) => sum + result.savedChars, 0),
truthChars
),
latencyP50: percentile(
results.map((result) => result.latencyMs),
50
),
latencyP90: percentile(
results.map((result) => result.latencyMs),
90
),
cachedPromptShare: promptTokens === 0 ? null : rate(cached, promptTokens),
gatingCurve
}
}