d3ro-voice/apps/desktop/tests/eval/suggestion-latency.eval.test.ts

125 lines
4.9 KiB
TypeScript

// tests/eval/suggestion-latency.eval.test.ts
//
// 첫 후보 지연 분해 — 실제 타이핑처럼 문장을 어절 단위로 늘려 가며 앱과 같은 프롬프트로 연달아 요청하고,
// 시간이 어디에 쓰이는지(프롬프트 읽기 / 캐시 재사용 / 생성)를 잰다. 프롬프트 순서를 바꿀 가치가 있는지 판단용.
//
// 실행: D3RO_EVAL_DB=<d3ro.db> D3RO_EVAL_OUT=<폴더> [D3RO_EVAL_SENTENCES=15]
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval/suggestion-latency.eval.test.ts
import { mkdirSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import Database from 'better-sqlite3'
import { describe, expect, it, vi } from 'vitest'
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
vi.mock('../../src/main/services/LoggerService', () => ({
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
}))
const DB_PATH = process.env.D3RO_EVAL_DB
const OUT_DIR = process.env.D3RO_EVAL_OUT
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
const SENTENCES = Number(process.env.D3RO_EVAL_SENTENCES ?? 15)
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
interface Timing {
sentence: number
step: number
promptTokens: number
cachedTokens: number
promptMs: number
evalMs: number
totalMs: number
wallMs: number
}
const ns = (value: number | undefined) => Math.round((value ?? 0) / 1e6)
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion latency breakdown', () => {
it(
'replays word-by-word typing',
async () => {
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
const db = new Database(DB_PATH as string, { readonly: true, fileMustExist: true })
const rows = db
.prepare("SELECT text FROM typing_samples WHERE source = 'typed' ORDER BY created_at DESC LIMIT 400")
.all() as Array<{ text: string }>
db.close()
const sentences = rows
.map((row) => row.text.trim())
.filter((text) => text.split(/\s+/u).length >= 6 && /[가-힣]/u.test(text) && !/https?:|[/\\=&]/u.test(text))
.slice(0, SENTENCES)
const timings: Timing[] = []
for (const [sentenceIndex, text] of sentences.entries()) {
const words = text.split(/\s+/u)
for (let step = 2; step < words.length; step += 1) {
const prefix = `${words.slice(0, step).join(' ')} `
const prompt = buildSuggestionPrompt({
prefix,
documentText: prefix,
suffix: null,
appName: 'Discord.exe',
windowTitle: null,
recentWrites: [],
candidates: 1,
maxChars: 160,
prefill: true
})
const raw = renderRawChatPrompt(MODEL, {
system: prompt.systemPrompt,
user: prompt.text,
assistantPrefix: prompt.assistantPrefix ?? ''
})
const startedAt = Date.now()
const response = await fetch(`${OLLAMA}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: MODEL,
prompt: raw,
raw: true,
stream: false,
keep_alive: '10m',
options: { temperature: 0.3, num_predict: 64, stop: rawChatStopSequences(MODEL) }
})
})
const data = (await response.json()) as {
prompt_eval_count?: number
prompt_eval_cached_count?: number
prompt_eval_duration?: number
eval_duration?: number
total_duration?: number
}
timings.push({
sentence: sentenceIndex,
step,
promptTokens: data.prompt_eval_count ?? 0,
cachedTokens: data.prompt_eval_cached_count ?? 0,
promptMs: ns(data.prompt_eval_duration),
evalMs: ns(data.eval_duration),
totalMs: ns(data.total_duration),
wallMs: Date.now() - startedAt
})
}
}
const avg = (pick: (t: Timing) => number) =>
Math.round(timings.reduce((sum, t) => sum + pick(t), 0) / Math.max(1, timings.length))
const summary = {
requests: timings.length,
avgPromptTokens: avg((t) => t.promptTokens),
avgCachedTokens: avg((t) => t.cachedTokens),
cachedShare: Math.round((avg((t) => t.cachedTokens) / Math.max(1, avg((t) => t.promptTokens))) * 1000) / 1000,
avgPromptMs: avg((t) => t.promptMs),
avgEvalMs: avg((t) => t.evalMs),
avgTotalMs: avg((t) => t.totalMs),
avgWallMs: avg((t) => t.wallMs)
}
mkdirSync(OUT_DIR as string, { recursive: true })
writeFileSync(join(OUT_DIR as string, 'latency.json'), JSON.stringify({ summary, timings }, null, 2))
expect(timings.length).toBeGreaterThan(0)
},
30 * 60_000
)
})