// tests/eval/suggestion-latency.eval.test.ts // // 첫 후보 지연 분해 — 실제 타이핑처럼 문장을 어절 단위로 늘려 가며 앱과 같은 프롬프트로 연달아 요청하고, // 시간이 어디에 쓰이는지(프롬프트 읽기 / 캐시 재사용 / 생성)를 잰다. 프롬프트 순서를 바꿀 가치가 있는지 판단용. // // 실행: D3RO_EVAL_DB= D3RO_EVAL_OUT=<폴더> [D3RO_EVAL_SENTENCES=15] // ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval/suggestion-latency.eval.test.ts import { mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import Database from 'better-sqlite3' import { describe, expect, it, vi } from 'vitest' import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template' vi.mock('../../src/main/services/LoggerService', () => ({ getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }) })) const DB_PATH = process.env.D3RO_EVAL_DB const OUT_DIR = process.env.D3RO_EVAL_OUT const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b' const SENTENCES = Number(process.env.D3RO_EVAL_SENTENCES ?? 15) const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434' interface Timing { sentence: number step: number promptTokens: number cachedTokens: number promptMs: number evalMs: number totalMs: number wallMs: number } const ns = (value: number | undefined) => Math.round((value ?? 0) / 1e6) describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion latency breakdown', () => { it( 'replays word-by-word typing', async () => { const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts') const db = new Database(DB_PATH as string, { readonly: true, fileMustExist: true }) const rows = db .prepare("SELECT text FROM typing_samples WHERE source = 'typed' ORDER BY created_at DESC LIMIT 400") .all() as Array<{ text: string }> db.close() const sentences = rows .map((row) => row.text.trim()) .filter((text) => text.split(/\s+/u).length >= 6 && /[가-힣]/u.test(text) && !/https?:|[/\\=&]/u.test(text)) .slice(0, SENTENCES) const timings: Timing[] = [] for (const [sentenceIndex, text] of sentences.entries()) { const words = text.split(/\s+/u) for (let step = 2; step < words.length; step += 1) { const prefix = `${words.slice(0, step).join(' ')} ` const prompt = buildSuggestionPrompt({ prefix, documentText: prefix, suffix: null, appName: 'Discord.exe', windowTitle: null, recentWrites: [], candidates: 1, maxChars: 160, prefill: true }) const raw = renderRawChatPrompt(MODEL, { system: prompt.systemPrompt, user: prompt.text, assistantPrefix: prompt.assistantPrefix ?? '' }) const startedAt = Date.now() const response = await fetch(`${OLLAMA}/api/generate`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: MODEL, prompt: raw, raw: true, stream: false, keep_alive: '10m', options: { temperature: 0.3, num_predict: 64, stop: rawChatStopSequences(MODEL) } }) }) const data = (await response.json()) as { prompt_eval_count?: number prompt_eval_cached_count?: number prompt_eval_duration?: number eval_duration?: number total_duration?: number } timings.push({ sentence: sentenceIndex, step, promptTokens: data.prompt_eval_count ?? 0, cachedTokens: data.prompt_eval_cached_count ?? 0, promptMs: ns(data.prompt_eval_duration), evalMs: ns(data.eval_duration), totalMs: ns(data.total_duration), wallMs: Date.now() - startedAt }) } } const avg = (pick: (t: Timing) => number) => Math.round(timings.reduce((sum, t) => sum + pick(t), 0) / Math.max(1, timings.length)) const summary = { requests: timings.length, avgPromptTokens: avg((t) => t.promptTokens), avgCachedTokens: avg((t) => t.cachedTokens), cachedShare: Math.round((avg((t) => t.cachedTokens) / Math.max(1, avg((t) => t.promptTokens))) * 1000) / 1000, avgPromptMs: avg((t) => t.promptMs), avgEvalMs: avg((t) => t.evalMs), avgTotalMs: avg((t) => t.totalMs), avgWallMs: avg((t) => t.wallMs) } mkdirSync(OUT_DIR as string, { recursive: true }) writeFileSync(join(OUT_DIR as string, 'latency.json'), JSON.stringify({ summary, timings }, null, 2)) expect(timings.length).toBeGreaterThan(0) }, 30 * 60_000 ) })