125 lines
4.9 KiB
TypeScript
125 lines
4.9 KiB
TypeScript
// tests/eval/suggestion-latency.eval.test.ts
|
|
//
|
|
// 첫 후보 지연 분해 — 실제 타이핑처럼 문장을 어절 단위로 늘려 가며 앱과 같은 프롬프트로 연달아 요청하고,
|
|
// 시간이 어디에 쓰이는지(프롬프트 읽기 / 캐시 재사용 / 생성)를 잰다. 프롬프트 순서를 바꿀 가치가 있는지 판단용.
|
|
//
|
|
// 실행: D3RO_EVAL_DB=<d3ro.db> D3RO_EVAL_OUT=<폴더> [D3RO_EVAL_SENTENCES=15]
|
|
// ELECTRON_RUN_AS_NODE=1 electron vitest run --config vitest.config.ts tests/eval/suggestion-latency.eval.test.ts
|
|
|
|
import { mkdirSync, writeFileSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import Database from 'better-sqlite3'
|
|
import { describe, expect, it, vi } from 'vitest'
|
|
import { renderRawChatPrompt, rawChatStopSequences } from '@d3ro/core/chat-template'
|
|
|
|
vi.mock('../../src/main/services/LoggerService', () => ({
|
|
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
|
|
}))
|
|
|
|
const DB_PATH = process.env.D3RO_EVAL_DB
|
|
const OUT_DIR = process.env.D3RO_EVAL_OUT
|
|
const MODEL = process.env.D3RO_EVAL_MODEL ?? 'gemma4:e4b'
|
|
const SENTENCES = Number(process.env.D3RO_EVAL_SENTENCES ?? 15)
|
|
const OLLAMA = process.env.D3RO_EVAL_OLLAMA ?? 'http://127.0.0.1:11434'
|
|
|
|
interface Timing {
|
|
sentence: number
|
|
step: number
|
|
promptTokens: number
|
|
cachedTokens: number
|
|
promptMs: number
|
|
evalMs: number
|
|
totalMs: number
|
|
wallMs: number
|
|
}
|
|
|
|
const ns = (value: number | undefined) => Math.round((value ?? 0) / 1e6)
|
|
|
|
describe.skipIf(!DB_PATH || !OUT_DIR)('suggestion latency breakdown', () => {
|
|
it(
|
|
'replays word-by-word typing',
|
|
async () => {
|
|
const { buildSuggestionPrompt } = await import('../../src/main/services/llm-prompts')
|
|
const db = new Database(DB_PATH as string, { readonly: true, fileMustExist: true })
|
|
const rows = db
|
|
.prepare("SELECT text FROM typing_samples WHERE source = 'typed' ORDER BY created_at DESC LIMIT 400")
|
|
.all() as Array<{ text: string }>
|
|
db.close()
|
|
const sentences = rows
|
|
.map((row) => row.text.trim())
|
|
.filter((text) => text.split(/\s+/u).length >= 6 && /[가-힣]/u.test(text) && !/https?:|[/\\=&]/u.test(text))
|
|
.slice(0, SENTENCES)
|
|
|
|
const timings: Timing[] = []
|
|
for (const [sentenceIndex, text] of sentences.entries()) {
|
|
const words = text.split(/\s+/u)
|
|
for (let step = 2; step < words.length; step += 1) {
|
|
const prefix = `${words.slice(0, step).join(' ')} `
|
|
const prompt = buildSuggestionPrompt({
|
|
prefix,
|
|
documentText: prefix,
|
|
suffix: null,
|
|
appName: 'Discord.exe',
|
|
windowTitle: null,
|
|
recentWrites: [],
|
|
candidates: 1,
|
|
maxChars: 160,
|
|
prefill: true
|
|
})
|
|
const raw = renderRawChatPrompt(MODEL, {
|
|
system: prompt.systemPrompt,
|
|
user: prompt.text,
|
|
assistantPrefix: prompt.assistantPrefix ?? ''
|
|
})
|
|
const startedAt = Date.now()
|
|
const response = await fetch(`${OLLAMA}/api/generate`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
model: MODEL,
|
|
prompt: raw,
|
|
raw: true,
|
|
stream: false,
|
|
keep_alive: '10m',
|
|
options: { temperature: 0.3, num_predict: 64, stop: rawChatStopSequences(MODEL) }
|
|
})
|
|
})
|
|
const data = (await response.json()) as {
|
|
prompt_eval_count?: number
|
|
prompt_eval_cached_count?: number
|
|
prompt_eval_duration?: number
|
|
eval_duration?: number
|
|
total_duration?: number
|
|
}
|
|
timings.push({
|
|
sentence: sentenceIndex,
|
|
step,
|
|
promptTokens: data.prompt_eval_count ?? 0,
|
|
cachedTokens: data.prompt_eval_cached_count ?? 0,
|
|
promptMs: ns(data.prompt_eval_duration),
|
|
evalMs: ns(data.eval_duration),
|
|
totalMs: ns(data.total_duration),
|
|
wallMs: Date.now() - startedAt
|
|
})
|
|
}
|
|
}
|
|
|
|
const avg = (pick: (t: Timing) => number) =>
|
|
Math.round(timings.reduce((sum, t) => sum + pick(t), 0) / Math.max(1, timings.length))
|
|
const summary = {
|
|
requests: timings.length,
|
|
avgPromptTokens: avg((t) => t.promptTokens),
|
|
avgCachedTokens: avg((t) => t.cachedTokens),
|
|
cachedShare: Math.round((avg((t) => t.cachedTokens) / Math.max(1, avg((t) => t.promptTokens))) * 1000) / 1000,
|
|
avgPromptMs: avg((t) => t.promptMs),
|
|
avgEvalMs: avg((t) => t.evalMs),
|
|
avgTotalMs: avg((t) => t.totalMs),
|
|
avgWallMs: avg((t) => t.wallMs)
|
|
}
|
|
mkdirSync(OUT_DIR as string, { recursive: true })
|
|
writeFileSync(join(OUT_DIR as string, 'latency.json'), JSON.stringify({ summary, timings }, null, 2))
|
|
expect(timings.length).toBeGreaterThan(0)
|
|
},
|
|
30 * 60_000
|
|
)
|
|
})
|