d3ro-voice/apps/desktop/tests/main/services/local-llm-service.test.ts
Yun Chan d63854db67
Some checks failed
ci / 정본·보안·린트·타입·테스트 (push) Successful in 56s
ci / .NET API 서버 테스트 (push) Successful in 10s
deploy-site / deploy (push) Successful in 38s
ci / 모바일 린트·타입·Jest (push) Successful in 44s
ci / 워크스페이스 빌드 검증 (push) Successful in 54s
ci / Supabase Edge Functions + Cloudflare Worker (push) Successful in 22s
portable-unsigned / portable-windows (push) Failing after 35s
release / release-windows (push) Failing after 13s
fix(desktop): stop pinning local models in memory and cap audio buffers
The PC-wide slowdown was not a leak but model residency: boot warmup loaded
gemma4:e4b (~11.6 GB RAM incl. 5.9 GB pinned host buffer, no mmap on Windows
CUDA) and kept it for 30m, next to the large-v3 sidecar. Measured on the HEAD
build: free RAM 7.9 GB -> 32 MB within 20 s; after this change 10.8 GB with no
Ollama model loaded until typing starts.

- single LOCAL_LLM_KEEP_ALIVE (10m) for every local request; no boot warmup
- unload this app's resident Ollama models (keep_alive 0) on quit
- sidecar POST /unload for the aux slot; captions release it on stop
- partial dictation copies only the tail window, not the whole recording
- cap the voice-conversation listen buffer at 5 minutes
- mic test: generation guard so re-entry cannot orphan lease/timers
- startRealtime: single-flight and removeChannel to stop duplicate bindings
- meeting WAV save writes header and PCM without a concat copy

Docs: AI-01 policy note, GAP-PERF-01 with remaining work.
2026-10-11 11:35:15 +09:00

226 lines
8.6 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { ErrorCode } from '@d3ro/core/errors'
import { initInMemoryConfig, resetInMemoryConfig } from '../../../src/main/services/ConfigService'
import {
getLocalLLMService,
resetLocalLLMServiceForTests,
} from '../../../src/main/services/LocalLLMService'
const encoder = new TextEncoder()
function completedStream(lines: string[]): ReadableStream<Uint8Array> {
return new ReadableStream({
start(controller) {
controller.enqueue(encoder.encode(lines.join('\n')))
controller.close()
},
})
}
function completedGenerateResponse(): Response {
return new Response(JSON.stringify({
model: 'gemma4:e4b',
response: 'ok',
done: true,
}), { status: 200 })
}
function markAvailable(): void {
const service = getLocalLLMService() as unknown as { _available: boolean }
service._available = true
}
describe('LocalLLMService request lifecycle', () => {
beforeEach(() => {
initInMemoryConfig()
resetLocalLLMServiceForTests()
markAvailable()
})
afterEach(() => {
resetLocalLLMServiceForTests()
resetInMemoryConfig()
vi.unstubAllGlobals()
vi.useRealTimers()
})
it('sends the bounded default num_predict to chat requests', async () => {
const cancel = vi.fn()
const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
const body = JSON.parse(String(init?.body)) as { options: { num_predict: number } }
expect(body.options.num_predict).toBe(512)
return new Response(new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode([
JSON.stringify({ message: { content: 'hello' }, done: false }),
JSON.stringify({ message: { content: '' }, done: true }),
].join('\n') + '\n'))
},
cancel,
}), { status: 200 })
})
vi.stubGlobal('fetch', fetchMock)
const output: string[] = []
for await (const token of getLocalLLMService().chatStream([{ role: 'user', content: 'hello' }])) {
output.push(token)
}
expect(output).toEqual(['hello'])
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(cancel).toHaveBeenCalledTimes(1)
})
it('propagates external cancellation and request deadlines to fetch', async () => {
const signals: AbortSignal[] = []
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise<Response>((_resolve, reject) => {
const signal = init?.signal as AbortSignal
signals.push(signal)
signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true })
}))
vi.stubGlobal('fetch', fetchMock)
const external = new AbortController()
const externalRequest = getLocalLLMService().generate('first', { signal: external.signal })
const externalExpectation = expect(externalRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
external.abort()
await externalExpectation
expect(signals[0].aborted).toBe(true)
vi.useFakeTimers()
const timeoutRequest = getLocalLLMService().generate('second', { timeoutMs: 25 })
const timeoutExpectation = expect(timeoutRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingTimeout })
await vi.advanceTimersByTimeAsync(25)
await timeoutExpectation
expect(signals[1].aborted).toBe(true)
})
it('keeps other active requests cancellable after one request completes', async () => {
const signals: AbortSignal[] = []
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => {
const signal = init?.signal as AbortSignal
signals.push(signal)
if (signals.length === 1) return Promise.resolve(completedGenerateResponse())
return new Promise<Response>((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true })
})
})
vi.stubGlobal('fetch', fetchMock)
const first = getLocalLLMService().generate('first')
const second = getLocalLLMService().generate('second')
const third = getLocalLLMService().generate('third')
await expect(first).resolves.toMatchObject({ text: 'ok' })
const secondExpectation = expect(second).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
const thirdExpectation = expect(third).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
getLocalLLMService().cancelGeneration()
await secondExpectation
await thirdExpectation
expect(signals[1].aborted).toBe(true)
expect(signals[2].aborted).toBe(true)
})
it('rejects a stream that reaches EOF without a done frame', async () => {
vi.stubGlobal('fetch', vi.fn(async () => new Response(completedStream([
JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }),
]), { status: 200 })))
const stream = getLocalLLMService().streamGenerate('hello')
await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false })
await expect(stream.next()).rejects.toMatchObject({ code: ErrorCode.LLMProcessingFailed })
})
it('finishes on a done frame without waiting for the server to close the stream', async () => {
const cancel = vi.fn()
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode([
JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }),
JSON.stringify({ model: 'gemma4:e4b', response: '', done: true }),
].join('\n') + '\n'))
},
cancel,
})
vi.stubGlobal('fetch', vi.fn(async () => new Response(body, { status: 200 })))
const stream = getLocalLLMService().streamGenerate('hello', { timeoutMs: 500 })
await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false })
await expect(stream.next()).resolves.toMatchObject({ value: expect.objectContaining({ text: 'partial' }), done: true })
expect(cancel).toHaveBeenCalledTimes(1)
})
it('deduplicates concurrent ensureRunning calls and polling startup', async () => {
const service = getLocalLLMService()
const privateService = service as unknown as {
_ensureRunning: () => Promise<'running' | 'starting' | 'not-installed' | 'failed'>
_checkAvailability: () => Promise<void>
}
let release: ((value: 'running') => void) | undefined
const pending = new Promise<'running'>((resolve) => {
release = resolve
})
const ensure = vi.spyOn(privateService, '_ensureRunning').mockReturnValue(pending)
const first = service.ensureRunning()
const second = service.ensureRunning()
expect(ensure).toHaveBeenCalledTimes(1)
release?.('running')
await expect(Promise.all([first, second])).resolves.toEqual(['running', 'running'])
vi.useFakeTimers()
const availability = vi.spyOn(privateService, '_checkAvailability').mockResolvedValue()
service.startPolling()
service.startPolling()
expect(availability).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5000)
expect(availability).toHaveBeenCalledTimes(2)
service.stopPolling()
})
})
describe('LocalLLMService model residency (2026-10-11 memory policy)', () => {
beforeEach(() => {
initInMemoryConfig()
resetLocalLLMServiceForTests()
markAvailable()
})
afterEach(() => {
resetLocalLLMServiceForTests()
resetInMemoryConfig()
vi.unstubAllGlobals()
})
it('applies the shared 10m keep_alive when a caller does not set one', async () => {
const bodies: Array<{ keep_alive?: string | number }> = []
vi.stubGlobal('fetch', vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
bodies.push(JSON.parse(String(init?.body)) as { keep_alive?: string })
return completedGenerateResponse()
}))
await getLocalLLMService().generate('hi', { model: 'gemma4:e4b' })
expect(bodies[0].keep_alive).toBe('10m')
})
it('unloads only the models this app loaded, once, with keep_alive 0', async () => {
const bodies: Array<{ model: string; keep_alive?: string | number; prompt?: string }> = []
vi.stubGlobal('fetch', vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
bodies.push(JSON.parse(String(init?.body)) as { model: string; keep_alive?: string })
return completedGenerateResponse()
}))
const service = getLocalLLMService()
await service.unloadResidentModels()
expect(bodies).toHaveLength(0)
await service.generate('hi', { model: 'gemma4:e4b' })
await service.unloadResidentModels()
await service.unloadResidentModels()
const unloads = bodies.filter((body) => body.keep_alive === 0)
expect(unloads).toEqual([{ model: 'gemma4:e4b', keep_alive: 0 }])
})
})