Catch pre-header fetch and stream rejections in LocalLLMService._doPullModel and map disposed or aborted pulls to ErrorCode.LLMProcessingCancelled (and unexpected failures to ErrorCode.LLMServerUnreachable). Convert disposed fetch aborts and JSON body aborts in LocalSTTService._fetchDownload and downloadModel into ErrorCode.STTModelDownloadCancelled while wrapping network or timeout errors as ErrorCode.STTModelDownloadFailed.
407 lines
15 KiB
TypeScript
407 lines
15 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
|
import { configSet, initInMemoryConfig, resetInMemoryConfig } from '../../../src/main/services/ConfigService'
|
|
import { availableParallelism } from 'node:os'
|
|
import { inferenceCpuThreads } from '../../../src/main/utils/inference-cpu-budget'
|
|
import {
|
|
getLocalLLMService,
|
|
resetLocalLLMServiceForTests,
|
|
} from '../../../src/main/services/LocalLLMService'
|
|
|
|
const encoder = new TextEncoder()
|
|
|
|
function completedStream(lines: string[]): ReadableStream<Uint8Array> {
|
|
return new ReadableStream({
|
|
start(controller) {
|
|
controller.enqueue(encoder.encode(lines.join('\n')))
|
|
controller.close()
|
|
},
|
|
})
|
|
}
|
|
|
|
function completedGenerateResponse(): Response {
|
|
return new Response(JSON.stringify({
|
|
model: 'gemma4:e4b',
|
|
response: 'ok',
|
|
done: true,
|
|
}), { status: 200 })
|
|
}
|
|
|
|
function markAvailable(): void {
|
|
const service = getLocalLLMService() as unknown as { _available: boolean }
|
|
service._available = true
|
|
}
|
|
|
|
describe('LocalLLMService request lifecycle', () => {
|
|
beforeEach(() => {
|
|
initInMemoryConfig()
|
|
resetLocalLLMServiceForTests()
|
|
markAvailable()
|
|
})
|
|
|
|
afterEach(() => {
|
|
resetLocalLLMServiceForTests()
|
|
resetInMemoryConfig()
|
|
vi.unstubAllGlobals()
|
|
vi.useRealTimers()
|
|
})
|
|
|
|
it.each(['http://127.0.0.1:11434', 'http://192.168.1.2:11434'])(
|
|
'applies CPU headroom across generate, warm-up streaming and chat only for loopback (%s)', async (serverUrl) => {
|
|
configSet('ollamaServerUrl', serverUrl)
|
|
const bodies: Array<{ options: { num_thread?: number }; stream: boolean }> = []
|
|
vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
const body = JSON.parse(String(init?.body)) as (typeof bodies)[number]
|
|
bodies.push(body)
|
|
if (!body.stream) return completedGenerateResponse()
|
|
const frame = String(input).endsWith('/api/chat')
|
|
? { message: { content: 'ok' }, done: true }
|
|
: { model: 'gemma4:e4b', response: 'ok', done: true }
|
|
return new Response(completedStream([JSON.stringify(frame)]), { status: 200 })
|
|
}))
|
|
|
|
const service = getLocalLLMService()
|
|
await service.generate('hello')
|
|
for await (const token of service.streamGenerate('hi', { maxTokens: 1 })) void token
|
|
for await (const token of service.chatStream([{ role: 'user', content: 'hello' }])) void token
|
|
|
|
expect(bodies).toHaveLength(3)
|
|
for (const body of bodies) {
|
|
if (serverUrl.includes('127.0.0.1')) {
|
|
expect(body.options.num_thread).toBe(inferenceCpuThreads(availableParallelism()))
|
|
} else {
|
|
expect(body.options).not.toHaveProperty('num_thread')
|
|
}
|
|
}
|
|
}
|
|
)
|
|
|
|
it('sends the bounded default num_predict to chat requests', async () => {
|
|
const cancel = vi.fn()
|
|
const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
|
|
const body = JSON.parse(String(init?.body)) as { options: { num_predict: number } }
|
|
expect(body.options.num_predict).toBe(512)
|
|
return new Response(new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(encoder.encode([
|
|
JSON.stringify({ message: { content: 'hello' }, done: false }),
|
|
JSON.stringify({ message: { content: '' }, done: true }),
|
|
].join('\n') + '\n'))
|
|
},
|
|
cancel,
|
|
}), { status: 200 })
|
|
})
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const output: string[] = []
|
|
for await (const token of getLocalLLMService().chatStream([{ role: 'user', content: 'hello' }])) {
|
|
output.push(token)
|
|
}
|
|
|
|
expect(output).toEqual(['hello'])
|
|
expect(fetchMock).toHaveBeenCalledTimes(1)
|
|
expect(cancel).toHaveBeenCalledTimes(1)
|
|
})
|
|
|
|
it('propagates external cancellation and request deadlines to fetch', async () => {
|
|
const signals: AbortSignal[] = []
|
|
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise<Response>((_resolve, reject) => {
|
|
const signal = init?.signal as AbortSignal
|
|
signals.push(signal)
|
|
signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true })
|
|
}))
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const external = new AbortController()
|
|
const externalRequest = getLocalLLMService().generate('first', { signal: external.signal })
|
|
const externalExpectation = expect(externalRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
|
|
external.abort()
|
|
await externalExpectation
|
|
expect(signals[0].aborted).toBe(true)
|
|
|
|
vi.useFakeTimers()
|
|
const timeoutRequest = getLocalLLMService().generate('second', { timeoutMs: 25 })
|
|
const timeoutExpectation = expect(timeoutRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingTimeout })
|
|
await vi.advanceTimersByTimeAsync(25)
|
|
await timeoutExpectation
|
|
expect(signals[1].aborted).toBe(true)
|
|
})
|
|
|
|
it('keeps other active requests cancellable after one request completes', async () => {
|
|
const signals: AbortSignal[] = []
|
|
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => {
|
|
const signal = init?.signal as AbortSignal
|
|
signals.push(signal)
|
|
if (signals.length === 1) return Promise.resolve(completedGenerateResponse())
|
|
return new Promise<Response>((_resolve, reject) => {
|
|
signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true })
|
|
})
|
|
})
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const first = getLocalLLMService().generate('first')
|
|
const second = getLocalLLMService().generate('second')
|
|
const third = getLocalLLMService().generate('third')
|
|
await expect(first).resolves.toMatchObject({ text: 'ok' })
|
|
|
|
const secondExpectation = expect(second).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
|
|
const thirdExpectation = expect(third).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
|
|
getLocalLLMService().cancelGeneration()
|
|
|
|
await secondExpectation
|
|
await thirdExpectation
|
|
expect(signals[1].aborted).toBe(true)
|
|
expect(signals[2].aborted).toBe(true)
|
|
})
|
|
|
|
it('rejects a stream that reaches EOF without a done frame', async () => {
|
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(completedStream([
|
|
JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }),
|
|
]), { status: 200 })))
|
|
|
|
const stream = getLocalLLMService().streamGenerate('hello')
|
|
await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false })
|
|
await expect(stream.next()).rejects.toMatchObject({ code: ErrorCode.LLMProcessingFailed })
|
|
})
|
|
|
|
it('finishes on a done frame without waiting for the server to close the stream', async () => {
|
|
const cancel = vi.fn()
|
|
const body = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(encoder.encode([
|
|
JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }),
|
|
JSON.stringify({ model: 'gemma4:e4b', response: '', done: true }),
|
|
].join('\n') + '\n'))
|
|
},
|
|
cancel,
|
|
})
|
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(body, { status: 200 })))
|
|
|
|
const stream = getLocalLLMService().streamGenerate('hello', { timeoutMs: 500 })
|
|
await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false })
|
|
await expect(stream.next()).resolves.toMatchObject({ value: expect.objectContaining({ text: 'partial' }), done: true })
|
|
expect(cancel).toHaveBeenCalledTimes(1)
|
|
})
|
|
|
|
it('deduplicates concurrent ensureRunning calls and polling startup', async () => {
|
|
const service = getLocalLLMService()
|
|
const privateService = service as unknown as {
|
|
_ensureRunning: () => Promise<'running' | 'starting' | 'not-installed' | 'failed'>
|
|
_checkAvailability: () => Promise<void>
|
|
}
|
|
let release: ((value: 'running') => void) | undefined
|
|
const pending = new Promise<'running'>((resolve) => {
|
|
release = resolve
|
|
})
|
|
const ensure = vi.spyOn(privateService, '_ensureRunning').mockReturnValue(pending)
|
|
|
|
const first = service.ensureRunning()
|
|
const second = service.ensureRunning()
|
|
expect(ensure).toHaveBeenCalledTimes(1)
|
|
release?.('running')
|
|
await expect(Promise.all([first, second])).resolves.toEqual(['running', 'running'])
|
|
|
|
vi.useFakeTimers()
|
|
const availability = vi.spyOn(privateService, '_checkAvailability').mockResolvedValue()
|
|
service.startPolling()
|
|
service.startPolling()
|
|
expect(availability).toHaveBeenCalledTimes(1)
|
|
await vi.advanceTimersByTimeAsync(5000)
|
|
expect(availability).toHaveBeenCalledTimes(2)
|
|
service.stopPolling()
|
|
})
|
|
})
|
|
|
|
describe('LocalLLMService model residency (2026-10-11 memory policy)', () => {
|
|
beforeEach(() => {
|
|
initInMemoryConfig()
|
|
resetLocalLLMServiceForTests()
|
|
markAvailable()
|
|
})
|
|
|
|
afterEach(() => {
|
|
resetLocalLLMServiceForTests()
|
|
resetInMemoryConfig()
|
|
vi.unstubAllGlobals()
|
|
})
|
|
|
|
it('applies the shared 10m keep_alive when a caller does not set one', async () => {
|
|
const bodies: Array<{ keep_alive?: string | number }> = []
|
|
vi.stubGlobal('fetch', vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
|
|
bodies.push(JSON.parse(String(init?.body)) as { keep_alive?: string })
|
|
return completedGenerateResponse()
|
|
}))
|
|
|
|
await getLocalLLMService().generate('hi', { model: 'gemma4:e4b' })
|
|
|
|
expect(bodies[0].keep_alive).toBe('10m')
|
|
})
|
|
|
|
it('unloads only the models this app loaded, once, with keep_alive 0', async () => {
|
|
const bodies: Array<{ model: string; keep_alive?: string | number; prompt?: string }> = []
|
|
vi.stubGlobal('fetch', vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
|
|
bodies.push(JSON.parse(String(init?.body)) as { model: string; keep_alive?: string })
|
|
return completedGenerateResponse()
|
|
}))
|
|
const service = getLocalLLMService()
|
|
|
|
await service.unloadResidentModels()
|
|
expect(bodies).toHaveLength(0)
|
|
|
|
await service.generate('hi', { model: 'gemma4:e4b' })
|
|
await service.unloadResidentModels()
|
|
await service.unloadResidentModels()
|
|
|
|
const unloads = bodies.filter((body) => body.keep_alive === 0)
|
|
expect(unloads).toEqual([{ model: 'gemma4:e4b', keep_alive: 0 }])
|
|
})
|
|
})
|
|
|
|
describe('LocalLLMService cancellation and dispose safety', () => {
|
|
beforeEach(() => {
|
|
initInMemoryConfig()
|
|
resetLocalLLMServiceForTests()
|
|
markAvailable()
|
|
})
|
|
|
|
afterEach(() => {
|
|
resetLocalLLMServiceForTests()
|
|
resetInMemoryConfig()
|
|
vi.unstubAllGlobals()
|
|
})
|
|
|
|
it('processText_signalAbortedMidGenerate_rejectsWithCancelled', async () => {
|
|
let seenSignal: AbortSignal | undefined
|
|
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => {
|
|
seenSignal = init?.signal ?? undefined
|
|
return new Promise<Response>((_resolve, reject) => {
|
|
const timer = setTimeout(() => reject(new Error('fetch was not aborted')), 300)
|
|
init?.signal?.addEventListener(
|
|
'abort',
|
|
() => {
|
|
clearTimeout(timer)
|
|
reject(new Error('aborted'))
|
|
},
|
|
{ once: true },
|
|
)
|
|
})
|
|
})
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const controller = new AbortController()
|
|
const promise = (
|
|
getLocalLLMService() as unknown as {
|
|
processText: (
|
|
text: string,
|
|
action: 'refine',
|
|
targetLanguage?: string,
|
|
customPrompt?: string,
|
|
options?: { signal?: AbortSignal },
|
|
) => Promise<string>
|
|
}
|
|
).processText('안녕', 'refine', undefined, undefined, { signal: controller.signal })
|
|
|
|
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1))
|
|
controller.abort()
|
|
|
|
await expect(promise).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
|
|
expect(seenSignal?.aborted).toBe(true)
|
|
})
|
|
|
|
it('pullModel_disposedMidStream_abortsFetchCancelsReaderAndMarksUnavailable', async () => {
|
|
let safetyTimer: ReturnType<typeof setTimeout> | null = null
|
|
const cancelReader = vi.fn(async () => {
|
|
if (safetyTimer) clearTimeout(safetyTimer)
|
|
})
|
|
let seenSignal: AbortSignal | undefined
|
|
let pullFetchAborted = false
|
|
|
|
const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
|
|
seenSignal = init?.signal ?? undefined
|
|
init?.signal?.addEventListener('abort', () => {
|
|
pullFetchAborted = true
|
|
})
|
|
const stream = new ReadableStream<Uint8Array>({
|
|
pull() {
|
|
return new Promise<void>((_resolve, reject) => {
|
|
safetyTimer = setTimeout(() => reject(new Error('pull was not cancelled')), 300)
|
|
})
|
|
},
|
|
cancel: cancelReader,
|
|
})
|
|
return new Response(stream, { status: 200 })
|
|
})
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const service = getLocalLLMService()
|
|
const pullPromise = service.pullModel('gemma4:e4b')
|
|
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1))
|
|
await new Promise((r) => setTimeout(r, 0))
|
|
|
|
service.dispose()
|
|
|
|
await expect(pullPromise).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled })
|
|
expect(pullFetchAborted || Boolean(seenSignal?.aborted)).toBe(true)
|
|
expect(cancelReader).toHaveBeenCalledTimes(1)
|
|
expect(service.isAvailable()).toBe(false)
|
|
})
|
|
|
|
it('checkConnection_disposedWhilePending_doesNotRestoreAvailability', async () => {
|
|
const service = getLocalLLMService()
|
|
;(service as unknown as { _available: boolean })._available = false
|
|
|
|
let resolveVersion!: (value: Response) => void
|
|
vi.stubGlobal(
|
|
'fetch',
|
|
vi.fn((input: RequestInfo | URL) => {
|
|
if (String(input).endsWith('/api/tags')) {
|
|
return Promise.resolve(new Response(JSON.stringify({ models: [] }), { status: 200 }))
|
|
}
|
|
return new Promise<Response>((resolve) => {
|
|
resolveVersion = resolve
|
|
})
|
|
}),
|
|
)
|
|
|
|
const checkPromise = service.checkConnection()
|
|
await new Promise((r) => setTimeout(r, 0))
|
|
|
|
service.dispose()
|
|
resolveVersion(new Response(JSON.stringify({ version: '0.20.0' }), { status: 200 }))
|
|
const status = await checkPromise
|
|
|
|
expect(status.available).toBe(false)
|
|
expect(service.isAvailable()).toBe(false)
|
|
})
|
|
|
|
it('pullModel_disposedBeforeHeadersArrive_rejectsWithLLMProcessingCancelled', async () => {
|
|
let seenSignal: AbortSignal | undefined
|
|
const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => {
|
|
seenSignal = init?.signal ?? undefined
|
|
return new Promise<Response>((_resolve, reject) => {
|
|
const timer = setTimeout(() => reject(new Error('pull fetch was not aborted')), 300)
|
|
init?.signal?.addEventListener(
|
|
'abort',
|
|
() => {
|
|
clearTimeout(timer)
|
|
reject(new DOMException('The operation was aborted.', 'AbortError'))
|
|
},
|
|
{ once: true },
|
|
)
|
|
})
|
|
})
|
|
vi.stubGlobal('fetch', fetchMock)
|
|
|
|
const service = getLocalLLMService()
|
|
const pullPromise = service.pullModel('gemma4:e4b')
|
|
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1))
|
|
|
|
service.dispose()
|
|
|
|
await expect(pullPromise).rejects.toSatisfy(
|
|
(err: unknown) => err instanceof D3ROError && err.code === ErrorCode.LLMProcessingCancelled,
|
|
)
|
|
expect(seenSignal?.aborted).toBe(true)
|
|
expect(service.isAvailable()).toBe(false)
|
|
})
|
|
})
|