377 lines
14 KiB
TypeScript
377 lines
14 KiB
TypeScript
// tests/main/services/stt-sidecar-redteam-r2-6.test.ts
|
|
// 1) 클라우드 STT 실패 → 로컬 폴백이 무음/미설치 상황에서 엔진·모델 내려받기를 조용히 시작하던 버그
|
|
// 2) 기동 중(헬스 전) 사이드카를 '실행 중'으로 보고 통과시키던 버그 (SidecarSupervisor.ready)
|
|
// 3) 헬스 성공마다 재시작 카운트가 0으로 돌아가 로딩 중 반복 크래시가 끝나지 않던 버그
|
|
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
import { EventEmitter } from 'events'
|
|
import { mkdirSync, rmSync, writeFileSync } from 'fs'
|
|
import { join } from 'path'
|
|
import type { ChildProcess } from 'child_process'
|
|
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
|
|
|
const launchMock = vi.hoisted(() => ({ offline: true }))
|
|
|
|
vi.mock('../../../src/main/services/stt/sidecarLaunch', () => ({
|
|
resolveSidecarLaunch: vi.fn(async () => ({ command: 'fake-sidecar', args: [], source: 'venv' as const })),
|
|
isSidecarLaunchAvailableOffline: vi.fn(() => launchMock.offline),
|
|
}))
|
|
|
|
import { getSTTManager, resetSTTManagerForTests } from '../../../src/main/services/stt/STTManager'
|
|
import {
|
|
LocalSTTService,
|
|
getLocalSTTService,
|
|
resetLocalSTTServiceForTests,
|
|
} from '../../../src/main/services/LocalSTTService'
|
|
import type { TranscribeOptions, TranscriptionResult } from '../../../src/main/services/LocalSTTService'
|
|
import { SidecarSupervisor, type SidecarFetchFn } from '../../../src/main/services/stt/SidecarSupervisor'
|
|
import { isRecoverableByLocalFallback } from '../../../src/main/services/stt/fallbackPolicy'
|
|
import { initInMemoryConfig, resetInMemoryConfig, configSet } from '../../../src/main/services/ConfigService'
|
|
import { getWhisperModelsDir } from '../../../src/main/utils/paths'
|
|
|
|
// ── 공용 헬퍼 ────────────────────────────────────────────
|
|
|
|
type LocalInternals = {
|
|
_modelReady: boolean
|
|
_sendToSidecar: (buf: Buffer, options?: TranscribeOptions) => Promise<TranscriptionResult>
|
|
}
|
|
|
|
const okResult = (text: string): TranscriptionResult => ({
|
|
text,
|
|
segments: [],
|
|
language: 'ko',
|
|
duration: 1,
|
|
processingTime: 1,
|
|
})
|
|
|
|
async function flush(times = 5): Promise<void> {
|
|
for (let i = 0; i < times; i++) {
|
|
await new Promise<void>((resolve) => setImmediate(resolve))
|
|
}
|
|
}
|
|
|
|
class FakeChild extends EventEmitter {
|
|
exitCode: number | null = null
|
|
signalCode: NodeJS.Signals | null = null
|
|
stdout = null
|
|
stderr = null
|
|
kill = vi.fn((): boolean => {
|
|
this.die(null, 'SIGKILL')
|
|
return true
|
|
})
|
|
|
|
die(code: number | null, signal: NodeJS.Signals | null = null): void {
|
|
if (this.exitCode !== null || this.signalCode !== null) return
|
|
this.exitCode = code
|
|
this.signalCode = signal
|
|
this.emit('exit', code, signal)
|
|
}
|
|
}
|
|
|
|
interface Harness {
|
|
supervisor: SidecarSupervisor
|
|
fetch: SidecarFetchFn
|
|
children: FakeChild[]
|
|
spawn: ReturnType<typeof vi.fn>
|
|
fetchCalls: string[]
|
|
setHealthy: (value: boolean) => void
|
|
onLoad: (handler: () => Promise<Response>) => void
|
|
}
|
|
|
|
function jsonResponse(body: unknown, status = 200): Response {
|
|
return new Response(JSON.stringify(body), {
|
|
status,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
})
|
|
}
|
|
|
|
function createHarness(): Harness {
|
|
const children: FakeChild[] = []
|
|
const fetchCalls: string[] = []
|
|
let healthy = false
|
|
let clock = 0
|
|
let loadHandler: () => Promise<Response> = async () =>
|
|
jsonResponse({ status: 'ok', model_id: 'tiny', load_time_ms: 1 })
|
|
|
|
const spawn = vi.fn(() => {
|
|
const child = new FakeChild()
|
|
children.push(child)
|
|
setImmediate(() => child.emit('spawn'))
|
|
return child as unknown as ChildProcess
|
|
})
|
|
|
|
const fetchFn: SidecarFetchFn = async (input) => {
|
|
fetchCalls.push(input)
|
|
if (input.endsWith('/health')) {
|
|
if (!healthy) throw new TypeError('fetch failed (ECONNREFUSED)')
|
|
return jsonResponse({ status: 'ok', model: null, gpu: false })
|
|
}
|
|
if (input.endsWith('/load')) return loadHandler()
|
|
if (input.endsWith('/shutdown')) return jsonResponse({ ok: true })
|
|
throw new TypeError(`unexpected ${input}`)
|
|
}
|
|
|
|
const supervisor = new SidecarSupervisor({
|
|
resolveLaunch: async () => ({ command: 'fake-sidecar', args: [], source: 'venv' }),
|
|
modelsDir: () => 'models',
|
|
baseUrlFor: (port) => `http://127.0.0.1:${port}`,
|
|
spawn,
|
|
fetch: fetchFn,
|
|
findFreePort: async (start) => start,
|
|
now: () => clock,
|
|
sleep: async (ms) => {
|
|
clock += ms
|
|
await new Promise<void>((resolve) => setImmediate(resolve))
|
|
},
|
|
})
|
|
|
|
return {
|
|
supervisor,
|
|
fetch: fetchFn,
|
|
children,
|
|
spawn,
|
|
fetchCalls,
|
|
setHealthy: (value) => {
|
|
healthy = value
|
|
},
|
|
onLoad: (handler) => {
|
|
loadHandler = handler
|
|
},
|
|
}
|
|
}
|
|
|
|
// ── 1. 클라우드 실패 → 로컬 폴백 ─────────────────────────
|
|
|
|
describe('STTManager — 로컬 폴백은 복구 가능한 실패 + 설치된 로컬 엔진에서만', () => {
|
|
const MODEL_ID = 'r2-6-installed-model'
|
|
|
|
beforeEach(() => {
|
|
initInMemoryConfig()
|
|
resetSTTManagerForTests()
|
|
resetLocalSTTServiceForTests()
|
|
vi.restoreAllMocks()
|
|
launchMock.offline = true
|
|
})
|
|
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals()
|
|
rmSync(join(getWhisperModelsDir(), MODEL_ID), { recursive: true, force: true })
|
|
resetInMemoryConfig()
|
|
resetSTTManagerForTests()
|
|
resetLocalSTTServiceForTests()
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
function useOpenAI(fetchResponse: unknown): void {
|
|
const mgr = getSTTManager()
|
|
mgr.setProvider('openai')
|
|
mgr.setProviderConfig('openai', { apiKey: 'sk-test' })
|
|
configSet('sttFallbackToLocal', true)
|
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fetchResponse))
|
|
}
|
|
|
|
function installModel(): void {
|
|
const dir = join(getWhisperModelsDir(), MODEL_ID)
|
|
mkdirSync(dir, { recursive: true })
|
|
writeFileSync(join(dir, 'model.bin'), 'x')
|
|
configSet('sttModelId', MODEL_ID)
|
|
}
|
|
|
|
it('무음(STTNoAudioData)은 로컬로 넘기지 않고 그대로 던진다', async () => {
|
|
useOpenAI({ ok: true, status: 200, json: async () => ({ text: ' ' }) })
|
|
const localTranscribe = vi.spyOn(getLocalSTTService(), 'transcribe')
|
|
|
|
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
|
code: ErrorCode.STTNoAudioData,
|
|
})
|
|
expect(localTranscribe).not.toHaveBeenCalled()
|
|
})
|
|
|
|
it('로컬 모델이 설치되지 않았으면 초기화(=내려받기)를 시작하지 않고 원래 클라우드 에러로 끝난다', async () => {
|
|
useOpenAI({ ok: false, status: 503, text: async () => 'offline' })
|
|
configSet('sttModelId', 'large-v3-turbo')
|
|
const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined)
|
|
|
|
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
|
code: ErrorCode.STTTranscriptionFailed,
|
|
})
|
|
expect(init).not.toHaveBeenCalled()
|
|
})
|
|
|
|
it('엔진이 설치되지 않았으면(런타임 내려받기 필요) 초기화하지 않는다', async () => {
|
|
useOpenAI({ ok: false, status: 503, text: async () => 'offline' })
|
|
installModel()
|
|
launchMock.offline = false
|
|
const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined)
|
|
|
|
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
|
code: ErrorCode.STTTranscriptionFailed,
|
|
})
|
|
expect(init).not.toHaveBeenCalled()
|
|
})
|
|
|
|
it('모델과 엔진이 로컬에 있으면 폴백해 로컬 결과를 돌려준다', async () => {
|
|
useOpenAI({ ok: false, status: 401, text: async () => 'Unauthorized' })
|
|
installModel()
|
|
const local = getLocalSTTService()
|
|
const internals = local as unknown as LocalInternals
|
|
const init = vi.spyOn(local, 'initialize').mockImplementation(async () => {
|
|
internals._modelReady = true
|
|
})
|
|
vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('로컬 결과'))
|
|
|
|
const result = await getSTTManager().transcribe(Buffer.alloc(32000), { language: 'ko' })
|
|
expect(result.text).toBe('로컬 결과')
|
|
expect(init).toHaveBeenCalledTimes(1)
|
|
})
|
|
|
|
it('폴백 정책: 무음/너무 짧음/취소는 복구 불가, 그 밖(인증·서버·네트워크)은 복구 가능', () => {
|
|
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTNoAudioData, 'x'))).toBe(false)
|
|
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTAudioTooShort, 'x'))).toBe(false)
|
|
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionCancelled, 'x'))).toBe(false)
|
|
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionFailed, '401'))).toBe(true)
|
|
expect(isRecoverableByLocalFallback(new Error('network'))).toBe(true)
|
|
})
|
|
})
|
|
|
|
// ── 2. SidecarSupervisor — 기동 중 호출자는 헬스까지 기다린다 ──
|
|
|
|
describe('SidecarSupervisor', () => {
|
|
it('기동 중(헬스 전)에 들어온 두 번째 호출자는 즉시 통과하지 않고 헬스까지 기다린다', async () => {
|
|
const h = createHarness()
|
|
const first = h.supervisor.ready()
|
|
await flush()
|
|
expect(h.spawn).toHaveBeenCalledTimes(1)
|
|
expect(h.supervisor.state).toBe('starting')
|
|
|
|
let secondSettled = false
|
|
const second = h.supervisor.ready().then((url) => {
|
|
secondSettled = true
|
|
return url
|
|
})
|
|
await flush(10)
|
|
expect(secondSettled).toBe(false)
|
|
expect(h.supervisor.isHealthy).toBe(false)
|
|
|
|
h.setHealthy(true)
|
|
const [a, b] = await Promise.all([first, second])
|
|
expect(a).toBe(b)
|
|
expect(h.supervisor.isHealthy).toBe(true)
|
|
expect(h.spawn).toHaveBeenCalledTimes(1)
|
|
})
|
|
|
|
it('크래시 백오프 중 ready()는 새로 spawn하지 않고 예약된 재시작에 합류한다', async () => {
|
|
const h = createHarness()
|
|
h.setHealthy(true)
|
|
await h.supervisor.ready()
|
|
const crashed = vi.fn()
|
|
h.supervisor.on('crashed', crashed)
|
|
|
|
h.children[0].die(1)
|
|
expect(h.supervisor.state).toBe('crashed')
|
|
expect(crashed).toHaveBeenCalledWith(expect.objectContaining({ attempt: 1 }))
|
|
|
|
const [a, b] = await Promise.all([h.supervisor.ready(), h.supervisor.ready()])
|
|
expect(a).toBe(b)
|
|
expect(h.spawn).toHaveBeenCalledTimes(2)
|
|
expect(h.supervisor.isHealthy).toBe(true)
|
|
})
|
|
|
|
it('markStable 후에는 재시작 카운트가 초기화된다', async () => {
|
|
const h = createHarness()
|
|
h.setHealthy(true)
|
|
await h.supervisor.ready()
|
|
h.children[0].die(1)
|
|
await h.supervisor.ready()
|
|
expect(h.supervisor.restartCount).toBe(1)
|
|
h.supervisor.markStable()
|
|
expect(h.supervisor.restartCount).toBe(0)
|
|
})
|
|
|
|
it('dispose 뒤 예약된 재시작은 취소되고 ready()는 거부된다', async () => {
|
|
const h = createHarness()
|
|
h.setHealthy(true)
|
|
await h.supervisor.ready()
|
|
h.children[0].die(1)
|
|
const pending = h.supervisor.ready()
|
|
await h.supervisor.dispose()
|
|
await expect(pending).rejects.toBeInstanceOf(D3ROError)
|
|
await expect(h.supervisor.ready()).rejects.toBeInstanceOf(D3ROError)
|
|
expect(h.spawn).toHaveBeenCalledTimes(1)
|
|
})
|
|
})
|
|
|
|
// ── 3. LocalSTTService + 감독자 — 크래시 재시작 정책 ──────
|
|
|
|
describe('LocalSTTService — 사이드카 크래시 재시작', () => {
|
|
beforeEach(() => {
|
|
initInMemoryConfig()
|
|
})
|
|
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals()
|
|
resetInMemoryConfig()
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
it('모델 로딩 중 반복 크래시는 MAX 재시작에서 멈추고 STTSidecarCrashed를 알린다', async () => {
|
|
const h = createHarness()
|
|
vi.stubGlobal('fetch', h.fetch)
|
|
h.setHealthy(true)
|
|
// /load 도중 사이드카가 죽는다 (OOM/CUDA 실패 등)
|
|
h.onLoad(async () => {
|
|
h.children[h.children.length - 1].die(1)
|
|
throw new TypeError('fetch failed')
|
|
})
|
|
const local = new LocalSTTService({ supervisor: h.supervisor })
|
|
const errors: D3ROError[] = []
|
|
local.on('error', ({ error }) => errors.push(error))
|
|
|
|
await expect(local.initialize('tiny')).rejects.toBeInstanceOf(D3ROError)
|
|
for (let i = 0; i < 200 && h.supervisor.state !== 'stopped'; i++) {
|
|
await flush()
|
|
}
|
|
|
|
expect(h.supervisor.state).toBe('stopped')
|
|
// 최초 1회 + 재시작 3회 — 헬스 성공이 카운트를 되돌리지 않는다
|
|
expect(h.spawn).toHaveBeenCalledTimes(4)
|
|
expect(errors.some((e) => e.code === ErrorCode.STTSidecarCrashed)).toBe(true)
|
|
expect(local.getStatus().engineState).toBe('error')
|
|
await local.dispose()
|
|
})
|
|
|
|
it('모델이 없던(UIA만 쓰던) 사이드카가 죽으면 되살리기만 하고 모델은 올리지 않는다', async () => {
|
|
const h = createHarness()
|
|
vi.stubGlobal('fetch', h.fetch)
|
|
h.setHealthy(true)
|
|
const local = new LocalSTTService({ supervisor: h.supervisor })
|
|
|
|
await local.ensureSidecar()
|
|
h.children[0].die(1)
|
|
await local.ensureSidecar()
|
|
await flush()
|
|
|
|
expect(h.spawn).toHaveBeenCalledTimes(2)
|
|
expect(h.fetchCalls.some((url) => url.endsWith('/load'))).toBe(false)
|
|
await local.dispose()
|
|
})
|
|
|
|
it('크래시 재시작 대기 중 들어온 transcribe는 재시작(모델 재로딩) 결과를 기다려 전사한다', async () => {
|
|
const h = createHarness()
|
|
vi.stubGlobal('fetch', h.fetch)
|
|
h.setHealthy(true)
|
|
const local = new LocalSTTService({ supervisor: h.supervisor })
|
|
const internals = local as unknown as LocalInternals
|
|
await local.initialize('tiny')
|
|
|
|
h.children[0].die(1)
|
|
const send = vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('재시작 후 전사'))
|
|
const result = await local.transcribe(Buffer.alloc(32000))
|
|
|
|
expect(result.text).toBe('재시작 후 전사')
|
|
expect(send).toHaveBeenCalledTimes(1)
|
|
expect(h.fetchCalls.filter((url) => url.endsWith('/load'))).toHaveLength(2)
|
|
expect(local.currentModelId).toBe('tiny')
|
|
await local.dispose()
|
|
})
|
|
})
|