// tests/main/services/stt-sidecar-redteam-r2-6.test.ts // 1) 클라우드 STT 실패 → 로컬 폴백이 무음/미설치 상황에서 엔진·모델 내려받기를 조용히 시작하던 버그 // 2) 기동 중(헬스 전) 사이드카를 '실행 중'으로 보고 통과시키던 버그 (SidecarSupervisor.ready) // 3) 헬스 성공마다 재시작 카운트가 0으로 돌아가 로딩 중 반복 크래시가 끝나지 않던 버그 import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { EventEmitter } from 'events' import { mkdirSync, rmSync, writeFileSync } from 'fs' import { join } from 'path' import type { ChildProcess } from 'child_process' import { D3ROError, ErrorCode } from '@d3ro/core/errors' const launchMock = vi.hoisted(() => ({ offline: true })) vi.mock('../../../src/main/services/stt/sidecarLaunch', () => ({ resolveSidecarLaunch: vi.fn(async () => ({ command: 'fake-sidecar', args: [], source: 'venv' as const })), isSidecarLaunchAvailableOffline: vi.fn(() => launchMock.offline), })) import { getSTTManager, resetSTTManagerForTests } from '../../../src/main/services/stt/STTManager' import { LocalSTTService, getLocalSTTService, resetLocalSTTServiceForTests, } from '../../../src/main/services/LocalSTTService' import type { TranscribeOptions, TranscriptionResult } from '../../../src/main/services/LocalSTTService' import { SidecarSupervisor, type SidecarFetchFn } from '../../../src/main/services/stt/SidecarSupervisor' import { isRecoverableByLocalFallback } from '../../../src/main/services/stt/fallbackPolicy' import { initInMemoryConfig, resetInMemoryConfig, configSet } from '../../../src/main/services/ConfigService' import { getWhisperModelsDir } from '../../../src/main/utils/paths' // ── 공용 헬퍼 ──────────────────────────────────────────── type LocalInternals = { _modelReady: boolean _sendToSidecar: (buf: Buffer, options?: TranscribeOptions) => Promise } const okResult = (text: string): TranscriptionResult => ({ text, segments: [], language: 'ko', duration: 1, processingTime: 1, }) async function flush(times = 5): Promise { for (let i = 0; i < times; i++) { await new Promise((resolve) => setImmediate(resolve)) } } class FakeChild extends EventEmitter { exitCode: number | null = null signalCode: NodeJS.Signals | null = null stdout = null stderr = null kill = vi.fn((): boolean => { this.die(null, 'SIGKILL') return true }) die(code: number | null, signal: NodeJS.Signals | null = null): void { if (this.exitCode !== null || this.signalCode !== null) return this.exitCode = code this.signalCode = signal this.emit('exit', code, signal) } } interface Harness { supervisor: SidecarSupervisor fetch: SidecarFetchFn children: FakeChild[] spawn: ReturnType fetchCalls: string[] setHealthy: (value: boolean) => void onLoad: (handler: () => Promise) => void } function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' }, }) } function createHarness(): Harness { const children: FakeChild[] = [] const fetchCalls: string[] = [] let healthy = false let clock = 0 let loadHandler: () => Promise = async () => jsonResponse({ status: 'ok', model_id: 'tiny', load_time_ms: 1 }) const spawn = vi.fn(() => { const child = new FakeChild() children.push(child) setImmediate(() => child.emit('spawn')) return child as unknown as ChildProcess }) const fetchFn: SidecarFetchFn = async (input) => { fetchCalls.push(input) if (input.endsWith('/health')) { if (!healthy) throw new TypeError('fetch failed (ECONNREFUSED)') return jsonResponse({ status: 'ok', model: null, gpu: false }) } if (input.endsWith('/load')) return loadHandler() if (input.endsWith('/shutdown')) return jsonResponse({ ok: true }) throw new TypeError(`unexpected ${input}`) } const supervisor = new SidecarSupervisor({ resolveLaunch: async () => ({ command: 'fake-sidecar', args: [], source: 'venv' }), modelsDir: () => 'models', baseUrlFor: (port) => `http://127.0.0.1:${port}`, spawn, fetch: fetchFn, findFreePort: async (start) => start, now: () => clock, sleep: async (ms) => { clock += ms await new Promise((resolve) => setImmediate(resolve)) }, }) return { supervisor, fetch: fetchFn, children, spawn, fetchCalls, setHealthy: (value) => { healthy = value }, onLoad: (handler) => { loadHandler = handler }, } } // ── 1. 클라우드 실패 → 로컬 폴백 ───────────────────────── describe('STTManager — 로컬 폴백은 복구 가능한 실패 + 설치된 로컬 엔진에서만', () => { const MODEL_ID = 'r2-6-installed-model' beforeEach(() => { initInMemoryConfig() resetSTTManagerForTests() resetLocalSTTServiceForTests() vi.restoreAllMocks() launchMock.offline = true }) afterEach(() => { vi.unstubAllGlobals() rmSync(join(getWhisperModelsDir(), MODEL_ID), { recursive: true, force: true }) resetInMemoryConfig() resetSTTManagerForTests() resetLocalSTTServiceForTests() vi.restoreAllMocks() }) function useOpenAI(fetchResponse: unknown): void { const mgr = getSTTManager() mgr.setProvider('openai') mgr.setProviderConfig('openai', { apiKey: 'sk-test' }) configSet('sttFallbackToLocal', true) vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fetchResponse)) } function installModel(): void { const dir = join(getWhisperModelsDir(), MODEL_ID) mkdirSync(dir, { recursive: true }) writeFileSync(join(dir, 'model.bin'), 'x') configSet('sttModelId', MODEL_ID) } it('무음(STTNoAudioData)은 로컬로 넘기지 않고 그대로 던진다', async () => { useOpenAI({ ok: true, status: 200, json: async () => ({ text: ' ' }) }) const localTranscribe = vi.spyOn(getLocalSTTService(), 'transcribe') await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({ code: ErrorCode.STTNoAudioData, }) expect(localTranscribe).not.toHaveBeenCalled() }) it('로컬 모델이 설치되지 않았으면 초기화(=내려받기)를 시작하지 않고 원래 클라우드 에러로 끝난다', async () => { useOpenAI({ ok: false, status: 503, text: async () => 'offline' }) configSet('sttModelId', 'large-v3-turbo') const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined) await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({ code: ErrorCode.STTTranscriptionFailed, }) expect(init).not.toHaveBeenCalled() }) it('엔진이 설치되지 않았으면(런타임 내려받기 필요) 초기화하지 않는다', async () => { useOpenAI({ ok: false, status: 503, text: async () => 'offline' }) installModel() launchMock.offline = false const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined) await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({ code: ErrorCode.STTTranscriptionFailed, }) expect(init).not.toHaveBeenCalled() }) it('모델과 엔진이 로컬에 있으면 폴백해 로컬 결과를 돌려준다', async () => { useOpenAI({ ok: false, status: 401, text: async () => 'Unauthorized' }) installModel() const local = getLocalSTTService() const internals = local as unknown as LocalInternals const init = vi.spyOn(local, 'initialize').mockImplementation(async () => { internals._modelReady = true }) vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('로컬 결과')) const result = await getSTTManager().transcribe(Buffer.alloc(32000), { language: 'ko' }) expect(result.text).toBe('로컬 결과') expect(init).toHaveBeenCalledTimes(1) }) it('폴백 정책: 무음/너무 짧음/취소는 복구 불가, 그 밖(인증·서버·네트워크)은 복구 가능', () => { expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTNoAudioData, 'x'))).toBe(false) expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTAudioTooShort, 'x'))).toBe(false) expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionCancelled, 'x'))).toBe(false) expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionFailed, '401'))).toBe(true) expect(isRecoverableByLocalFallback(new Error('network'))).toBe(true) }) }) // ── 2. SidecarSupervisor — 기동 중 호출자는 헬스까지 기다린다 ── describe('SidecarSupervisor', () => { it('기동 중(헬스 전)에 들어온 두 번째 호출자는 즉시 통과하지 않고 헬스까지 기다린다', async () => { const h = createHarness() const first = h.supervisor.ready() await flush() expect(h.spawn).toHaveBeenCalledTimes(1) expect(h.supervisor.state).toBe('starting') let secondSettled = false const second = h.supervisor.ready().then((url) => { secondSettled = true return url }) await flush(10) expect(secondSettled).toBe(false) expect(h.supervisor.isHealthy).toBe(false) h.setHealthy(true) const [a, b] = await Promise.all([first, second]) expect(a).toBe(b) expect(h.supervisor.isHealthy).toBe(true) expect(h.spawn).toHaveBeenCalledTimes(1) }) it('크래시 백오프 중 ready()는 새로 spawn하지 않고 예약된 재시작에 합류한다', async () => { const h = createHarness() h.setHealthy(true) await h.supervisor.ready() const crashed = vi.fn() h.supervisor.on('crashed', crashed) h.children[0].die(1) expect(h.supervisor.state).toBe('crashed') expect(crashed).toHaveBeenCalledWith(expect.objectContaining({ attempt: 1 })) const [a, b] = await Promise.all([h.supervisor.ready(), h.supervisor.ready()]) expect(a).toBe(b) expect(h.spawn).toHaveBeenCalledTimes(2) expect(h.supervisor.isHealthy).toBe(true) }) it('markStable 후에는 재시작 카운트가 초기화된다', async () => { const h = createHarness() h.setHealthy(true) await h.supervisor.ready() h.children[0].die(1) await h.supervisor.ready() expect(h.supervisor.restartCount).toBe(1) h.supervisor.markStable() expect(h.supervisor.restartCount).toBe(0) }) it('dispose 뒤 예약된 재시작은 취소되고 ready()는 거부된다', async () => { const h = createHarness() h.setHealthy(true) await h.supervisor.ready() h.children[0].die(1) const pending = h.supervisor.ready() await h.supervisor.dispose() await expect(pending).rejects.toBeInstanceOf(D3ROError) await expect(h.supervisor.ready()).rejects.toBeInstanceOf(D3ROError) expect(h.spawn).toHaveBeenCalledTimes(1) }) }) // ── 3. LocalSTTService + 감독자 — 크래시 재시작 정책 ────── describe('LocalSTTService — 사이드카 크래시 재시작', () => { beforeEach(() => { initInMemoryConfig() }) afterEach(() => { vi.unstubAllGlobals() resetInMemoryConfig() vi.restoreAllMocks() }) it('모델 로딩 중 반복 크래시는 MAX 재시작에서 멈추고 STTSidecarCrashed를 알린다', async () => { const h = createHarness() vi.stubGlobal('fetch', h.fetch) h.setHealthy(true) // /load 도중 사이드카가 죽는다 (OOM/CUDA 실패 등) h.onLoad(async () => { h.children[h.children.length - 1].die(1) throw new TypeError('fetch failed') }) const local = new LocalSTTService({ supervisor: h.supervisor }) const errors: D3ROError[] = [] local.on('error', ({ error }) => errors.push(error)) await expect(local.initialize('tiny')).rejects.toBeInstanceOf(D3ROError) for (let i = 0; i < 200 && h.supervisor.state !== 'stopped'; i++) { await flush() } expect(h.supervisor.state).toBe('stopped') // 최초 1회 + 재시작 3회 — 헬스 성공이 카운트를 되돌리지 않는다 expect(h.spawn).toHaveBeenCalledTimes(4) expect(errors.some((e) => e.code === ErrorCode.STTSidecarCrashed)).toBe(true) expect(local.getStatus().engineState).toBe('error') await local.dispose() }) it('모델이 없던(UIA만 쓰던) 사이드카가 죽으면 되살리기만 하고 모델은 올리지 않는다', async () => { const h = createHarness() vi.stubGlobal('fetch', h.fetch) h.setHealthy(true) const local = new LocalSTTService({ supervisor: h.supervisor }) await local.ensureSidecar() h.children[0].die(1) await local.ensureSidecar() await flush() expect(h.spawn).toHaveBeenCalledTimes(2) expect(h.fetchCalls.some((url) => url.endsWith('/load'))).toBe(false) await local.dispose() }) it('크래시 재시작 대기 중 들어온 transcribe는 재시작(모델 재로딩) 결과를 기다려 전사한다', async () => { const h = createHarness() vi.stubGlobal('fetch', h.fetch) h.setHealthy(true) const local = new LocalSTTService({ supervisor: h.supervisor }) const internals = local as unknown as LocalInternals await local.initialize('tiny') h.children[0].die(1) const send = vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('재시작 후 전사')) const result = await local.transcribe(Buffer.alloc(32000)) expect(result.text).toBe('재시작 후 전사') expect(send).toHaveBeenCalledTimes(1) expect(h.fetchCalls.filter((url) => url.endsWith('/load'))).toHaveLength(2) expect(local.currentModelId).toBe('tiny') await local.dispose() }) })