fix(stt): gate local fallback on installed engine and extract SidecarSupervisor
This commit is contained in:
Yun Chan
parent
4588b65dfa
commit
2cd462333f
7 changed files with 1057 additions and 343 deletions
|
|
@ -0,0 +1,377 @@
|
|||
// tests/main/services/stt-sidecar-redteam-r2-6.test.ts
|
||||
// 1) 클라우드 STT 실패 → 로컬 폴백이 무음/미설치 상황에서 엔진·모델 내려받기를 조용히 시작하던 버그
|
||||
// 2) 기동 중(헬스 전) 사이드카를 '실행 중'으로 보고 통과시키던 버그 (SidecarSupervisor.ready)
|
||||
// 3) 헬스 성공마다 재시작 카운트가 0으로 돌아가 로딩 중 반복 크래시가 끝나지 않던 버그
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { EventEmitter } from 'events'
|
||||
import { mkdirSync, rmSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import type { ChildProcess } from 'child_process'
|
||||
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
||||
|
||||
const launchMock = vi.hoisted(() => ({ offline: true }))
|
||||
|
||||
vi.mock('../../../src/main/services/stt/sidecarLaunch', () => ({
|
||||
resolveSidecarLaunch: vi.fn(async () => ({ command: 'fake-sidecar', args: [], source: 'venv' as const })),
|
||||
isSidecarLaunchAvailableOffline: vi.fn(() => launchMock.offline),
|
||||
}))
|
||||
|
||||
import { getSTTManager, resetSTTManagerForTests } from '../../../src/main/services/stt/STTManager'
|
||||
import {
|
||||
LocalSTTService,
|
||||
getLocalSTTService,
|
||||
resetLocalSTTServiceForTests,
|
||||
} from '../../../src/main/services/LocalSTTService'
|
||||
import type { TranscribeOptions, TranscriptionResult } from '../../../src/main/services/LocalSTTService'
|
||||
import { SidecarSupervisor, type SidecarFetchFn } from '../../../src/main/services/stt/SidecarSupervisor'
|
||||
import { isRecoverableByLocalFallback } from '../../../src/main/services/stt/fallbackPolicy'
|
||||
import { initInMemoryConfig, resetInMemoryConfig, configSet } from '../../../src/main/services/ConfigService'
|
||||
import { getWhisperModelsDir } from '../../../src/main/utils/paths'
|
||||
|
||||
// ── 공용 헬퍼 ────────────────────────────────────────────
|
||||
|
||||
type LocalInternals = {
|
||||
_modelReady: boolean
|
||||
_sendToSidecar: (buf: Buffer, options?: TranscribeOptions) => Promise<TranscriptionResult>
|
||||
}
|
||||
|
||||
const okResult = (text: string): TranscriptionResult => ({
|
||||
text,
|
||||
segments: [],
|
||||
language: 'ko',
|
||||
duration: 1,
|
||||
processingTime: 1,
|
||||
})
|
||||
|
||||
async function flush(times = 5): Promise<void> {
|
||||
for (let i = 0; i < times; i++) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
}
|
||||
}
|
||||
|
||||
class FakeChild extends EventEmitter {
|
||||
exitCode: number | null = null
|
||||
signalCode: NodeJS.Signals | null = null
|
||||
stdout = null
|
||||
stderr = null
|
||||
kill = vi.fn((): boolean => {
|
||||
this.die(null, 'SIGKILL')
|
||||
return true
|
||||
})
|
||||
|
||||
die(code: number | null, signal: NodeJS.Signals | null = null): void {
|
||||
if (this.exitCode !== null || this.signalCode !== null) return
|
||||
this.exitCode = code
|
||||
this.signalCode = signal
|
||||
this.emit('exit', code, signal)
|
||||
}
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
supervisor: SidecarSupervisor
|
||||
fetch: SidecarFetchFn
|
||||
children: FakeChild[]
|
||||
spawn: ReturnType<typeof vi.fn>
|
||||
fetchCalls: string[]
|
||||
setHealthy: (value: boolean) => void
|
||||
onLoad: (handler: () => Promise<Response>) => void
|
||||
}
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
function createHarness(): Harness {
|
||||
const children: FakeChild[] = []
|
||||
const fetchCalls: string[] = []
|
||||
let healthy = false
|
||||
let clock = 0
|
||||
let loadHandler: () => Promise<Response> = async () =>
|
||||
jsonResponse({ status: 'ok', model_id: 'tiny', load_time_ms: 1 })
|
||||
|
||||
const spawn = vi.fn(() => {
|
||||
const child = new FakeChild()
|
||||
children.push(child)
|
||||
setImmediate(() => child.emit('spawn'))
|
||||
return child as unknown as ChildProcess
|
||||
})
|
||||
|
||||
const fetchFn: SidecarFetchFn = async (input) => {
|
||||
fetchCalls.push(input)
|
||||
if (input.endsWith('/health')) {
|
||||
if (!healthy) throw new TypeError('fetch failed (ECONNREFUSED)')
|
||||
return jsonResponse({ status: 'ok', model: null, gpu: false })
|
||||
}
|
||||
if (input.endsWith('/load')) return loadHandler()
|
||||
if (input.endsWith('/shutdown')) return jsonResponse({ ok: true })
|
||||
throw new TypeError(`unexpected ${input}`)
|
||||
}
|
||||
|
||||
const supervisor = new SidecarSupervisor({
|
||||
resolveLaunch: async () => ({ command: 'fake-sidecar', args: [], source: 'venv' }),
|
||||
modelsDir: () => 'models',
|
||||
baseUrlFor: (port) => `http://127.0.0.1:${port}`,
|
||||
spawn,
|
||||
fetch: fetchFn,
|
||||
findFreePort: async (start) => start,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
clock += ms
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
supervisor,
|
||||
fetch: fetchFn,
|
||||
children,
|
||||
spawn,
|
||||
fetchCalls,
|
||||
setHealthy: (value) => {
|
||||
healthy = value
|
||||
},
|
||||
onLoad: (handler) => {
|
||||
loadHandler = handler
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ── 1. 클라우드 실패 → 로컬 폴백 ─────────────────────────
|
||||
|
||||
describe('STTManager — 로컬 폴백은 복구 가능한 실패 + 설치된 로컬 엔진에서만', () => {
|
||||
const MODEL_ID = 'r2-6-installed-model'
|
||||
|
||||
beforeEach(() => {
|
||||
initInMemoryConfig()
|
||||
resetSTTManagerForTests()
|
||||
resetLocalSTTServiceForTests()
|
||||
vi.restoreAllMocks()
|
||||
launchMock.offline = true
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
rmSync(join(getWhisperModelsDir(), MODEL_ID), { recursive: true, force: true })
|
||||
resetInMemoryConfig()
|
||||
resetSTTManagerForTests()
|
||||
resetLocalSTTServiceForTests()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function useOpenAI(fetchResponse: unknown): void {
|
||||
const mgr = getSTTManager()
|
||||
mgr.setProvider('openai')
|
||||
mgr.setProviderConfig('openai', { apiKey: 'sk-test' })
|
||||
configSet('sttFallbackToLocal', true)
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fetchResponse))
|
||||
}
|
||||
|
||||
function installModel(): void {
|
||||
const dir = join(getWhisperModelsDir(), MODEL_ID)
|
||||
mkdirSync(dir, { recursive: true })
|
||||
writeFileSync(join(dir, 'model.bin'), 'x')
|
||||
configSet('sttModelId', MODEL_ID)
|
||||
}
|
||||
|
||||
it('무음(STTNoAudioData)은 로컬로 넘기지 않고 그대로 던진다', async () => {
|
||||
useOpenAI({ ok: true, status: 200, json: async () => ({ text: ' ' }) })
|
||||
const localTranscribe = vi.spyOn(getLocalSTTService(), 'transcribe')
|
||||
|
||||
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
||||
code: ErrorCode.STTNoAudioData,
|
||||
})
|
||||
expect(localTranscribe).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('로컬 모델이 설치되지 않았으면 초기화(=내려받기)를 시작하지 않고 원래 클라우드 에러로 끝난다', async () => {
|
||||
useOpenAI({ ok: false, status: 503, text: async () => 'offline' })
|
||||
configSet('sttModelId', 'large-v3-turbo')
|
||||
const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined)
|
||||
|
||||
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
||||
code: ErrorCode.STTTranscriptionFailed,
|
||||
})
|
||||
expect(init).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('엔진이 설치되지 않았으면(런타임 내려받기 필요) 초기화하지 않는다', async () => {
|
||||
useOpenAI({ ok: false, status: 503, text: async () => 'offline' })
|
||||
installModel()
|
||||
launchMock.offline = false
|
||||
const init = vi.spyOn(getLocalSTTService(), 'initialize').mockResolvedValue(undefined)
|
||||
|
||||
await expect(getSTTManager().transcribe(Buffer.alloc(32000))).rejects.toMatchObject({
|
||||
code: ErrorCode.STTTranscriptionFailed,
|
||||
})
|
||||
expect(init).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('모델과 엔진이 로컬에 있으면 폴백해 로컬 결과를 돌려준다', async () => {
|
||||
useOpenAI({ ok: false, status: 401, text: async () => 'Unauthorized' })
|
||||
installModel()
|
||||
const local = getLocalSTTService()
|
||||
const internals = local as unknown as LocalInternals
|
||||
const init = vi.spyOn(local, 'initialize').mockImplementation(async () => {
|
||||
internals._modelReady = true
|
||||
})
|
||||
vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('로컬 결과'))
|
||||
|
||||
const result = await getSTTManager().transcribe(Buffer.alloc(32000), { language: 'ko' })
|
||||
expect(result.text).toBe('로컬 결과')
|
||||
expect(init).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('폴백 정책: 무음/너무 짧음/취소는 복구 불가, 그 밖(인증·서버·네트워크)은 복구 가능', () => {
|
||||
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTNoAudioData, 'x'))).toBe(false)
|
||||
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTAudioTooShort, 'x'))).toBe(false)
|
||||
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionCancelled, 'x'))).toBe(false)
|
||||
expect(isRecoverableByLocalFallback(new D3ROError(ErrorCode.STTTranscriptionFailed, '401'))).toBe(true)
|
||||
expect(isRecoverableByLocalFallback(new Error('network'))).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
// ── 2. SidecarSupervisor — 기동 중 호출자는 헬스까지 기다린다 ──
|
||||
|
||||
describe('SidecarSupervisor', () => {
|
||||
it('기동 중(헬스 전)에 들어온 두 번째 호출자는 즉시 통과하지 않고 헬스까지 기다린다', async () => {
|
||||
const h = createHarness()
|
||||
const first = h.supervisor.ready()
|
||||
await flush()
|
||||
expect(h.spawn).toHaveBeenCalledTimes(1)
|
||||
expect(h.supervisor.state).toBe('starting')
|
||||
|
||||
let secondSettled = false
|
||||
const second = h.supervisor.ready().then((url) => {
|
||||
secondSettled = true
|
||||
return url
|
||||
})
|
||||
await flush(10)
|
||||
expect(secondSettled).toBe(false)
|
||||
expect(h.supervisor.isHealthy).toBe(false)
|
||||
|
||||
h.setHealthy(true)
|
||||
const [a, b] = await Promise.all([first, second])
|
||||
expect(a).toBe(b)
|
||||
expect(h.supervisor.isHealthy).toBe(true)
|
||||
expect(h.spawn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('크래시 백오프 중 ready()는 새로 spawn하지 않고 예약된 재시작에 합류한다', async () => {
|
||||
const h = createHarness()
|
||||
h.setHealthy(true)
|
||||
await h.supervisor.ready()
|
||||
const crashed = vi.fn()
|
||||
h.supervisor.on('crashed', crashed)
|
||||
|
||||
h.children[0].die(1)
|
||||
expect(h.supervisor.state).toBe('crashed')
|
||||
expect(crashed).toHaveBeenCalledWith(expect.objectContaining({ attempt: 1 }))
|
||||
|
||||
const [a, b] = await Promise.all([h.supervisor.ready(), h.supervisor.ready()])
|
||||
expect(a).toBe(b)
|
||||
expect(h.spawn).toHaveBeenCalledTimes(2)
|
||||
expect(h.supervisor.isHealthy).toBe(true)
|
||||
})
|
||||
|
||||
it('markStable 후에는 재시작 카운트가 초기화된다', async () => {
|
||||
const h = createHarness()
|
||||
h.setHealthy(true)
|
||||
await h.supervisor.ready()
|
||||
h.children[0].die(1)
|
||||
await h.supervisor.ready()
|
||||
expect(h.supervisor.restartCount).toBe(1)
|
||||
h.supervisor.markStable()
|
||||
expect(h.supervisor.restartCount).toBe(0)
|
||||
})
|
||||
|
||||
it('dispose 뒤 예약된 재시작은 취소되고 ready()는 거부된다', async () => {
|
||||
const h = createHarness()
|
||||
h.setHealthy(true)
|
||||
await h.supervisor.ready()
|
||||
h.children[0].die(1)
|
||||
const pending = h.supervisor.ready()
|
||||
await h.supervisor.dispose()
|
||||
await expect(pending).rejects.toBeInstanceOf(D3ROError)
|
||||
await expect(h.supervisor.ready()).rejects.toBeInstanceOf(D3ROError)
|
||||
expect(h.spawn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
// ── 3. LocalSTTService + 감독자 — 크래시 재시작 정책 ──────
|
||||
|
||||
describe('LocalSTTService — 사이드카 크래시 재시작', () => {
|
||||
beforeEach(() => {
|
||||
initInMemoryConfig()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
resetInMemoryConfig()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('모델 로딩 중 반복 크래시는 MAX 재시작에서 멈추고 STTSidecarCrashed를 알린다', async () => {
|
||||
const h = createHarness()
|
||||
vi.stubGlobal('fetch', h.fetch)
|
||||
h.setHealthy(true)
|
||||
// /load 도중 사이드카가 죽는다 (OOM/CUDA 실패 등)
|
||||
h.onLoad(async () => {
|
||||
h.children[h.children.length - 1].die(1)
|
||||
throw new TypeError('fetch failed')
|
||||
})
|
||||
const local = new LocalSTTService({ supervisor: h.supervisor })
|
||||
const errors: D3ROError[] = []
|
||||
local.on('error', ({ error }) => errors.push(error))
|
||||
|
||||
await expect(local.initialize('tiny')).rejects.toBeInstanceOf(D3ROError)
|
||||
for (let i = 0; i < 200 && h.supervisor.state !== 'stopped'; i++) {
|
||||
await flush()
|
||||
}
|
||||
|
||||
expect(h.supervisor.state).toBe('stopped')
|
||||
// 최초 1회 + 재시작 3회 — 헬스 성공이 카운트를 되돌리지 않는다
|
||||
expect(h.spawn).toHaveBeenCalledTimes(4)
|
||||
expect(errors.some((e) => e.code === ErrorCode.STTSidecarCrashed)).toBe(true)
|
||||
expect(local.getStatus().engineState).toBe('error')
|
||||
await local.dispose()
|
||||
})
|
||||
|
||||
it('모델이 없던(UIA만 쓰던) 사이드카가 죽으면 되살리기만 하고 모델은 올리지 않는다', async () => {
|
||||
const h = createHarness()
|
||||
vi.stubGlobal('fetch', h.fetch)
|
||||
h.setHealthy(true)
|
||||
const local = new LocalSTTService({ supervisor: h.supervisor })
|
||||
|
||||
await local.ensureSidecar()
|
||||
h.children[0].die(1)
|
||||
await local.ensureSidecar()
|
||||
await flush()
|
||||
|
||||
expect(h.spawn).toHaveBeenCalledTimes(2)
|
||||
expect(h.fetchCalls.some((url) => url.endsWith('/load'))).toBe(false)
|
||||
await local.dispose()
|
||||
})
|
||||
|
||||
it('크래시 재시작 대기 중 들어온 transcribe는 재시작(모델 재로딩) 결과를 기다려 전사한다', async () => {
|
||||
const h = createHarness()
|
||||
vi.stubGlobal('fetch', h.fetch)
|
||||
h.setHealthy(true)
|
||||
const local = new LocalSTTService({ supervisor: h.supervisor })
|
||||
const internals = local as unknown as LocalInternals
|
||||
await local.initialize('tiny')
|
||||
|
||||
h.children[0].die(1)
|
||||
const send = vi.spyOn(internals, '_sendToSidecar').mockResolvedValue(okResult('재시작 후 전사'))
|
||||
const result = await local.transcribe(Buffer.alloc(32000))
|
||||
|
||||
expect(result.text).toBe('재시작 후 전사')
|
||||
expect(send).toHaveBeenCalledTimes(1)
|
||||
expect(h.fetchCalls.filter((url) => url.endsWith('/load'))).toHaveLength(2)
|
||||
expect(local.currentModelId).toBe('tiny')
|
||||
await local.dispose()
|
||||
})
|
||||
})
|
||||
|
|
@ -651,7 +651,8 @@ describe('Complex User Journeys & Multi-Provider STT Orchestration (105 Scenario
|
|||
expect(res.text).toBe('타임아웃 로컬 대체')
|
||||
})
|
||||
|
||||
it('50. Provider returning empty text triggers fallback or error', async () => {
|
||||
// 무음(빈 전사)은 로컬로 다시 돌려도 결과가 같으므로 폴백하지 않고 STTNoAudioData 로 끝난다.
|
||||
it('50. Provider returning empty text ends with no-audio error and skips local fallback', async () => {
|
||||
const mgr = getSTTManager()
|
||||
mgr.setProvider('openai')
|
||||
mgr.setProviderConfig('openai', { apiKey: 'sk-test' })
|
||||
|
|
@ -667,7 +668,7 @@ describe('Complex User Journeys & Multi-Provider STT Orchestration (105 Scenario
|
|||
)
|
||||
|
||||
const { getLocalSTTService } = await import('../../src/main/services/LocalSTTService')
|
||||
vi.spyOn(getLocalSTTService(), 'transcribe').mockResolvedValue({
|
||||
const localTranscribe = vi.spyOn(getLocalSTTService(), 'transcribe').mockResolvedValue({
|
||||
text: '빈값 감지 후 로컬 복구',
|
||||
language: 'ko',
|
||||
duration: 1.0,
|
||||
|
|
@ -675,11 +676,11 @@ describe('Complex User Journeys & Multi-Provider STT Orchestration (105 Scenario
|
|||
segments: [],
|
||||
})
|
||||
|
||||
const res = await mgr.transcribe(makePcmBuffer(1.0))
|
||||
expect(res.text).toBe('빈값 감지 후 로컬 복구')
|
||||
await expect(mgr.transcribe(makePcmBuffer(1.0))).rejects.toMatchObject({ code: ErrorCode.STTNoAudioData })
|
||||
expect(localTranscribe).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('51. Provider returning null text triggers fallback', async () => {
|
||||
it('51. Provider returning null text ends with no-audio error and skips local fallback', async () => {
|
||||
const mgr = getSTTManager()
|
||||
mgr.setProvider('custom')
|
||||
mgr.setProviderConfig('custom', { baseUrl: 'http://custom:8000' })
|
||||
|
|
@ -695,7 +696,7 @@ describe('Complex User Journeys & Multi-Provider STT Orchestration (105 Scenario
|
|||
)
|
||||
|
||||
const { getLocalSTTService } = await import('../../src/main/services/LocalSTTService')
|
||||
vi.spyOn(getLocalSTTService(), 'transcribe').mockResolvedValue({
|
||||
const localTranscribe = vi.spyOn(getLocalSTTService(), 'transcribe').mockResolvedValue({
|
||||
text: '널값 감지 로컬 복구',
|
||||
language: 'ko',
|
||||
duration: 1.0,
|
||||
|
|
@ -703,8 +704,8 @@ describe('Complex User Journeys & Multi-Provider STT Orchestration (105 Scenario
|
|||
segments: [],
|
||||
})
|
||||
|
||||
const res = await mgr.transcribe(makePcmBuffer(1.0))
|
||||
expect(res.text).toBe('널값 감지 로컬 복구')
|
||||
await expect(mgr.transcribe(makePcmBuffer(1.0))).rejects.toMatchObject({ code: ErrorCode.STTNoAudioData })
|
||||
expect(localTranscribe).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('52. Local Whisper execution failure throws STTTranscriptionFailed', async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue