feat(desktop): make local speech transcription work end to end

Local dictation had never produced a transcript on an installed build. The
engine itself was healthy; every connection to it was broken.

Installed builds shipped no speech engine at all: the packaging config had no
entry for the faster-whisper sidecar and no pipeline step built one, so the app
always fell back to a system Python without the runtime. Development was broken
too, because the sidecar and SoX paths were resolved against the Vite output
directory instead of the app root, which also meant recording failed with a SoX
ENOENT. On hosts where localhost resolves only to IPv6, every local request was
refused outright, which silently disabled both local transcription and the local
LLM.

The sidecar is now built and bundled (including the Silero VAD data it needs),
gated by a packaging check that fails when the engine or its data is missing.
Paths are discovered from the app root and fail loudly when the engine is
absent. Local engine URLs are normalized to the IPv4 loopback, decoding is tuned
so repeated hallucinations cannot compound (the same transcript now takes about
a fifth of the time), the engine is warmed up at startup, and holding the hotkey
now shows the text forming live in the recording tip.
This commit is contained in:
Yun Chan 2026-09-18 00:48:47 +09:00
parent 359b244dc9
commit 2d585bfc29
52 changed files with 1450 additions and 3861 deletions

View file

@ -27,6 +27,7 @@ import {
hideRecordingTip,
updateRecordingTipState,
sendAudioLevelToTip,
sendPartialTranscriptToTip,
showResultPopup,
} from '../windows/WindowManager'
import type { ScreenContext } from '@d3ro/core/types'
@ -98,6 +99,18 @@ const TERMINAL_STATES = new Set<RecognitionState>([
RecognitionState.DESTROYED
])
// ── 실시간 부분 전사(미리보기) ──
// 16kHz 16bit mono = 32 bytes/ms
const BYTES_PER_MS = 32
/** 부분 전사 주기 */
const PARTIAL_INTERVAL_MS = 1500
/** 부분 전사를 시작할 최소 녹음 길이 */
const PARTIAL_MIN_AUDIO_MS = 1200
/** 부분 전사에 보낼 최대 오디오 창(끝부분만) — 오래 말해도 지연이 늘지 않게 한다 */
const PARTIAL_MAX_WINDOW_MS = 7500
/** 녹음 종료 시 진행 중 부분 전사를 기다리는 최대 시간 */
const PARTIAL_DRAIN_TIMEOUT_MS = 2500
// ============================================================
// VoiceModeService
// ============================================================
@ -123,6 +136,10 @@ class VoiceModeService extends EventEmitter {
/** 녹음 종료 후 STT 준비 대기 타이머 — 전사 시작 시 반드시 해제 */
private _sttWaitTimer: NodeJS.Timeout | null = null
// 실시간 부분 전사(미리보기)
private _partialTimer: NodeJS.Timeout | null = null
private _partialInFlight: Promise<void> | null = null
// Action Queue (이벤트 직렬화)
private _actionQueue: VoiceAction[] = []
private _isProcessingQueue = false
@ -462,6 +479,8 @@ class VoiceModeService extends EventEmitter {
this._setAudioState(AudioState.STREAMING)
logger.info('Audio capture started')
this._startPartialLoop()
this._tryFlushAll()
} catch (error) {
if (this._isInTerminalState()) return
@ -488,6 +507,7 @@ class VoiceModeService extends EventEmitter {
this._audioLevelHandler = null
}
this._audioStarted = false
this._stopPartialLoop()
try {
await audio.stop()
@ -496,6 +516,73 @@ class VoiceModeService extends EventEmitter {
}
}
// ── 실시간 부분 전사(미리보기) ─────────────────────────────
/**
* 녹음 중 주기적으로 지금까지의 오디오를 전사해 RecordingTip에 미리보기를 띄운다.
* 최종 삽입 텍스트와는 완전히 분리된 경로이며, 실패는 조용히 무시된다.
*/
private _startPartialLoop(): void {
this._stopPartialLoop()
if ((configGet('sttProvider') ?? 'local') !== 'local') return
this._partialTimer = setInterval(() => {
void this._runPartial()
}, PARTIAL_INTERVAL_MS)
}
private _stopPartialLoop(): void {
if (this._partialTimer) {
clearInterval(this._partialTimer)
this._partialTimer = null
}
}
/** 진행 중인 부분 전사가 끝나기를 최대 PARTIAL_DRAIN_TIMEOUT_MS까지 기다린다. */
private async _drainPartial(): Promise<void> {
const inFlight = this._partialInFlight
if (!inFlight) return
await Promise.race([
inFlight,
new Promise<void>((resolve) => setTimeout(resolve, PARTIAL_DRAIN_TIMEOUT_MS)),
])
}
private async _runPartial(): Promise<void> {
if (!this._audioStarted || this._partialInFlight) return
if (this._isInTerminalState()) return
if (!this._sttReady) return
if (this._audioBufferBytes < PARTIAL_MIN_AUDIO_MS * BYTES_PER_MS) return
const sessionId = this._session?.id
const merged = Buffer.concat(this._audioBuffer)
const maxBytes = PARTIAL_MAX_WINDOW_MS * BYTES_PER_MS
const window = merged.length > maxBytes ? merged.subarray(merged.length - maxBytes) : merged
const language = configGet('sttLanguage')
const task = (async (): Promise<void> => {
try {
const text = await getSTTManager().transcribePartial(window, {
language: language === 'auto' ? undefined : language,
vadFilter: false,
})
// 녹음이 끝났거나 세션이 바뀌었으면 미리보기를 버린다.
if (!this._audioStarted || this._isInTerminalState()) return
if (this._session?.id !== sessionId) return
if (!text) return
sendPartialTranscriptToTip(text)
} catch (err) {
logger.debug(
`부분 전사 미리보기 무시: ${err instanceof Error ? err.message : String(err)}`,
)
} finally {
this._partialInFlight = null
}
})()
this._partialInFlight = task
}
// ── 이중 조건 플러시 ───────────────────────────────────
private _tryFlushAll(): void {
@ -553,6 +640,10 @@ class VoiceModeService extends EventEmitter {
// DictionaryService 미초기화 시 무시
}
// 사이드카는 요청을 직렬 처리하므로, 진행 중인 미리보기 요청이 최종 전사를
// 지연시키지 않도록 먼저 배수한다(최대 PARTIAL_DRAIN_TIMEOUT_MS).
await this._drainPartial()
const result: TranscriptionResult = await stt.transcribe(merged, {
language: language === 'auto' ? undefined : language,
initialPrompt,
@ -864,6 +955,8 @@ class VoiceModeService extends EventEmitter {
private _resetToIdle(): void {
this._clearSttWaitTimer()
this._stopPartialLoop()
this._partialInFlight = null
this._session = null
this._audioBuffer = []
this._audioBufferBytes = 0