feat(desktop): make local speech transcription work end to end
Local dictation had never produced a transcript on an installed build. The engine itself was healthy; every connection to it was broken. Installed builds shipped no speech engine at all: the packaging config had no entry for the faster-whisper sidecar and no pipeline step built one, so the app always fell back to a system Python without the runtime. Development was broken too, because the sidecar and SoX paths were resolved against the Vite output directory instead of the app root, which also meant recording failed with a SoX ENOENT. On hosts where localhost resolves only to IPv6, every local request was refused outright, which silently disabled both local transcription and the local LLM. The sidecar is now built and bundled (including the Silero VAD data it needs), gated by a packaging check that fails when the engine or its data is missing. Paths are discovered from the app root and fail loudly when the engine is absent. Local engine URLs are normalized to the IPv4 loopback, decoding is tuned so repeated hallucinations cannot compound (the same transcript now takes about a fifth of the time), the engine is warmed up at startup, and holding the hotkey now shows the text forming live in the recording tip.
This commit is contained in:
parent
359b244dc9
commit
2d585bfc29
52 changed files with 1450 additions and 3861 deletions
|
|
@ -27,6 +27,7 @@ import {
|
|||
hideRecordingTip,
|
||||
updateRecordingTipState,
|
||||
sendAudioLevelToTip,
|
||||
sendPartialTranscriptToTip,
|
||||
showResultPopup,
|
||||
} from '../windows/WindowManager'
|
||||
import type { ScreenContext } from '@d3ro/core/types'
|
||||
|
|
@ -98,6 +99,18 @@ const TERMINAL_STATES = new Set<RecognitionState>([
|
|||
RecognitionState.DESTROYED
|
||||
])
|
||||
|
||||
// ── 실시간 부분 전사(미리보기) ──
|
||||
// 16kHz 16bit mono = 32 bytes/ms
|
||||
const BYTES_PER_MS = 32
|
||||
/** 부분 전사 주기 */
|
||||
const PARTIAL_INTERVAL_MS = 1500
|
||||
/** 부분 전사를 시작할 최소 녹음 길이 */
|
||||
const PARTIAL_MIN_AUDIO_MS = 1200
|
||||
/** 부분 전사에 보낼 최대 오디오 창(끝부분만) — 오래 말해도 지연이 늘지 않게 한다 */
|
||||
const PARTIAL_MAX_WINDOW_MS = 7500
|
||||
/** 녹음 종료 시 진행 중 부분 전사를 기다리는 최대 시간 */
|
||||
const PARTIAL_DRAIN_TIMEOUT_MS = 2500
|
||||
|
||||
// ============================================================
|
||||
// VoiceModeService
|
||||
// ============================================================
|
||||
|
|
@ -123,6 +136,10 @@ class VoiceModeService extends EventEmitter {
|
|||
/** 녹음 종료 후 STT 준비 대기 타이머 — 전사 시작 시 반드시 해제 */
|
||||
private _sttWaitTimer: NodeJS.Timeout | null = null
|
||||
|
||||
// 실시간 부분 전사(미리보기)
|
||||
private _partialTimer: NodeJS.Timeout | null = null
|
||||
private _partialInFlight: Promise<void> | null = null
|
||||
|
||||
// Action Queue (이벤트 직렬화)
|
||||
private _actionQueue: VoiceAction[] = []
|
||||
private _isProcessingQueue = false
|
||||
|
|
@ -462,6 +479,8 @@ class VoiceModeService extends EventEmitter {
|
|||
this._setAudioState(AudioState.STREAMING)
|
||||
logger.info('Audio capture started')
|
||||
|
||||
this._startPartialLoop()
|
||||
|
||||
this._tryFlushAll()
|
||||
} catch (error) {
|
||||
if (this._isInTerminalState()) return
|
||||
|
|
@ -488,6 +507,7 @@ class VoiceModeService extends EventEmitter {
|
|||
this._audioLevelHandler = null
|
||||
}
|
||||
this._audioStarted = false
|
||||
this._stopPartialLoop()
|
||||
|
||||
try {
|
||||
await audio.stop()
|
||||
|
|
@ -496,6 +516,73 @@ class VoiceModeService extends EventEmitter {
|
|||
}
|
||||
}
|
||||
|
||||
// ── 실시간 부분 전사(미리보기) ─────────────────────────────
|
||||
|
||||
/**
|
||||
* 녹음 중 주기적으로 지금까지의 오디오를 전사해 RecordingTip에 미리보기를 띄운다.
|
||||
* 최종 삽입 텍스트와는 완전히 분리된 경로이며, 실패는 조용히 무시된다.
|
||||
*/
|
||||
private _startPartialLoop(): void {
|
||||
this._stopPartialLoop()
|
||||
if ((configGet('sttProvider') ?? 'local') !== 'local') return
|
||||
|
||||
this._partialTimer = setInterval(() => {
|
||||
void this._runPartial()
|
||||
}, PARTIAL_INTERVAL_MS)
|
||||
}
|
||||
|
||||
private _stopPartialLoop(): void {
|
||||
if (this._partialTimer) {
|
||||
clearInterval(this._partialTimer)
|
||||
this._partialTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
/** 진행 중인 부분 전사가 끝나기를 최대 PARTIAL_DRAIN_TIMEOUT_MS까지 기다린다. */
|
||||
private async _drainPartial(): Promise<void> {
|
||||
const inFlight = this._partialInFlight
|
||||
if (!inFlight) return
|
||||
await Promise.race([
|
||||
inFlight,
|
||||
new Promise<void>((resolve) => setTimeout(resolve, PARTIAL_DRAIN_TIMEOUT_MS)),
|
||||
])
|
||||
}
|
||||
|
||||
private async _runPartial(): Promise<void> {
|
||||
if (!this._audioStarted || this._partialInFlight) return
|
||||
if (this._isInTerminalState()) return
|
||||
if (!this._sttReady) return
|
||||
if (this._audioBufferBytes < PARTIAL_MIN_AUDIO_MS * BYTES_PER_MS) return
|
||||
|
||||
const sessionId = this._session?.id
|
||||
const merged = Buffer.concat(this._audioBuffer)
|
||||
const maxBytes = PARTIAL_MAX_WINDOW_MS * BYTES_PER_MS
|
||||
const window = merged.length > maxBytes ? merged.subarray(merged.length - maxBytes) : merged
|
||||
const language = configGet('sttLanguage')
|
||||
|
||||
const task = (async (): Promise<void> => {
|
||||
try {
|
||||
const text = await getSTTManager().transcribePartial(window, {
|
||||
language: language === 'auto' ? undefined : language,
|
||||
vadFilter: false,
|
||||
})
|
||||
// 녹음이 끝났거나 세션이 바뀌었으면 미리보기를 버린다.
|
||||
if (!this._audioStarted || this._isInTerminalState()) return
|
||||
if (this._session?.id !== sessionId) return
|
||||
if (!text) return
|
||||
sendPartialTranscriptToTip(text)
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
`부분 전사 미리보기 무시: ${err instanceof Error ? err.message : String(err)}`,
|
||||
)
|
||||
} finally {
|
||||
this._partialInFlight = null
|
||||
}
|
||||
})()
|
||||
|
||||
this._partialInFlight = task
|
||||
}
|
||||
|
||||
// ── 이중 조건 플러시 ───────────────────────────────────
|
||||
|
||||
private _tryFlushAll(): void {
|
||||
|
|
@ -553,6 +640,10 @@ class VoiceModeService extends EventEmitter {
|
|||
// DictionaryService 미초기화 시 무시
|
||||
}
|
||||
|
||||
// 사이드카는 요청을 직렬 처리하므로, 진행 중인 미리보기 요청이 최종 전사를
|
||||
// 지연시키지 않도록 먼저 배수한다(최대 PARTIAL_DRAIN_TIMEOUT_MS).
|
||||
await this._drainPartial()
|
||||
|
||||
const result: TranscriptionResult = await stt.transcribe(merged, {
|
||||
language: language === 'auto' ? undefined : language,
|
||||
initialPrompt,
|
||||
|
|
@ -864,6 +955,8 @@ class VoiceModeService extends EventEmitter {
|
|||
|
||||
private _resetToIdle(): void {
|
||||
this._clearSttWaitTimer()
|
||||
this._stopPartialLoop()
|
||||
this._partialInFlight = null
|
||||
this._session = null
|
||||
this._audioBuffer = []
|
||||
this._audioBufferBytes = 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue