d3ro-voice/apps/desktop/src/main/services/VoiceModeService.ts
Yun Chan 2fe20fa7b5
Some checks failed
deploy-site / deploy (push) Failing after 39s
release / release-windows (push) Failing after 3m41s
portable-unsigned / portable-windows (push) Failing after 12m23s
release: ship v1.6.0 with paged suggestions and a cleaner phrase memory
Next-sentence suggestions now arrive one at a time up to twelve, shown three
per page with Ctrl+Alt+Up/Down to move, Left/Right to page, Enter to accept
and Esc to close; old default bindings migrate and the panel guide follows the
live bindings. The overlay is redesigned, stays put while candidates stream
and sits outside the input box when no caret is reported.

The personal phrase memory stops learning from terminals, code editors and
the coding-agent hub, ignores symbol-heavy lines and empty-field placeholders,
and prunes existing entries that break those rules.

Fixes suggestion keys starting dictation, installs stuck on a pre-1.5.0
speech engine without the focus endpoint, Ollama runner windows flashing
while typing, the speech engine starting twice, and cold-model timeouts.
Live captions can be dragged to a remembered position and show a waiting
notice until the first line arrives.

Bumps the product version to 1.6.0 (Android/iOS build 1060000).
2026-09-24 19:56:28 +09:00

1219 lines
43 KiB
TypeScript

// src/main/services/VoiceModeService.ts
// 전체 음성 파이프라인 오케스트레이터.
// 설계서 01의 IVoiceModeService 구현. Speakly VoiceModeService의
// 상태 머신, 이중 조건 플러시, Action Queue 패턴 적용.
import { EventEmitter } from 'events'
import { randomUUID } from 'crypto'
import { writeFile, mkdir } from 'fs/promises'
import { join } from 'path'
import { app } from 'electron'
import { getLogger } from './LoggerService'
import { getAudioCaptureService } from './AudioCaptureService'
import { getLocalSTTService } from './LocalSTTService'
import type { TranscriptionResult } from './LocalSTTService'
import { getSTTManager } from './stt/STTManager'
import { getKeyBindingService } from './KeyBindingService'
import type { KeyBindingTriggerPayload } from './KeyBindingService'
import { configGet } from './ConfigService'
import { getTextInsertService } from './TextInsertService'
import { getLocalLLMService } from './LocalLLMService'
import { buildInstructionInvocation, resolveTargetLanguage } from './llm-prompts'
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
import { TIMING } from '@d3ro/core/constants'
import { RecognitionState, AudioState } from '@d3ro/core/types'
import type { KeyBindingActionId, VoiceMode, VoiceState, LLMAction } from '@d3ro/core/types'
import {
showRecordingTip,
hideRecordingTip,
updateRecordingTipState,
sendAudioLevelToTip,
sendPartialTranscriptToTip,
showResultPopup,
} from '../windows/WindowManager'
import type { ScreenContext } from '@d3ro/core/types'
const logger = getLogger('VoiceModeService')
// ============================================================
// 내부 타입
// ============================================================
interface VoiceSession {
id: string
mode: VoiceMode
startedAt: number
recognitionState: RecognitionState
audioState: AudioState
audioBufferDurationMs: number
transcription: string
processedText: string | null
accidentalPress: boolean
screenContext: ScreenContext | null
}
interface VoiceAction {
type: 'press' | 'release' | 'escape'
timestamp: number
mode: VoiceMode
actionId: KeyBindingActionId
/** hold-to-talk 여부 — release 에서 세션을 끊을지 결정한다 */
holdMode: boolean
}
interface VoiceModeEvents {
'session-started': (payload: { session: VoiceSession }) => void
'recognition-state-changed': (payload: {
previous: RecognitionState
current: RecognitionState
}) => void
'audio-state-changed': (payload: {
previous: AudioState
current: AudioState
}) => void
'transcription-update': (payload: { text: string; isFinal: boolean }) => void
'session-completed': (payload: { session: VoiceSession; finalText: string }) => void
'session-cancelled': (payload: {
session: VoiceSession
reason: 'user' | 'timeout' | 'too-short'
}) => void
'audio-level': (payload: { level: number }) => void
error: (payload: {
error: D3ROError
session: VoiceSession | null
/** 'warning'은 세션을 막지 않는 알림 (기본 'error') */
severity?: 'error' | 'warning'
}) => void
/**
* Phase 3.2: Premium LLM 호출이 실패해 Local로 자동 fallback된 경우 emit.
* renderer에서 이 이벤트를 받아 Snackbar 경고 배너를 띄운다.
*/
'premium-llm-fallback': (payload: { reason: string }) => void
}
// ============================================================
// 터미널 상태 헬퍼
// ============================================================
const TERMINAL_STATES = new Set<RecognitionState>([
RecognitionState.COMPLETED,
RecognitionState.CANCELLED,
RecognitionState.ERROR,
RecognitionState.DESTROYED
])
// ── 실시간 부분 전사(미리보기) ──
// 16kHz 16bit mono = 32 bytes/ms
const BYTES_PER_MS = 32
/** 부분 전사 주기 */
const PARTIAL_INTERVAL_MS = 1500
/** 부분 전사를 시작할 최소 녹음 길이 */
const PARTIAL_MIN_AUDIO_MS = 1200
/** 부분 전사에 보낼 최대 오디오 창(끝부분만) — 오래 말해도 지연이 늘지 않게 한다 */
const PARTIAL_MAX_WINDOW_MS = 7500
/** 녹음 종료 시 진행 중 부분 전사를 기다리는 최대 시간 */
const PARTIAL_DRAIN_TIMEOUT_MS = 2500
// ============================================================
// VoiceModeService
// ============================================================
class VoiceModeService extends EventEmitter {
private _session: VoiceSession | null = null
private _recognitionState = RecognitionState.IDLE
private _audioState = AudioState.IDLE
// 이중 조건 플러시
private _sttReady = false
private _audioStarted = false
private _audioBuffer: Buffer[] = []
private _audioBufferBytes = 0
// 녹음 오디오 저장용
private _lastAudioBuffer: Buffer | null = null
// 에러 가드
private _errorEmitted = false
// 에러 popup 3초 hide 예약 타이머 (다음 세션 시작 시 취소해야 현재 recording tip이 살아남음)
private _errorHideTimer: NodeJS.Timeout | null = null
/** 녹음 종료 후 STT 준비 대기 타이머 — 전사 시작 시 반드시 해제 */
private _sttWaitTimer: NodeJS.Timeout | null = null
// 실시간 부분 전사(미리보기)
private _partialTimer: NodeJS.Timeout | null = null
private _partialInFlight: Promise<void> | null = null
// Action Queue (이벤트 직렬화)
private _actionQueue: VoiceAction[] = []
private _isProcessingQueue = false
// 리스너 해제용 참조
private _audioDataHandler: ((payload: { buffer: Buffer }) => void) | null = null
private _audioLevelHandler: ((payload: { level: number }) => void) | null = null
private _keyBindingHandler: ((payload: KeyBindingTriggerPayload) => void) | null = null
private _disposed = false
get currentSession(): VoiceSession | null {
return this._session
}
get isActive(): boolean {
return this._session !== null && !this._isInTerminalState()
}
getState(): VoiceState {
return {
recognitionState: this._recognitionState,
audioState: this._audioState,
mode: configGet('defaultLLMAction') === 'translate' ? 'hands-free' : 'dictation',
sessionId: this._session?.id ?? null,
recordingStartedAt: this._session?.startedAt ?? null
}
}
// ── 초기화 ──────────────────────────────────────────────
/**
* KeyBindingService 이벤트를 구독하여 키바인딩 → 세션 제어를 연결한다.
* bootstrap에서 호출한다.
*/
connectKeyBindings(): void {
this._keyBindingHandler = (payload) => {
switch (payload.actionId) {
case 'caption':
// Phase 10.1: caption은 VoiceModeService가 아닌 CaptionService로 라우팅.
// 토글이므로 press만 처리하고 release는 버린다.
if (payload.type === 'pressed') this._toggleCaption()
return
case 'dictation':
case 'hands-free':
case 'command':
break
default:
// 음성 세션 액션만 여기서 처리한다. 팝업·제안 액션은 bootstrap이 직접 구독한다
// (허용 목록이 아니면 새 액션이 생길 때마다 받아쓰기로 오인돼 녹음이 켜진다).
return
}
const holdMode = this._resolveHoldMode(payload.actionId, payload.holdMode)
const action: VoiceAction = {
type: payload.type === 'pressed' ? 'press' : 'release',
timestamp: payload.timestamp,
mode: this._resolveMode(payload.actionId, payload.isDoublePress),
actionId: payload.actionId,
holdMode
}
this._enqueueAction(action)
}
getKeyBindingService().on('triggered', this._keyBindingHandler)
logger.info('Key binding events connected')
}
// ── 세션 제어 ──────────────────────────────────────────
async startSession(mode: VoiceMode): Promise<void> {
if (this._disposed) return
if (this._session && !this._isInTerminalState()) {
logger.warn('Session already active, ignoring startSession')
return
}
// Phase 11: 라이센스 쿼터 체크
try {
const { getLicenseService } = await import('./LicenseService')
const { Feature } = await import('@d3ro/core/types')
const license = getLicenseService()
const access = license.canUse(Feature.DICTATION)
if (!access.allowed) {
const reason = access.reason === 'quota_exceeded' ? 'quota_exceeded' : access.reason === 'login_required' ? 'login_required' : 'tier_required'
license.promptUpgrade(Feature.DICTATION, reason)
logger.warn(`Dictation blocked: ${access.reason}`)
const code = access.reason === 'quota_exceeded' ? ErrorCode.QuotaExceeded : ErrorCode.TierRequired
this.emit('error', {
error: new D3ROError(code, `Dictation blocked: ${access.reason}`),
session: null,
})
return
}
license.consumeQuota(Feature.DICTATION)
} catch (err) {
if (err instanceof D3ROError) {
this.emit('error', { error: err, session: null })
return
}
// LicenseService 미초기화 시 허용 (graceful)
}
// Phase 10.1: 자막 모드 활성 중이면 dictation 세션 시작 불가 (AudioCaptureService 공유)
try {
const { getCaptionService } = await import('./CaptionService')
const captionState = getCaptionService().getState()
if (captionState === 'active' || captionState === 'starting') {
logger.warn('Cannot start dictation: caption mode active')
this.emit('error', {
error: new D3ROError(ErrorCode.CaptionAlreadyActive, 'Cannot start dictation: caption mode active'),
session: null,
})
return
}
} catch {
// CaptionService 미초기화 시 무시
}
// Phase 10.2: 스크린 컨텍스트 캡처 (녹음 시작 전, 활성 앱 정보 보존)
let screenContext: ScreenContext | null = null
try {
const { getScreenContextService } = await import('./ScreenContextService')
const ctx = getScreenContextService()
if (ctx.isEnabled()) {
const captureSelected = configGet('screenContextEnabled' as keyof import('@d3ro/core/types').AppConfig) as unknown as boolean
const result = await ctx.captureContext(captureSelected)
screenContext = result.context
logger.info(`Screen context captured: ${screenContext.appName ?? 'unknown'}`)
}
} catch {
// 컨텍스트 캡처 실패 시 무시 — 핵심 기능 아님
}
// 세션 생성
this._session = {
id: randomUUID(),
mode,
startedAt: Date.now(),
recognitionState: RecognitionState.PREPARING,
audioState: AudioState.IDLE,
audioBufferDurationMs: 0,
transcription: '',
processedText: null,
accidentalPress: false,
screenContext,
}
this._errorEmitted = false
this._sttReady = false
this._audioStarted = false
this._audioBuffer = []
this._audioBufferBytes = 0
// 이전 에러의 3초 hide 타이머가 살아있으면 취소 (그대로 두면 새 recording tip을
// ~2초 지점에서 숨겨버리는 버그 발생). _errorEmitted 리셋과 동일 타이밍.
if (this._errorHideTimer) {
clearTimeout(this._errorHideTimer)
this._errorHideTimer = null
}
this._setRecognitionState(RecognitionState.PREPARING)
this.emit('session-started', { session: this._session })
logger.info(`Session started: ${this._session.id} (mode: ${mode})`)
const provider = configGet('sttProvider') ?? 'local'
if (provider === 'local') {
const stt = getLocalSTTService()
const sttModelId = configGet('sttModelId')
const sttModel = stt.getModels().find((m) => m.id === sttModelId)
if (sttModel && !sttModel.downloaded) {
this._handleError(
new D3ROError(ErrorCode.STTModelNotFound, `STT model not installed: ${sttModelId}`),
)
return
}
}
// Speakly 패턴: 녹음 시작 시 즉시 RecordingTip 표시
showRecordingTip('recording')
// 이중 조건 플러시: STT 초기화 + 오디오 캡처를 병렬 시작.
// 주의: STT 초기화는 await하지 않는다 — sidecar 기동이 느리거나 실패할 때
// press 액션이 action queue를 점유해 release가 수십 초 지연되던 버그의 원인.
// (_initSTT는 내부에서 에러를 _handleError로 처리하므로 fire-and-forget 안전)
void this._initSTT()
await this._startAudio()
}
async stopSession(): Promise<void> {
if (!this._session || this._isInTerminalState()) return
const session = this._session
const duration = Date.now() - session.startedAt
// accidentalPress 체크 (700ms 미만)
if (duration < TIMING.MIN_AUDIO_DURATION) {
session.accidentalPress = true
logger.info(`Accidental press detected (${duration}ms < ${TIMING.MIN_AUDIO_DURATION}ms)`)
this._cancelSession('too-short')
return
}
// 오디오 캡처 중지
await this._stopAudio()
// Speakly 패턴: 녹음 종료 후 thinking 상태로 전환
updateRecordingTipState('thinking')
// 버퍼가 있으면 STT에 전달
if (this._audioBuffer.length > 0 && this._sttReady) {
await this._transcribe()
} else if (this._audioBuffer.length > 0 && !this._sttReady) {
// STT 아직 준비 안 됨 → _initSTT 완료 후 _tryFlushAll이 전사
logger.info('Waiting for STT to be ready before transcribing')
this._clearSttWaitTimer()
this._sttWaitTimer = setTimeout(() => {
// 전사가 이미 시작됐으면(RECOGNIZING) 취소하지 않는다.
// 예전엔 6초 타이머가 첫 모델 로딩 직후 전사를 잘라 '전사가 안 됨'으로 보였다.
if (this._session?.id === session.id && !this._sttReady && !this._isInTerminalState()) {
logger.warn('STT readiness timeout, cancelling session')
this._cancelSession('timeout')
}
}, TIMING.ABSOLUTE_MAX_WAIT)
} else {
// 오디오 없음
logger.warn('No audio buffer, cancelling session')
this._cancelSession('too-short')
}
}
cancelSession(): void {
this._cancelSession('user')
}
// ── 상태 머신 ──────────────────────────────────────────
private _isInTerminalState(): boolean {
return TERMINAL_STATES.has(this._recognitionState)
}
private _setRecognitionState(state: RecognitionState): void {
if (this._recognitionState === state) return
if (this._isInTerminalState() && state !== RecognitionState.IDLE) return
const previous = this._recognitionState
this._recognitionState = state
if (this._session) {
this._session.recognitionState = state
}
this.emit('recognition-state-changed', { previous, current: state })
logger.debug(`RecognitionState: ${previous} → ${state}`)
}
private _setAudioState(state: AudioState): void {
if (this._audioState === state) return
const previous = this._audioState
this._audioState = state
if (this._session) {
this._session.audioState = state
}
this.emit('audio-state-changed', { previous, current: state })
logger.debug(`AudioState: ${previous} → ${state}`)
}
// ── STT 초기화 ─────────────────────────────────────────
private async _initSTT(): Promise<void> {
try {
this._setRecognitionState(RecognitionState.CONNECTING)
const provider = configGet('sttProvider') ?? 'local'
if (provider === 'local') {
const stt = getLocalSTTService()
const modelId = configGet('sttModelId')
await stt.initialize(modelId)
}
if (this._isInTerminalState()) return
this._sttReady = true
this._clearSttWaitTimer()
this._setRecognitionState(RecognitionState.READY)
logger.info(`STT ready (provider: ${provider})`)
this._tryFlushAll()
} catch (error) {
if (this._isInTerminalState()) return
this._handleError(
new D3ROError(
ErrorCode.STTModelLoadFailed,
`STT initialization failed: ${error instanceof Error ? error.message : String(error)}`
)
)
}
}
// ── 오디오 캡처 ────────────────────────────────────────
private async _startAudio(): Promise<void> {
try {
this._setAudioState(AudioState.INITIALIZING)
const audio = getAudioCaptureService()
// 오디오 데이터 수신
this._audioDataHandler = (payload) => {
if (this._isInTerminalState()) return
this._audioBuffer.push(payload.buffer)
this._audioBufferBytes += payload.buffer.length
// 버퍼 duration 업데이트 (16kHz, 16bit, mono)
const durationMs = (this._audioBufferBytes / 2 / 16000) * 1000
if (this._session) {
this._session.audioBufferDurationMs = durationMs
}
this._tryFlushAll()
}
this._audioLevelHandler = (payload) => {
this.emit('audio-level', { level: payload.level })
// Speakly 패턴: 오디오 레벨을 RecordingTip 웨이브 바에 전달
sendAudioLevelToTip(payload.level)
}
audio.on('audio-data', this._audioDataHandler)
audio.on('audio-level', this._audioLevelHandler)
await audio.start()
if (this._isInTerminalState()) return
this._audioStarted = true
this._setAudioState(AudioState.STREAMING)
logger.info('Audio capture started')
this._startPartialLoop()
this._tryFlushAll()
} catch (error) {
if (this._isInTerminalState()) return
this._setAudioState(AudioState.STOPPED)
this._handleError(
new D3ROError(
ErrorCode.AudioCaptureStartFailed,
`Audio capture failed: ${error instanceof Error ? error.message : String(error)}`
)
)
}
}
private async _stopAudio(): Promise<void> {
this._setAudioState(AudioState.STOPPED)
const audio = getAudioCaptureService()
if (this._audioDataHandler) {
audio.off('audio-data', this._audioDataHandler)
this._audioDataHandler = null
}
if (this._audioLevelHandler) {
audio.off('audio-level', this._audioLevelHandler)
this._audioLevelHandler = null
}
this._audioStarted = false
this._stopPartialLoop()
try {
await audio.stop()
} catch (error) {
logger.warn(`Audio stop error: ${error instanceof Error ? error.message : String(error)}`)
}
}
// ── 실시간 부분 전사(미리보기) ─────────────────────────────
/**
* 녹음 중 주기적으로 지금까지의 오디오를 전사해 RecordingTip에 미리보기를 띄운다.
* 최종 삽입 텍스트와는 완전히 분리된 경로이며, 실패는 조용히 무시된다.
*/
private _startPartialLoop(): void {
this._stopPartialLoop()
if ((configGet('sttProvider') ?? 'local') !== 'local') return
this._partialTimer = setInterval(() => {
void this._runPartial()
}, PARTIAL_INTERVAL_MS)
}
private _stopPartialLoop(): void {
if (this._partialTimer) {
clearInterval(this._partialTimer)
this._partialTimer = null
}
}
/** 진행 중인 부분 전사가 끝나기를 최대 PARTIAL_DRAIN_TIMEOUT_MS까지 기다린다. */
private async _drainPartial(): Promise<void> {
const inFlight = this._partialInFlight
if (!inFlight) return
await Promise.race([
inFlight,
new Promise<void>((resolve) => setTimeout(resolve, PARTIAL_DRAIN_TIMEOUT_MS)),
])
}
private async _runPartial(): Promise<void> {
if (!this._audioStarted || this._partialInFlight) return
if (this._isInTerminalState()) return
if (!this._sttReady) return
if (this._audioBufferBytes < PARTIAL_MIN_AUDIO_MS * BYTES_PER_MS) return
const sessionId = this._session?.id
const merged = Buffer.concat(this._audioBuffer)
const maxBytes = PARTIAL_MAX_WINDOW_MS * BYTES_PER_MS
const window = merged.length > maxBytes ? merged.subarray(merged.length - maxBytes) : merged
const language = configGet('sttLanguage')
const task = (async (): Promise<void> => {
try {
const text = await getSTTManager().transcribePartial(window, {
language: language === 'auto' ? undefined : language,
vadFilter: false,
})
// 녹음이 끝났거나 세션이 바뀌었으면 미리보기를 버린다.
if (!this._audioStarted || this._isInTerminalState()) return
if (this._session?.id !== sessionId) return
if (!text) return
sendPartialTranscriptToTip(text)
} catch (err) {
logger.debug(
`부분 전사 미리보기 무시: ${err instanceof Error ? err.message : String(err)}`,
)
} finally {
this._partialInFlight = null
}
})()
this._partialInFlight = task
}
// ── 이중 조건 플러시 ───────────────────────────────────
private _tryFlushAll(): void {
// 녹음 진행 중에는 flush 하지 않음 (stopSession에서 _stopAudio 후 처리)
if (this._audioStarted) return
if (!this._sttReady) return
if (this._audioBuffer.length === 0) {
logger.warn(`_tryFlushAll skipped: audioBuffer is empty`)
return
}
if (this._isInTerminalState()) return
this._clearSttWaitTimer()
void this._transcribe().catch((err) => {
logger.error('Unhandled error in _transcribe flush:', err)
})
}
// ── 전사 ───────────────────────────────────────────────
private async _transcribe(): Promise<void> {
if (this._audioBuffer.length === 0) return
if (this._isInTerminalState()) return
this._setRecognitionState(RecognitionState.RECOGNIZING)
const merged = Buffer.concat(this._audioBuffer)
this._lastAudioBuffer = merged // WAV 저장용 복사본
this._audioBuffer = []
this._audioBufferBytes = 0
// Bug 13: 너무 짧은 오디오는 STT 호출 전에 에러 피드백.
// 16kHz 16bit mono 기준 0.5초 = 16000 bytes.
const minBytes = 16000 * 2 * 0.5
if (merged.length < minBytes) {
this._handleError(
new D3ROError(
ErrorCode.STTAudioTooShort,
'No speech detected. Please speak and try again.'
)
)
return
}
try {
const stt = getSTTManager()
const language = configGet('sttLanguage')
// Dictionary → STT initialPrompt 주입 (Speakly 패턴)
let initialPrompt: string | undefined
try {
const { getDictionaryService } = await import('./DictionaryService')
initialPrompt = getDictionaryService().getPromptHints() || undefined
} catch {
// DictionaryService 미초기화 시 무시
}
// 사이드카는 요청을 직렬 처리하므로, 진행 중인 미리보기 요청이 최종 전사를
// 지연시키지 않도록 먼저 배수한다(최대 PARTIAL_DRAIN_TIMEOUT_MS).
await this._drainPartial()
const result: TranscriptionResult = await stt.transcribe(merged, {
language: language === 'auto' ? undefined : language,
initialPrompt,
})
if (this._isInTerminalState()) return
// Bug 13: VAD가 전체 오디오를 무음 판정했거나 결과가 빈 텍스트일 때
// 조용히 빈 문자열로 complete 하지 않고 사용자에게 피드백.
if (!result.text || result.text.trim().length === 0) {
this._handleError(
new D3ROError(
ErrorCode.STTNoAudioData,
'No speech detected. Check your microphone and try again.'
)
)
return
}
if (this._session) {
this._session.transcription = result.text
}
this.emit('transcription-update', { text: result.text, isFinal: true })
// Phase 10.5: 음성 단축키 — 키워드 매칭으로 LLM 명령어 자동 선택
let effectiveText = result.text
let overrideAction: string | null = null
let overrideInstructionId: string | null = null
try {
const { getVoiceCommandService } = await import('./VoiceCommandService')
const vcSvc = getVoiceCommandService()
if (vcSvc.isEnabled()) {
const match = vcSvc.match(result.text)
if (match.matched && match.instructionId) {
effectiveText = match.cleanedText
overrideAction = 'custom'
overrideInstructionId = match.instructionId
logger.info(`Voice command matched: keyword="${match.matchedKeyword}", instruction=${match.instructionId}`)
}
}
} catch {
// VoiceCommandService 미초기화 시 무시
}
const llmAction = overrideAction ?? configGet('defaultLLMAction')
const backend = configGet('llmBackend')
const ollamaUnavailable = backend === 'local' && !getLocalLLMService().isAvailable()
const skipLLM = llmAction === 'none' || ollamaUnavailable
if (skipLLM) {
if (llmAction !== 'none' && ollamaUnavailable) {
this.emit('error', {
error: new D3ROError(
ErrorCode.LLMServerUnreachable,
'Ollama unavailable — inserting raw transcription without LLM post-processing',
),
session: this._session ? { ...this._session } : null,
severity: 'warning',
})
}
this._completeSession(effectiveText)
} else {
await this._processWithLLM(effectiveText, overrideInstructionId)
}
} catch (error) {
if (this._isInTerminalState()) return
this._handleError(
new D3ROError(
ErrorCode.STTTranscriptionFailed,
`Transcription failed: ${error instanceof Error ? error.message : String(error)}`
)
)
}
}
// ── LLM 후처리 ─────────────────────────────────────────
private async _getLLMProcessor(): Promise<{
service: { processText(text: string, action: LLMAction, targetLanguage?: string, customPrompt?: string): Promise<string> }
backend: 'local' | 'premium'
}> {
const backend = configGet('llmBackend')
if (backend === 'online') {
try {
const { getPremiumLLMService } = await import('./PremiumLLMService')
const premium = getPremiumLLMService()
if (premium.isAvailable()) {
return { service: premium, backend: 'premium' }
}
this._emitPremiumFallback('Premium unavailable — falling back to local Ollama')
} catch (err) {
this._emitPremiumFallback(
`Premium init failed: ${err instanceof Error ? err.message : String(err)}`,
)
}
}
return { service: getLocalLLMService(), backend: 'local' }
}
private _emitPremiumFallback(reason: string): void {
logger.warn(`Premium LLM fallback → local: ${reason}`)
this.emit('premium-llm-fallback', { reason })
}
private async _runProcessorWithFallback(
text: string,
action: LLMAction,
targetLanguage?: string,
customPrompt?: string,
): Promise<string> {
const processor = await this._getLLMProcessor()
try {
return await processor.service.processText(text, action, targetLanguage, customPrompt)
} catch (err) {
if (processor.backend === 'premium') {
this._emitPremiumFallback(
`Premium call failed: ${err instanceof Error ? err.message : String(err)}`,
)
return getLocalLLMService().processText(text, action, targetLanguage, customPrompt)
}
throw err
}
}
private async _processWithLLM(transcribedText: string, overrideInstructionId?: string | null): Promise<void> {
if (this._isInTerminalState()) return
try {
const configuredAction = configGet('defaultLLMAction')
// 음성 단축키가 특정 명령을 지목해 들어왔다면 기본 액션이 'none'이어도 처리한다.
// 명시적 요청을 기본 설정이 무효화하면 안 된다.
if (configuredAction === 'none' && !overrideInstructionId) {
this._completeSession(transcribedText)
return
}
// 여기서 'none'이 남아 있다면 overrideInstructionId가 반드시 있다 → custom 경로.
const action: LLMAction = configuredAction === 'none' ? 'custom' : configuredAction
// Phase 10.2: 스크린 컨텍스트를 LLM 프롬프트에 주입
let contextPrefix = ''
if (this._session?.screenContext) {
try {
const { getScreenContextService } = await import('./ScreenContextService')
contextPrefix = getScreenContextService().buildContextPrompt(this._session.screenContext)
} catch {
// 무시
}
}
// Phase 10.4: 체인 모드 처리
if (action === 'chain') {
try {
const { getChainService } = await import('./ChainService')
const activeChainId = configGet('activeChainId')
if (activeChainId) {
const chainResult = await getChainService().execute(activeChainId, contextPrefix + transcribedText)
if (this._isInTerminalState()) return
if (this._session) this._session.processedText = chainResult.finalText
this._completeSession(chainResult.finalText)
return
}
} catch (error) {
logger.warn(`Chain execution failed: ${error instanceof Error ? error.message : String(error)}`)
this._handleError(
error instanceof D3ROError
? error
: new D3ROError(ErrorCode.ChainExecutionFailed, `Chain execution failed: ${error instanceof Error ? error.message : String(error)}`),
)
return
}
}
let processedText: string
// 음성 단축키 오버라이드 또는 활성 명령어
const effectiveInstructionId = overrideInstructionId
?? configGet('activeInstructionId')
const targetLanguage = resolveTargetLanguage()
if (action === 'custom' || overrideInstructionId) {
const userText = contextPrefix + transcribedText
let invocationText = userText
let instructionPrompt: string | undefined
if (effectiveInstructionId) {
const { getCustomInstructionService } = await import('./CustomInstructionService')
const instruction = getCustomInstructionService().getById(effectiveInstructionId)
if (instruction) {
const invocation = buildInstructionInvocation(
instruction.prompt,
userText,
targetLanguage,
)
invocationText = invocation.text
instructionPrompt = invocation.systemPrompt
logger.info(`Using custom instruction: "${instruction.name}"`)
} else {
logger.warn(
`Custom instruction not found: ${effectiveInstructionId} — processing without an instruction`,
)
}
}
processedText = await this._runProcessorWithFallback(
invocationText,
'custom',
undefined,
instructionPrompt,
)
} else {
logger.info(
action === 'translate'
? `Processing with LLM (action: ${action}, targetLanguage: ${targetLanguage})`
: `Processing with LLM (action: ${action})`,
)
processedText = await this._runProcessorWithFallback(
contextPrefix + transcribedText,
action,
action === 'translate' ? targetLanguage : undefined,
)
}
if (this._isInTerminalState()) return
if (this._session) {
this._session.processedText = processedText
}
this._completeSession(processedText)
} catch (error) {
if (this._isInTerminalState()) return
this._handleError(
error instanceof D3ROError
? error
: new D3ROError(
ErrorCode.LLMProcessingFailed,
`LLM processing failed: ${error instanceof Error ? error.message : String(error)}`,
),
)
}
}
// ── 세션 완료/취소 ─────────────────────────────────────
private async _completeSession(finalText: string): Promise<void> {
if (!this._session) return
const session = { ...this._session }
logger.info(`Session completed: "${finalText.substring(0, 50)}${finalText.length > 50 ? '...' : ''}"`)
// Speakly 패턴: 먼저 RecordingTip 숨기고 IDLE 복귀 (다음 핫키 즉시 사용 가능)
hideRecordingTip()
this._setRecognitionState(RecognitionState.COMPLETED)
this._setAudioState(AudioState.STOPPED)
this.emit('session-completed', { session, finalText })
this._resetToIdle()
// 텍스트 삽입은 세션과 무관하게 비동기 실행 (세션 블로킹 방지)
if (configGet('autoInsert') && finalText.length > 0) {
try {
const insertMethod = configGet('insertMethod')
await getTextInsertService().insertText(finalText, insertMethod)
} catch (error) {
logger.warn(`Text insert failed: ${error instanceof Error ? error.message : String(error)}`)
// 삽입 실패 시 ResultPopup에 텍스트 표시
showResultPopup(finalText, 10000)
}
}
// 녹음 오디오 WAV 파일 저장 (비동기, 실패해도 무시)
this._saveAudioFile(session.id)
}
/** PCM 버퍼를 WAV 파일로 저장 */
private async _saveAudioFile(sessionId: string): Promise<void> {
if (!this._lastAudioBuffer || this._lastAudioBuffer.length === 0) return
try {
const recordingsDir = join(app.getPath('userData'), 'recordings')
await mkdir(recordingsDir, { recursive: true })
const wavPath = join(recordingsDir, `${sessionId}.wav`)
const pcmData = this._lastAudioBuffer
// WAV 헤더 생성 (16kHz, 16bit, mono)
const header = Buffer.alloc(44)
const dataSize = pcmData.length
const fileSize = dataSize + 36
header.write('RIFF', 0)
header.writeUInt32LE(fileSize, 4)
header.write('WAVE', 8)
header.write('fmt ', 12)
header.writeUInt32LE(16, 16) // fmt chunk size
header.writeUInt16LE(1, 20) // PCM format
header.writeUInt16LE(1, 22) // mono
header.writeUInt32LE(16000, 24) // sample rate
header.writeUInt32LE(32000, 28) // byte rate (16000 * 2)
header.writeUInt16LE(2, 32) // block align
header.writeUInt16LE(16, 34) // bits per sample
header.write('data', 36)
header.writeUInt32LE(dataSize, 40)
await writeFile(wavPath, Buffer.concat([header, pcmData]))
logger.info(`Audio saved: ${wavPath} (${Math.round(dataSize / 1024)}KB)`)
} catch (error) {
logger.warn(`Audio save failed: ${error instanceof Error ? error.message : String(error)}`)
} finally {
this._lastAudioBuffer = null
}
}
private _cancelSession(reason: 'user' | 'timeout' | 'too-short'): void {
if (!this._session) return
this._setRecognitionState(RecognitionState.CANCELLED)
this._setAudioState(AudioState.STOPPED)
const session = { ...this._session }
logger.info(`Session cancelled: ${reason}`)
// Speakly 패턴: RecordingTip 즉시 숨김
hideRecordingTip()
// 오디오 정리
this._stopAudio()
this.emit('session-cancelled', { session, reason })
this._resetToIdle()
}
private _clearSttWaitTimer(): void {
if (this._sttWaitTimer) {
clearTimeout(this._sttWaitTimer)
this._sttWaitTimer = null
}
}
private _resetToIdle(): void {
this._clearSttWaitTimer()
this._stopPartialLoop()
this._partialInFlight = null
this._session = null
this._audioBuffer = []
this._audioBufferBytes = 0
this._sttReady = false
this._audioStarted = false
this._errorEmitted = false
// 즉시 IDLE로 전이 (딜레이는 race condition 유발)
this._setRecognitionState(RecognitionState.IDLE)
this._setAudioState(AudioState.IDLE)
}
// ── 에러 처리 ──────────────────────────────────────────
private _handleError(error: D3ROError): void {
if (this._errorEmitted) return
this._errorEmitted = true
logger.error(`VoiceMode error [${error.code}]: ${error.message}`)
this._setRecognitionState(RecognitionState.ERROR)
this._stopAudio()
// Speakly 패턴: RecordingTip에 에러 표시 → 3초 후 자동 숨김.
// 핸들 보관 → 다음 세션 시작 시 clearTimeout으로 취소(없으면 새 recording tip 숨김).
updateRecordingTipState('error', { errorMessage: error.message })
if (this._errorHideTimer) {
clearTimeout(this._errorHideTimer)
}
this._errorHideTimer = setTimeout(() => {
this._errorHideTimer = null
hideRecordingTip()
}, 3000)
this.emit('error', { error, session: this._session ? { ...this._session } : null })
this._resetToIdle()
}
// ── Action Queue (이벤트 직렬화) ───────────────────────
private _enqueueAction(action: VoiceAction): void {
this._actionQueue.push(action)
this._processQueue()
}
private async _processQueue(): Promise<void> {
if (this._isProcessingQueue) return
this._isProcessingQueue = true
try {
while (this._actionQueue.length > 0) {
const action = this._actionQueue.shift()!
await this._processAction(action)
}
} finally {
this._isProcessingQueue = false
}
}
private async _processAction(action: VoiceAction): Promise<void> {
try {
switch (action.type) {
case 'press':
await this._handlePress(action)
break
case 'release':
await this._handleRelease(action)
break
case 'escape':
this.cancelSession()
break
}
} catch (error) {
logger.error(`Action processing error: ${error instanceof Error ? error.message : String(error)}`)
}
}
private async _handlePress(action: VoiceAction): Promise<void> {
if (action.mode === 'dictation') {
// Dictation: hold-to-talk — press로 시작
if (!this.isActive) {
await this.startSession('dictation')
}
} else if (action.mode === 'hands-free') {
// HandsFree: toggle
if (this.isActive) {
await this.stopSession()
} else {
await this.startSession('hands-free')
}
}
}
private async _handleRelease(action: VoiceAction): Promise<void> {
if (action.holdMode && action.mode === 'dictation' && this.isActive) {
// Dictation: hold-to-talk — release로 즉시 종료
// Speakly 패턴: 딜레이 없이 즉시 stop (딜레이가 race condition 유발)
await this.stopSession()
}
// HandsFree(토글): release 무시
}
// ── 유틸리티 ───────────────────────────────────────────
private _resolveMode(actionId: KeyBindingActionId, isDoublePress: boolean): VoiceMode {
if (actionId === 'hands-free' || isDoublePress) {
return 'hands-free'
}
return 'dictation'
}
/**
* 'command' 액션에는 아직 전용 핸들러가 없다.
* 현행 동작대로 dictation 파이프라인으로 fallback 하며, 그 경로는 hold-to-talk 이므로
* KEYBINDING_ACTIONS 의 holdMode(false)가 아니라 dictation 과 같은 값을 쓴다.
*/
private _resolveHoldMode(actionId: KeyBindingActionId, specHoldMode: boolean): boolean {
if (actionId === 'command') return true
return specHoldMode
}
// ── 자막 모드 토글 (Phase 10.1) ─────────────────────────
private async _toggleCaption(): Promise<void> {
try {
const { getCaptionService } = await import('./CaptionService')
const caption = getCaptionService()
const state = caption.getState()
if (state === 'active' || state === 'starting') {
// 자막 활성 중 → 정지
await caption.stop()
logger.info('Caption stopped via hotkey')
} else {
// dictation 세션이 활성이면 자막 시작 불가 (상호 배제)
if (this._session && !this._isInTerminalState()) {
logger.warn('Cannot start caption: dictation session active')
return
}
// 회의 모드 녹음 중이면 자막 시작 불가 (상호 배제)
const { getMeetingModeService } = await import('./MeetingModeService')
if (getMeetingModeService().getState() === 'recording') {
logger.warn('Cannot start caption: meeting mode recording active')
return
}
await caption.start()
logger.info('Caption started via hotkey')
}
} catch (error) {
logger.error(`Caption toggle failed: ${error instanceof Error ? error.message : String(error)}`)
}
}
// ── 종료 ───────────────────────────────────────────────
dispose(): void {
this._disposed = true
this._actionQueue = []
// 진행 중 세션 취소
if (this._session && !this._isInTerminalState()) {
this._cancelSession('user')
}
// 키바인딩 리스너 해제
if (this._keyBindingHandler) {
getKeyBindingService().off('triggered', this._keyBindingHandler)
this._keyBindingHandler = null
}
// 오디오 리스너 해제
this._stopAudio()
// 에러 hide 타이머 정리 (pending 시 메모리 leak 방지)
if (this._errorHideTimer) {
clearTimeout(this._errorHideTimer)
this._errorHideTimer = null
}
this._setRecognitionState(RecognitionState.DESTROYED)
this.removeAllListeners()
logger.info('VoiceModeService disposed')
}
// ── EventEmitter 타입 오버라이드 ───────────────────────
override on<K extends keyof VoiceModeEvents>(
event: K,
listener: VoiceModeEvents[K]
): this {
return super.on(event, listener)
}
override off<K extends keyof VoiceModeEvents>(
event: K,
listener: VoiceModeEvents[K]
): this {
return super.off(event, listener)
}
override emit<K extends keyof VoiceModeEvents>(
event: K,
...args: Parameters<VoiceModeEvents[K]>
): boolean {
return super.emit(event, ...args)
}
}
// ── 싱글톤 ─────────────────────────────────────────────
let instance: VoiceModeService | null = null
export function getVoiceModeService(): VoiceModeService {
if (!instance) {
instance = new VoiceModeService()
}
return instance
}
export function resetVoiceModeServiceForTests(): void {
if (instance) {
instance.removeAllListeners()
}
instance = null
}