// src/main/services/VoiceConversationService.ts // Phase 13.1: 음성 대화 모드 — STT → LLM(chat) → TTS 루프 // 싱글톤 + EventEmitter. 대화 히스토리 최근 10턴 유지. import { EventEmitter } from 'events' import { getLogger } from './LoggerService' import { getLocalLLMService } from './LocalLLMService' import { getLocalSTTService } from './LocalSTTService' import { getAudioCaptureService } from './AudioCaptureService' import { getTTSPlaybackService } from './TTSPlaybackService' import { getSoundEffectService } from './SoundEffectService' import { configGet } from './ConfigService' import { getMainWindow } from '../windows/WindowManager' import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' import { D3ROError, ErrorCode } from '@d3ro/core/errors' import type { ConversationState, ConversationMessage, ConversationSessionInfo, ConversationAssistantDelta, ConversationAssistantMessage, ConversationError, } from '@d3ro/core/types' const logger = getLogger('VoiceConversationService') /** 대화 히스토리 최대 턴 수 (user+assistant 쌍) */ const MAX_HISTORY_TURNS = 10 /** 시스템 프롬프트 */ const SYSTEM_PROMPT = `You are D3RO, a helpful local AI voice assistant. Respond concisely and naturally, as if having a spoken conversation. Keep answers brief (2-3 sentences) unless the user asks for detail. Respond in the same language the user speaks.` class VoiceConversationService extends EventEmitter { private _state: ConversationState = 'idle' private _messages: ConversationMessage[] = [] private _isActive = false private _audioBuffers: Buffer[] = [] private _audioListenerBound = false private _audioLevelListenerBound = false get state(): ConversationState { return this._state } get isActive(): boolean { return this._isActive } getSessionInfo(): ConversationSessionInfo { return { state: this._state, messages: [...this._messages], isActive: this._isActive, } } getHistory(): ConversationMessage[] { return [...this._messages] } /** * 대화 세션 시작. 마이크 캡처를 시작하고 listening 상태로 진입. */ async startSession(): Promise { if (this._isActive) { throw new D3ROError(ErrorCode.ConversationSessionAlreadyActive, 'Conversation session already active') } // 라이센스 체크 try { const { getLicenseService } = await import('./LicenseService') const { Feature } = await import('@d3ro/core/types') const license = getLicenseService() const access = license.canUse(Feature.VOICE_CONVERSATION) if (!access.allowed) { license.promptUpgrade(Feature.VOICE_CONVERSATION, 'tier_required') throw new D3ROError(ErrorCode.FeatureNotAvailable, 'Pro+ required for voice conversation') } } catch (err) { if (err instanceof D3ROError) throw err } this._isActive = true this._setState('listening') this._startListening() // Whisper 모델 사전 로드 (fire-and-forget). // finishListening → transcribe 호출 시점에 모델이 Ready 상태여야 // 전사가 바로 실행됨. 이미 로드됐거나 로딩 중이면 내부 가드로 no-op. // (Bug 12: Voice Conversation이 Meeting/Caption 과 달리 loadModel을 // 명시적으로 호출하지 않아 첫 transcribe가 영원히 pending하던 문제) void getLocalSTTService() .initialize() .catch((err) => { logger.warn('STT 모델 사전 로드 실패 (transcribe 시 재시도):', err) }) logger.info('Voice conversation session started') } /** * 대화 세션 종료. */ stopSession(): void { if (!this._isActive) return this._stopListening() getTTSPlaybackService().stop() getLocalLLMService().cancelGeneration() this._isActive = false this._setState('idle') logger.info('Voice conversation session stopped') } /** * 텍스트 메시지를 직접 전송 (키보드 입력). */ async sendTextMessage(text: string): Promise { if (!this._isActive) { throw new D3ROError(ErrorCode.ConversationNoActiveSession, 'No active conversation session') } await this._processUserMessage(text) } /** * 현재 LLM 응답 또는 TTS 재생을 취소. */ cancelResponse(): void { getLocalLLMService().cancelGeneration() getTTSPlaybackService().stop() if (this._isActive) { this._setState('listening') this._startListening() } } /** * 대화 히스토리 초기화. */ clearHistory(): void { this._messages = [] this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.STATE_CHANGED, this.getSessionInfo()) } // ── 내부 로직 ── private _setState(state: ConversationState): void { this._state = state this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.STATE_CHANGED, this.getSessionInfo()) this.emit('state-changed', state) } private _startListening(): void { this._audioBuffers = [] const audioService = getAudioCaptureService() if (!this._audioListenerBound) { audioService.on('audio-data', this._onAudioData) this._audioListenerBound = true } if (!this._audioLevelListenerBound) { audioService.on('audio-level', this._onAudioLevel) this._audioLevelListenerBound = true } audioService.start().catch((err) => { logger.error('Failed to start audio capture for conversation:', err) this._emitError('stt', 'Failed to start microphone') }) // 녹음 시작 사운드 (fire-and-forget) getSoundEffectService().play('recording-start') } private _stopListening(): void { const audioService = getAudioCaptureService() if (this._audioListenerBound) { audioService.off('audio-data', this._onAudioData) this._audioListenerBound = false } if (this._audioLevelListenerBound) { audioService.off('audio-level', this._onAudioLevel) this._audioLevelListenerBound = false } audioService.stop().catch(() => { /* ignore */ }) this._audioBuffers = [] } private _onAudioData = (payload: { buffer: Buffer }): void => { if (this._state !== 'listening') return this._audioBuffers.push(payload.buffer) } /** * AudioCaptureService가 100ms 간격으로 emit하는 audio-level을 * listening 상태일 때만 렌더러로 forwarding. VoiceRecordingPanel에서 * 9바 waveform 애니메이션 입력으로 사용. */ private _onAudioLevel = (payload: { level: number; timestamp: number }): void => { if (this._state !== 'listening') return this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.AUDIO_LEVEL, { level: payload.level }) } /** * 녹음 완료 (UI에서 stop 버튼 클릭 시 호출). * 수집된 오디오를 STT로 전사 후 LLM 대화 진행. */ async finishListening(): Promise { if (this._state !== 'listening' || this._audioBuffers.length === 0) { return } // _stopListening()이 내부적으로 this._audioBuffers = []로 리셋하므로, // concat은 반드시 stop 호출 **전**에 끝내야 한다. (Bug 11) const audioBuffer = Buffer.concat(this._audioBuffers) this._stopListening() this._setState('thinking') // 녹음 종료 사운드 (fire-and-forget) getSoundEffectService().play('recording-stop') // 최소 오디오 길이 체크 (500ms @ 16kHz 16bit mono) // Bug 13: 너무 짧으면 조용히 listening 복귀 대신 에러 피드백. const minBytes = 16000 * 2 * 0.5 if (audioBuffer.length < minBytes) { this._emitError('stt', 'No speech detected. Please speak and try again.') this._setState('listening') this._startListening() return } try { // STT const sttService = getLocalSTTService() const language = (configGet('sttLanguage') as string | undefined) ?? 'auto' const result = await sttService.transcribe(audioBuffer, { language, vadFilter: true }) if (!result.text || result.text.trim().length === 0) { // Bug 13: VAD가 전체 오디오를 무음 판정한 경우에도 사용자 피드백. this._emitError('stt', 'No speech detected. Check microphone and try again.') this._setState('listening') this._startListening() return } await this._processUserMessage(result.text.trim()) } catch (err) { logger.error('STT failed in conversation:', err) this._emitError('stt', err instanceof Error ? err.message : 'STT failed') this._setState('listening') this._startListening() } } private async _processUserMessage(text: string): Promise { // 사용자 메시지 추가 const userMsg: ConversationMessage = { id: crypto.randomUUID(), role: 'user', content: text, timestamp: Date.now(), } this._messages.push(userMsg) this._trimHistory() this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.USER_MESSAGE, userMsg) this._setState('thinking') try { // Ollama /api/chat 호출 (스트리밍) const llmService = getLocalLLMService() const chatMessages = this._buildChatMessages() const assistantMsgId = crypto.randomUUID() let accumulated = '' const ttsSentences: string[] = [] let sentenceBuffer = '' const generator = llmService.chatStream(chatMessages) for await (const token of generator) { accumulated += token // 렌더러에 델타 전송 const delta: ConversationAssistantDelta = { messageId: assistantMsgId, delta: token, accumulated, } this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.ASSISTANT_DELTA, delta) // 문장 단위 TTS 큐잉 sentenceBuffer += token const sentenceEnd = sentenceBuffer.match(/[.!?。!?]\s*/g) if (sentenceEnd) { const lastEnd = sentenceBuffer.lastIndexOf(sentenceEnd[sentenceEnd.length - 1]) const completeSentence = sentenceBuffer.slice( 0, lastEnd + sentenceEnd[sentenceEnd.length - 1].length, ) sentenceBuffer = sentenceBuffer.slice( lastEnd + sentenceEnd[sentenceEnd.length - 1].length, ) if (completeSentence.trim()) { ttsSentences.push(completeSentence.trim()) } } } // 남은 텍스트도 TTS 큐에 추가 if (sentenceBuffer.trim()) { ttsSentences.push(sentenceBuffer.trim()) } // 어시스턴트 메시지 저장 const assistantMsg: ConversationMessage = { id: assistantMsgId, role: 'assistant', content: accumulated, timestamp: Date.now(), } this._messages.push(assistantMsg) this._trimHistory() const completeEvent: ConversationAssistantMessage = { messageId: assistantMsgId, content: accumulated, } this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.ASSISTANT_MESSAGE, completeEvent) // TTS 재생 if (ttsSentences.length > 0) { this._setState('speaking') this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.TTS_STARTED, {}) const ttsService = getTTSPlaybackService() await ttsService.speakSentences(ttsSentences) this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.TTS_FINISHED, {}) // 응답 완료 chime (자동 listening 재진입 직전) getSoundEffectService().play('chime') } // 재생 완료 → 다시 listening if (this._isActive) { this._setState('listening') this._startListening() } } catch (err) { logger.error('LLM chat failed in conversation:', err) this._emitError('llm', err instanceof Error ? err.message : 'LLM failed') if (this._isActive) { this._setState('listening') this._startListening() } } } private _buildChatMessages(): Array<{ role: string; content: string }> { const chatMsgs: Array<{ role: string; content: string }> = [ { role: 'system', content: SYSTEM_PROMPT }, ] for (const msg of this._messages) { if (msg.role === 'user' || msg.role === 'assistant') { chatMsgs.push({ role: msg.role, content: msg.content }) } } return chatMsgs } private _trimHistory(): void { // user+assistant 쌍 기준으로 최근 MAX_HISTORY_TURNS개만 유지 const pairs: ConversationMessage[] = [] let turnCount = 0 for (let i = this._messages.length - 1; i >= 0; i--) { pairs.unshift(this._messages[i]) if (this._messages[i].role === 'user') { turnCount++ if (turnCount >= MAX_HISTORY_TURNS) break } } this._messages = pairs } private _emitError(phase: ConversationError['phase'], message: string): void { const error: ConversationError = { message, phase } this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.ERROR, error) this.emit('error', error) // 에러 사운드 (fire-and-forget) getSoundEffectService().play('error') } private _sendToRenderer(channel: string, data: unknown): void { try { const mainWindow = getMainWindow() if (mainWindow && !mainWindow.isDestroyed()) { mainWindow.webContents.send(channel, data) } } catch { // 윈도우 없으면 무시 } } dispose(): void { this.stopSession() this.removeAllListeners() } } // ── 싱글톤 ── let instance: VoiceConversationService | null = null export function getVoiceConversationService(): VoiceConversationService { if (!instance) { instance = new VoiceConversationService() } return instance }