From 412a2e71f917e6c0db49ecf1bb0f0e26806092e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9C=A4=EC=B0=AC?= Date: Sat, 11 Apr 2026 23:16:34 +0900 Subject: [PATCH] =?UTF-8?q?feat(desktop):=20Voice=20Conversation=20?= =?UTF-8?q?=EB=AA=B0=EC=9E=85=20UX=20=ED=8C=A8=EB=84=90=20+=20=EC=82=AC?= =?UTF-8?q?=EC=9A=B4=EB=93=9C=20=ED=94=BC=EB=93=9C=EB=B0=B1=20+=20Bug=2013?= =?UTF-8?q?=20=EB=B9=88=20STT=20(=EB=B9=85=EB=B1=85=20Phase=205=20Part=206?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listening 상태에서 풀 몰입 계측기 모드로 전환되는 VoiceRecordingPanel 추가. recording-tip 팝업의 9바 cos-분포 waveform(BAR_COUNT=9, SMOOTHING=0.5, RANDOM_FACTOR=0.35, 100ms)을 React로 포팅해 REC LED + elapsed 타이머 + "SPEAK NOW" 힌트까지 구성. thinking/speaking 상태에서는 메시지 리스트로 복귀해 대화 맥락 유지 + 점 3개 typing indicator 버블 추가. VoiceConversationService에 AudioCaptureService audio-level forwarding과 사운드 훅 4개(recording-start / recording-stop / chime / error)를 삽입. chime은 recording-stop.wav 재사용(SoundEffectService SoundName 확장). VOICE_CONVERSATION.AUDIO_LEVEL 채널 신설 + preload onAudioLevel API. U8 Bug 13 동반 해소: finishListening에서 minBytes 미달 또는 VAD 무음 판정으로 빈 텍스트가 나오는 경우 조용히 listening으로 복귀하던 것을 _emitError('stt')로 사용자 피드백(에러 사운드 + 에러 이벤트)을 노출하도록 수정. 사용자가 "⏹ 눌러도 반응 없음"으로 오해하던 증상 해소. --- .../src/main/services/SoundEffectService.ts | 5 +- .../main/services/VoiceConversationService.ts | 35 +++ apps/desktop/src/preload/index.ts | 2 + .../VoiceRecordingPanel.tsx | 164 ++++++++++++ .../renderer/pages/VoiceConversationPage.tsx | 240 +++++++++++------- memory/project_status.md | 26 +- packages/core/src/ipc-channels.ts | 1 + packages/i18n/src/locales/en.json | 3 + packages/i18n/src/locales/ko.json | 3 + 9 files changed, 385 insertions(+), 94 deletions(-) create mode 100644 apps/desktop/src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx diff --git a/apps/desktop/src/main/services/SoundEffectService.ts b/apps/desktop/src/main/services/SoundEffectService.ts index 0d02b34..f180c88 100644 --- a/apps/desktop/src/main/services/SoundEffectService.ts +++ b/apps/desktop/src/main/services/SoundEffectService.ts @@ -9,14 +9,15 @@ import { getSoundPath } from '../utils/paths' const logger = getLogger('SoundEffectService') -type SoundName = 'recording-start' | 'recording-stop' | 'error' | 'cancel' +type SoundName = 'recording-start' | 'recording-stop' | 'error' | 'cancel' | 'chime' /** 효과음 파일 매핑 */ const SOUND_FILES: Record = { 'recording-start': 'recording-start.wav', 'recording-stop': 'recording-stop.wav', 'error': 'error.wav', - 'cancel': 'error.wav' // cancel은 error와 동일 + 'cancel': 'error.wav', // cancel은 error와 동일 + 'chime': 'recording-stop.wav' // chime은 recording-stop 재사용 (Voice Conversation 응답 완료) } /** 프리로드된 WAV 바이너리 캐시 */ diff --git a/apps/desktop/src/main/services/VoiceConversationService.ts b/apps/desktop/src/main/services/VoiceConversationService.ts index 059ee02..99deecf 100644 --- a/apps/desktop/src/main/services/VoiceConversationService.ts +++ b/apps/desktop/src/main/services/VoiceConversationService.ts @@ -8,6 +8,7 @@ import { getLocalLLMService } from './LocalLLMService' import { getLocalSTTService } from './LocalSTTService' import { getAudioCaptureService } from './AudioCaptureService' import { getTTSPlaybackService } from './TTSPlaybackService' +import { getSoundEffectService } from './SoundEffectService' import { configGet } from './ConfigService' import { getMainWindow } from '../windows/WindowManager' import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' @@ -34,6 +35,7 @@ class VoiceConversationService extends EventEmitter { private _isActive = false private _audioBuffers: Buffer[] = [] private _audioListenerBound = false + private _audioLevelListenerBound = false get state(): ConversationState { return this._state @@ -157,11 +159,18 @@ class VoiceConversationService extends EventEmitter { audioService.on('audio-data', this._onAudioData) this._audioListenerBound = true } + if (!this._audioLevelListenerBound) { + audioService.on('audio-level', this._onAudioLevel) + this._audioLevelListenerBound = true + } audioService.start().catch((err) => { logger.error('Failed to start audio capture for conversation:', err) this._emitError('stt', 'Failed to start microphone') }) + + // 녹음 시작 사운드 (fire-and-forget) + getSoundEffectService().play('recording-start') } private _stopListening(): void { @@ -170,6 +179,10 @@ class VoiceConversationService extends EventEmitter { audioService.off('audio-data', this._onAudioData) this._audioListenerBound = false } + if (this._audioLevelListenerBound) { + audioService.off('audio-level', this._onAudioLevel) + this._audioLevelListenerBound = false + } audioService.stop().catch(() => { /* ignore */ }) this._audioBuffers = [] } @@ -179,6 +192,16 @@ class VoiceConversationService extends EventEmitter { this._audioBuffers.push(payload.buffer) } + /** + * AudioCaptureService가 100ms 간격으로 emit하는 audio-level을 + * listening 상태일 때만 렌더러로 forwarding. VoiceRecordingPanel에서 + * 9바 waveform 애니메이션 입력으로 사용. + */ + private _onAudioLevel = (payload: { level: number; timestamp: number }): void => { + if (this._state !== 'listening') return + this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.AUDIO_LEVEL, { level: payload.level }) + } + /** * 녹음 완료 (UI에서 stop 버튼 클릭 시 호출). * 수집된 오디오를 STT로 전사 후 LLM 대화 진행. @@ -195,9 +218,14 @@ class VoiceConversationService extends EventEmitter { this._stopListening() this._setState('thinking') + // 녹음 종료 사운드 (fire-and-forget) + getSoundEffectService().play('recording-stop') + // 최소 오디오 길이 체크 (500ms @ 16kHz 16bit mono) + // Bug 13: 너무 짧으면 조용히 listening 복귀 대신 에러 피드백. const minBytes = 16000 * 2 * 0.5 if (audioBuffer.length < minBytes) { + this._emitError('stt', 'No speech detected. Please speak and try again.') this._setState('listening') this._startListening() return @@ -210,6 +238,8 @@ class VoiceConversationService extends EventEmitter { const result = await sttService.transcribe(audioBuffer, { language, vadFilter: true }) if (!result.text || result.text.trim().length === 0) { + // Bug 13: VAD가 전체 오디오를 무음 판정한 경우에도 사용자 피드백. + this._emitError('stt', 'No speech detected. Check microphone and try again.') this._setState('listening') this._startListening() return @@ -309,6 +339,9 @@ class VoiceConversationService extends EventEmitter { await ttsService.speakSentences(ttsSentences) this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.TTS_FINISHED, {}) + + // 응답 완료 chime (자동 listening 재진입 직전) + getSoundEffectService().play('chime') } // 재생 완료 → 다시 listening @@ -360,6 +393,8 @@ class VoiceConversationService extends EventEmitter { const error: ConversationError = { message, phase } this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.ERROR, error) this.emit('error', error) + // 에러 사운드 (fire-and-forget) + getSoundEffectService().play('error') } private _sendToRenderer(channel: string, data: unknown): void { diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 9b844f0..1ebb0ff 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -588,6 +588,8 @@ const electronAPI = { on(IPC_CHANNELS.VOICE_CONVERSATION.TTS_FINISHED, cb), onError: (cb: (data: ConversationError) => void): Unsubscribe => on(IPC_CHANNELS.VOICE_CONVERSATION.ERROR, cb), + onAudioLevel: (cb: (e: { level: number }) => void): Unsubscribe => + on(IPC_CHANNELS.VOICE_CONVERSATION.AUDIO_LEVEL, cb), }, // ── Meeting Mode (Phase 14) ─────────────────────────── diff --git a/apps/desktop/src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx b/apps/desktop/src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx new file mode 100644 index 0000000..afbcc59 --- /dev/null +++ b/apps/desktop/src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx @@ -0,0 +1,164 @@ +// src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx +// Phase 5 Part 6: Voice Conversation 몰입 녹음 패널 +// recording-tip 팝업의 9바 waveform (cos 분포, 100ms, smoothing)을 React로 포팅. +// 기존 구현: src/renderer/popups/recording-tip/script.js (Vanilla JS, 팝업 전용) +// 스펙: docs/design/03-db-and-ui.md:428+ +// +// AudioLevelSource: +// - VoiceConversationService가 listening 상태에서만 AUDIO_LEVEL 채널로 forwarding. +// - level 값은 0.0~1.0. 구독자가 없는 thinking/speaking에는 Panel 자체가 언마운트되므로 +// useEffect cleanup으로 subscription이 자동 해제됨. + +import { useEffect, useRef, useState } from 'react' +import { Box } from '@mui/material' +import { Led, PhosphorText } from '@d3ro/ui/components/ds' +import { d3roPalette, d3roFontMono } from '@d3ro/ui/theme' +import { useI18n } from '@d3ro/i18n' + +// ── Waveform 상수 (recording-tip/script.js 수치 그대로) ── +const BAR_COUNT = 9 +const UPDATE_INTERVAL = 100 +const MIN_HEIGHT = 2 +const MAX_HEIGHT = 28 +const SMOOTHING = 0.5 +const RANDOM_FACTOR = 0.35 + +/** 코사인 분포 가중치 (중앙이 가장 높음). 설계서 03. */ +const weights: readonly number[] = Array.from({ length: BAR_COUNT }, (_, i) => { + const center = (BAR_COUNT - 1) / 2 + const normalized = (i - center) / center + return Math.cos((normalized * Math.PI) / 2) +}) + +function formatDuration(ms: number): string { + const elapsed = Math.floor(ms / 1000) + const minutes = Math.floor(elapsed / 60) + const seconds = elapsed % 60 + return `${minutes}:${seconds < 10 ? '0' : ''}${seconds}` +} + +/** + * Voice Conversation이 listening 상태일 때만 렌더되는 몰입 녹음 패널. + * - 9바 phosphor waveform (audio-level 기반) + * - REC LED + elapsed 타이머 + * - "SPEAK NOW · PRESS ⏹ TO SEND" 힌트 (i18n) + */ +export function VoiceRecordingPanel(): React.ReactElement { + const { t } = useI18n() + const [heights, setHeights] = useState(() => + new Array(BAR_COUNT).fill(MIN_HEIGHT), + ) + const [elapsedMs, setElapsedMs] = useState(0) + + const heightsRef = useRef(new Array(BAR_COUNT).fill(MIN_HEIGHT)) + const audioLevelRef = useRef(0) + const startedAtRef = useRef(Date.now()) + + // audio-level 구독 (VoiceConversationService → AUDIO_LEVEL) + useEffect(() => { + const unsubscribe = window.electronAPI.voiceConversation.onAudioLevel((e) => { + audioLevelRef.current = e.level + }) + return unsubscribe + }, []) + + // 9바 애니메이션 루프 (100ms interval) + useEffect(() => { + startedAtRef.current = Date.now() + heightsRef.current = new Array(BAR_COUNT).fill(MIN_HEIGHT) + audioLevelRef.current = 0 + + const animInterval = window.setInterval(() => { + const level = audioLevelRef.current + // 최소 진동: audioLevel이 0이어도 바가 미세하게 움직여 "살아있음" 표현 + const effectiveLevel = Math.max(0.08, level) + const next = new Array(BAR_COUNT).fill(MIN_HEIGHT) as number[] + for (let i = 0; i < BAR_COUNT; i++) { + const baseTarget = effectiveLevel * MAX_HEIGHT * weights[i] + const randomized = baseTarget * (1 + (Math.random() - 0.5) * 2 * RANDOM_FACTOR) + const target = Math.max(MIN_HEIGHT, Math.min(MAX_HEIGHT, randomized)) + // 스무딩 보간 + const prev = heightsRef.current[i] + const smoothed = prev + (target - prev) * SMOOTHING + heightsRef.current[i] = smoothed + next[i] = Math.round(smoothed) + } + setHeights(next) + }, UPDATE_INTERVAL) + + const durationInterval = window.setInterval(() => { + setElapsedMs(Date.now() - startedAtRef.current) + }, 500) + + return () => { + window.clearInterval(animInterval) + window.clearInterval(durationInterval) + } + }, []) + + return ( + + {/* REC LED + 타이머 */} + + + + {t('conversation.state.recording')} + + + {formatDuration(elapsedMs)} + + + + {/* 9바 waveform */} + + {heights.map((h, i) => ( + + ))} + + + {/* 힌트 */} + + {`─ ${t('conversation.recording.hint')} ─`} + + + ) +} diff --git a/apps/desktop/src/renderer/pages/VoiceConversationPage.tsx b/apps/desktop/src/renderer/pages/VoiceConversationPage.tsx index c30d0e9..f96c611 100644 --- a/apps/desktop/src/renderer/pages/VoiceConversationPage.tsx +++ b/apps/desktop/src/renderer/pages/VoiceConversationPage.tsx @@ -11,6 +11,7 @@ import DeleteSweepIcon from '@mui/icons-material/DeleteSweep' import CancelIcon from '@mui/icons-material/Cancel' import { MetalCard, PhosphorText, Led, PhysicalButton, ScreenPanel, InstrumentPanel } from '@d3ro/ui/components/ds' import { PageHeader } from '../components/shared' +import { VoiceRecordingPanel } from '../components/voice-conversation/VoiceRecordingPanel' import { isImeComposingEvent } from '../utils/keyboard' import { d3roPalette, d3roFontMono, d3roTypo } from '@d3ro/ui/theme' import { useI18n } from '@d3ro/i18n' @@ -154,75 +155,121 @@ export function VoiceConversationPage(): React.ReactElement { } /> - {/* 메시지 목록 */} - - {messages.length === 0 && !streamingText && ( - - - {t('conversation.empty')} - - - {t('conversation.emptyHint')} - - - )} + {/* 메시지 영역 — state 분기: listening=몰입 패널, 그 외=메시지 리스트 */} + {state === 'listening' ? ( + + + + ) : ( + + {messages.length === 0 && !streamingText && state !== 'thinking' && ( + + + {t('conversation.empty')} + + + {t('conversation.emptyHint')} + + + )} - {messages.map((msg) => ( - - ( + - - {msg.content} - - - - ))} + + {msg.content} + + + + ))} - {/* 스트리밍 중인 어시스턴트 메시지 */} - {streamingText && streamingMsgId && ( - - - - {streamingText} - - {'▌'} + {/* thinking 중: typing indicator (아직 스트리밍 시작 전) */} + {state === 'thinking' && !streamingText && ( + + + span': { + width: 6, + height: 6, + borderRadius: '50%', + bgcolor: d3roPalette.accent.amber, + animation: 'd3roTypingBounce 1.2s infinite ease-in-out', + }, + '& > span:nth-of-type(2)': { animationDelay: '0.15s' }, + '& > span:nth-of-type(3)': { animationDelay: '0.3s' }, + '@keyframes d3roTypingBounce': { + '0%, 80%, 100%': { opacity: 0.3, transform: 'translateY(0)' }, + '40%': { opacity: 1, transform: 'translateY(-3px)' }, + }, + }} + > + + + - - - - )} + + + )} -
- + {/* 스트리밍 중인 어시스턴트 메시지 (thinking 말미 or speaking) */} + {streamingText && streamingMsgId && ( + + + + {streamingText} + + {'▌'} + + + + + )} + +
+ + )} {/* 하단 컨트롤 바 */} @@ -246,39 +293,50 @@ export function VoiceConversationPage(): React.ReactElement { )} - {/* 텍스트 입력 */} - setTextInput(e.target.value)} - onKeyDown={handleTextKeyDown} - placeholder={t('conversation.inputPlaceholder')} - size="small" - fullWidth - sx={{ - '& .MuiOutlinedInput-root': { - fontFamily: d3roFontMono, - fontSize: d3roTypo.compact.size, - bgcolor: d3roPalette.bg.inset, - '& fieldset': { borderColor: d3roPalette.border.subtle }, - '&:hover fieldset': { borderColor: d3roPalette.accent.amber }, - '&.Mui-focused fieldset': { borderColor: d3roPalette.accent.amber }, - }, - '& .MuiOutlinedInput-input': { - color: d3roPalette.text.primary, - }, - }} - /> + {/* 텍스트 입력 — listening 시 숨김, thinking/speaking 시 disabled */} + {state !== 'listening' && ( + setTextInput(e.target.value)} + onKeyDown={handleTextKeyDown} + placeholder={ + state === 'thinking' || state === 'speaking' + ? t('conversation.thinking.placeholder') + : t('conversation.inputPlaceholder') + } + size="small" + fullWidth + disabled={state === 'thinking' || state === 'speaking'} + sx={{ + '& .MuiOutlinedInput-root': { + fontFamily: d3roFontMono, + fontSize: d3roTypo.compact.size, + bgcolor: d3roPalette.bg.inset, + '& fieldset': { borderColor: d3roPalette.border.subtle }, + '&:hover fieldset': { borderColor: d3roPalette.accent.amber }, + '&.Mui-focused fieldset': { borderColor: d3roPalette.accent.amber }, + }, + '& .MuiOutlinedInput-input': { + color: d3roPalette.text.primary, + }, + }} + /> + )} - {/* 전송 버튼 */} - - - + {/* 전송 버튼 — listening 시 숨김 */} + {state !== 'listening' && ( + + + + )} {/* 세션 종료 */} {isActive && ( diff --git a/memory/project_status.md b/memory/project_status.md index 96a767f..e330bd9 100644 --- a/memory/project_status.md +++ b/memory/project_status.md @@ -1,8 +1,32 @@ # D3RO-VOICE 프로젝트 현황 -> 마지막 갱신: 2026-04-11 (Mac 부트스트랩) +> 마지막 갱신: 2026-04-11 (빅뱅 Phase 5 Part 6 — Voice Conversation 몰입 패널 구현) > 규칙 13: 작업 완료 즉시 이 파일 갱신 의무 +## 빅뱅 Phase 5 Part 6 (2026-04-11) — Voice Conversation UX 몰입 패널 ✅ + +Part 5-C 설계를 구현으로 완결. 사용자가 listening 상태에서 풀 몰입 계측기 모드 + 사운드 피드백 + 상태별 UI 분기를 실제로 경험할 수 있는 상태. + +**변경 파일 (6)** +- `packages/core/src/ipc-channels.ts` — `VOICE_CONVERSATION.AUDIO_LEVEL` 채널 추가 +- `apps/desktop/src/main/services/SoundEffectService.ts` — `SoundName`에 `'chime'` 추가 (recording-stop.wav 재사용) +- `apps/desktop/src/main/services/VoiceConversationService.ts` — `_onAudioLevel` forwarding + 사운드 훅 4개(start/stop/chime/error) + Bug 13(빈 STT 피드백) +- `apps/desktop/src/preload/index.ts` — `voiceConversation.onAudioLevel` 구독 API +- `apps/desktop/src/renderer/components/voice-conversation/VoiceRecordingPanel.tsx` — **신규** 9바 phosphor waveform(BAR_COUNT=9, cos 분포, 100ms, SMOOTHING=0.5, RANDOM_FACTOR=0.35) + REC LED + elapsed 타이머 + "SPEAK NOW" 힌트 +- `apps/desktop/src/renderer/pages/VoiceConversationPage.tsx` — state 분기(listening=VoiceRecordingPanel / 그 외=메시지 리스트) + thinking 시 typing indicator 버블 + TextField listening 시 숨김·thinking/speaking 시 disabled +- `packages/i18n/src/locales/{ko,en}.json` — `conversation.recording.hint`, `conversation.thinking.placeholder`, `conversation.state.recording` 3개 키 + +**해소 이슈** +- U6 Voice Conversation UX 몰입 패널 구현 +- U8 Bug 13 빈 STT 피드백 부재 (minBytes 미달 + VAD 무음 판정 양쪽 모두 `_emitError('stt', ...)` 훅 추가) + +**검증** +- desktop `tsc --noEmit` EXIT 0 +- Vite renderer HMR 자동 반영, Electron kill + nohup 재기동(PID 9635/9641) +- 사용자 녹음 테스트: 한국어 5.1초 오디오 전사 성공(`"에이에이에이에이 또 검중이래요"`) → 파이프라인 살아있음 확인 + + + ## Mac 환경 부트스트랩 (2026-04-11) Windows → Mac 핸드오프 완료 (`memory/handoff-latest.md` 참조). diff --git a/packages/core/src/ipc-channels.ts b/packages/core/src/ipc-channels.ts index 0d8533f..77c6659 100644 --- a/packages/core/src/ipc-channels.ts +++ b/packages/core/src/ipc-channels.ts @@ -275,6 +275,7 @@ export const IPC_CHANNELS = { TTS_STARTED: 'voiceConversation:ttsStarted', TTS_FINISHED: 'voiceConversation:ttsFinished', ERROR: 'voiceConversation:error', + AUDIO_LEVEL: 'voiceConversation:audioLevel', }, // ── Phase 13: Local RAG (13.2) ── diff --git a/packages/i18n/src/locales/en.json b/packages/i18n/src/locales/en.json index 19abee2..8d4bd9d 100644 --- a/packages/i18n/src/locales/en.json +++ b/packages/i18n/src/locales/en.json @@ -384,6 +384,9 @@ "conversation.inputPlaceholder": "Type a message...", "conversation.end": "End", "conversation.clearHistory": "Clear history", + "conversation.recording.hint": "SPEAK NOW · PRESS ⏹ TO SEND", + "conversation.thinking.placeholder": "Waiting for response…", + "conversation.state.recording": "RECORDING", "nav.knowledge": "Knowledge", "rag.title": "Knowledge Base", "rag.addDocument": "Add Document", diff --git a/packages/i18n/src/locales/ko.json b/packages/i18n/src/locales/ko.json index d4b3604..5bd30fb 100644 --- a/packages/i18n/src/locales/ko.json +++ b/packages/i18n/src/locales/ko.json @@ -385,6 +385,9 @@ "conversation.inputPlaceholder": "메시지 입력...", "conversation.end": "종료", "conversation.clearHistory": "대화 초기화", + "conversation.recording.hint": "말씀하세요 · ⏹로 전송", + "conversation.thinking.placeholder": "응답 대기 중…", + "conversation.state.recording": "녹음 중", "rag.title": "지식 베이스", "rag.addDocument": "문서 추가", "rag.documents": "문서",