Phase 12~13 전체 구현: Pro+ 피처 6종 + 음성 대화 + RAG + OS 자동화
Phase 12: - FileTranscriptionService: ffmpeg PCM 변환 + 30초 청크 순차 STT - MeetingSummaryService: 자막 세션 → LLM 자동 요약 + DB summaryText - DictationTemplateService: 필드별 음성 입력 상태 머신 + 프리셋 3개 Phase 13.1: - VoiceConversationService: STT→Ollama /api/chat→TTS 대화 루프 (10턴) - TTSPlaybackService: Windows SAPI 문장 단위 큐 재생 - LocalLLMService.chatStream: Ollama /api/chat 스트리밍 Phase 13.2: - RAGService: Ollama 임베딩 + SQLite 벡터 + 코사인 유사도 검색 - KnowledgeBasePage: 문서 관리 + 질문/답변 UI - PDF 파서: zlib FlateDecode 해제 + BT/ET 텍스트 추출 Phase 13.3: - VoiceActionService: LLM JSON 액션 플랜 생성 + 실행 - 프리셋 6개 (크롬/메모장/탐색기/볼륨), 위험 명령 차단 공통: IPC ~70채널, 에러코드 780-878, i18n 100+키 버그픽스: 라이선스 로컬 키 우선, i18n featureLabel, DOM 중첩
This commit is contained in:
Yun Chan
parent
a31f96bbb8
commit
eb83682269
38 changed files with 5678 additions and 19 deletions
291
src/renderer/pages/VoiceConversationPage.tsx
Normal file
291
src/renderer/pages/VoiceConversationPage.tsx
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
// src/renderer/pages/VoiceConversationPage.tsx
|
||||
// Phase 13.1: 음성 대화 모드 UI
|
||||
// STT→LLM→TTS 대화 루프. 채팅 메시지 목록 + 녹음 버튼.
|
||||
|
||||
import { useState, useEffect, useCallback, useRef } from 'react'
|
||||
import { Box, IconButton, TextField, Tooltip } from '@mui/material'
|
||||
import MicIcon from '@mui/icons-material/Mic'
|
||||
import StopIcon from '@mui/icons-material/Stop'
|
||||
import SendIcon from '@mui/icons-material/Send'
|
||||
import DeleteSweepIcon from '@mui/icons-material/DeleteSweep'
|
||||
import CancelIcon from '@mui/icons-material/Cancel'
|
||||
import { MetalCard, PhosphorText, Led, PhysicalButton, ScreenPanel, InstrumentPanel } from '../components/ds'
|
||||
import { PageHeader } from '../components/shared'
|
||||
import { d3roPalette, d3roFontMono, d3roTypo } from '../theme'
|
||||
import { useI18n } from '../i18n'
|
||||
import type {
|
||||
ConversationState,
|
||||
ConversationMessage,
|
||||
ConversationAssistantDelta,
|
||||
} from '@shared/types'
|
||||
|
||||
export function VoiceConversationPage(): React.ReactElement {
|
||||
const { t } = useI18n()
|
||||
const [state, setState] = useState<ConversationState>('idle')
|
||||
const [messages, setMessages] = useState<ConversationMessage[]>([])
|
||||
const [isActive, setIsActive] = useState(false)
|
||||
const [streamingText, setStreamingText] = useState('')
|
||||
const [streamingMsgId, setStreamingMsgId] = useState<string | null>(null)
|
||||
const [textInput, setTextInput] = useState('')
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
const scrollToBottom = useCallback(() => {
|
||||
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' })
|
||||
}, [])
|
||||
|
||||
// IPC 이벤트 구독
|
||||
useEffect(() => {
|
||||
const unsubState = window.electronAPI.voiceConversation.onStateChanged((info) => {
|
||||
setState(info.state)
|
||||
setMessages(info.messages)
|
||||
setIsActive(info.isActive)
|
||||
})
|
||||
const unsubUser = window.electronAPI.voiceConversation.onUserMessage((msg) => {
|
||||
setMessages((prev) => [...prev, msg])
|
||||
setStreamingText('')
|
||||
setStreamingMsgId(null)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubDelta = window.electronAPI.voiceConversation.onAssistantDelta((data: ConversationAssistantDelta) => {
|
||||
setStreamingMsgId(data.messageId)
|
||||
setStreamingText(data.accumulated)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubComplete = window.electronAPI.voiceConversation.onAssistantMessage((msg) => {
|
||||
setMessages((prev) => [...prev, { id: msg.messageId, role: 'assistant', content: msg.content, timestamp: Date.now() }])
|
||||
setStreamingText('')
|
||||
setStreamingMsgId(null)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubError = window.electronAPI.voiceConversation.onError(() => {
|
||||
// 에러 시 자동 복구 (서비스에서 listening으로 전환)
|
||||
})
|
||||
|
||||
// 초기 상태 로드
|
||||
window.electronAPI.voiceConversation.getState().then((r) => {
|
||||
if (r.success) {
|
||||
setState(r.data.state)
|
||||
setMessages(r.data.messages)
|
||||
setIsActive(r.data.isActive)
|
||||
}
|
||||
})
|
||||
|
||||
return () => {
|
||||
unsubState()
|
||||
unsubUser()
|
||||
unsubDelta()
|
||||
unsubComplete()
|
||||
unsubError()
|
||||
}
|
||||
}, [scrollToBottom])
|
||||
|
||||
const handleStartSession = useCallback(async () => {
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
}, [])
|
||||
|
||||
const handleStopSession = useCallback(async () => {
|
||||
await window.electronAPI.voiceConversation.stopSession()
|
||||
}, [])
|
||||
|
||||
const handleFinishListening = useCallback(async () => {
|
||||
await window.electronAPI.voiceConversation.finishListening()
|
||||
}, [])
|
||||
|
||||
const handleSendText = useCallback(async () => {
|
||||
if (!textInput.trim()) return
|
||||
const text = textInput.trim()
|
||||
setTextInput('')
|
||||
if (!isActive) {
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
}
|
||||
await window.electronAPI.voiceConversation.sendMessage({ text })
|
||||
}, [textInput, isActive])
|
||||
|
||||
const handleClearHistory = useCallback(async () => {
|
||||
await window.electronAPI.voiceConversation.clearHistory()
|
||||
setMessages([])
|
||||
}, [])
|
||||
|
||||
const handleCancelResponse = useCallback(async () => {
|
||||
await window.electronAPI.voiceConversation.cancelResponse()
|
||||
}, [])
|
||||
|
||||
const handleTextKeyDown = useCallback((e: React.KeyboardEvent) => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) {
|
||||
e.preventDefault()
|
||||
handleSendText()
|
||||
}
|
||||
}, [handleSendText])
|
||||
|
||||
const stateLabel = {
|
||||
idle: t('conversation.idle'),
|
||||
listening: t('conversation.listening'),
|
||||
thinking: t('conversation.thinking'),
|
||||
speaking: t('conversation.speaking'),
|
||||
}
|
||||
|
||||
const stateLedColor = {
|
||||
idle: 'amber' as const,
|
||||
listening: 'red' as const,
|
||||
thinking: 'amber' as const,
|
||||
speaking: 'green' as const,
|
||||
}
|
||||
|
||||
return (
|
||||
<Box sx={{ maxWidth: 800, mx: 'auto', p: 4, pb: 8, display: 'flex', flexDirection: 'column', height: '100%' }}>
|
||||
<PageHeader
|
||||
title={t('conversation.title').toUpperCase()}
|
||||
action={
|
||||
<Box sx={{ display: 'flex', gap: 1, alignItems: 'center' }}>
|
||||
<Led color={stateLedColor[state]} pulse={state === 'listening' || state === 'thinking'} size={8} />
|
||||
<PhosphorText variant="label" sx={{ color: d3roPalette.text.dimLabel }}>
|
||||
{stateLabel[state].toUpperCase()}
|
||||
</PhosphorText>
|
||||
{messages.length > 0 && (
|
||||
<Tooltip title={t('conversation.clearHistory')}>
|
||||
<IconButton size="small" onClick={handleClearHistory} sx={{ color: d3roPalette.text.inactive }}>
|
||||
<DeleteSweepIcon sx={{ fontSize: 18 }} />
|
||||
</IconButton>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Box>
|
||||
}
|
||||
/>
|
||||
|
||||
{/* 메시지 목록 */}
|
||||
<Box
|
||||
sx={{
|
||||
flex: 1,
|
||||
overflow: 'auto',
|
||||
mt: 2,
|
||||
mb: 2,
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 1.5,
|
||||
}}
|
||||
>
|
||||
{messages.length === 0 && !streamingText && (
|
||||
<Box sx={{ textAlign: 'center', mt: 8 }}>
|
||||
<PhosphorText variant="heading" sx={{ color: d3roPalette.text.inactive, mb: 1 }}>
|
||||
{t('conversation.empty')}
|
||||
</PhosphorText>
|
||||
<PhosphorText variant="dim">
|
||||
{t('conversation.emptyHint')}
|
||||
</PhosphorText>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{messages.map((msg) => (
|
||||
<Box
|
||||
key={msg.id}
|
||||
sx={{
|
||||
display: 'flex',
|
||||
justifyContent: msg.role === 'user' ? 'flex-end' : 'flex-start',
|
||||
}}
|
||||
>
|
||||
<MetalCard
|
||||
sx={{
|
||||
maxWidth: '75%',
|
||||
...(msg.role === 'user' && {
|
||||
bgcolor: d3roPalette.accent.amber,
|
||||
'& *': { color: `${d3roPalette.bg.chassis} !important` },
|
||||
}),
|
||||
}}
|
||||
>
|
||||
<PhosphorText
|
||||
variant="compact"
|
||||
sx={{
|
||||
whiteSpace: 'pre-wrap',
|
||||
lineHeight: 1.6,
|
||||
}}
|
||||
>
|
||||
{msg.content}
|
||||
</PhosphorText>
|
||||
</MetalCard>
|
||||
</Box>
|
||||
))}
|
||||
|
||||
{/* 스트리밍 중인 어시스턴트 메시지 */}
|
||||
{streamingText && streamingMsgId && (
|
||||
<Box sx={{ display: 'flex', justifyContent: 'flex-start' }}>
|
||||
<MetalCard sx={{ maxWidth: '75%' }}>
|
||||
<PhosphorText variant="compact" sx={{ whiteSpace: 'pre-wrap', lineHeight: 1.6 }}>
|
||||
{streamingText}
|
||||
<Box component="span" sx={{ animation: 'blink 1s infinite', color: d3roPalette.accent.amber }}>
|
||||
{'▌'}
|
||||
</Box>
|
||||
</PhosphorText>
|
||||
</MetalCard>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<div ref={messagesEndRef} />
|
||||
</Box>
|
||||
|
||||
{/* 하단 컨트롤 바 */}
|
||||
<MetalCard>
|
||||
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
|
||||
{/* 녹음 버튼 */}
|
||||
{!isActive ? (
|
||||
<PhysicalButton onClick={handleStartSession} sx={{ minWidth: 48, px: 2 }}>
|
||||
<MicIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
) : state === 'listening' ? (
|
||||
<PhysicalButton selected onClick={handleFinishListening} sx={{ minWidth: 48, px: 2 }}>
|
||||
<StopIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
) : state === 'thinking' || state === 'speaking' ? (
|
||||
<PhysicalButton onClick={handleCancelResponse} sx={{ minWidth: 48, px: 2 }}>
|
||||
<CancelIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
) : (
|
||||
<PhysicalButton onClick={handleFinishListening} sx={{ minWidth: 48, px: 2 }}>
|
||||
<MicIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
)}
|
||||
|
||||
{/* 텍스트 입력 */}
|
||||
<TextField
|
||||
value={textInput}
|
||||
onChange={(e) => setTextInput(e.target.value)}
|
||||
onKeyDown={handleTextKeyDown}
|
||||
placeholder={t('conversation.inputPlaceholder')}
|
||||
size="small"
|
||||
fullWidth
|
||||
sx={{
|
||||
'& .MuiOutlinedInput-root': {
|
||||
fontFamily: d3roFontMono,
|
||||
fontSize: d3roTypo.compact.size,
|
||||
bgcolor: d3roPalette.bg.inset,
|
||||
'& fieldset': { borderColor: d3roPalette.border.subtle },
|
||||
'&:hover fieldset': { borderColor: d3roPalette.accent.amber },
|
||||
'&.Mui-focused fieldset': { borderColor: d3roPalette.accent.amber },
|
||||
},
|
||||
'& .MuiOutlinedInput-input': {
|
||||
color: d3roPalette.text.primary,
|
||||
},
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* 전송 버튼 */}
|
||||
<IconButton
|
||||
onClick={handleSendText}
|
||||
disabled={!textInput.trim()}
|
||||
sx={{
|
||||
color: textInput.trim() ? d3roPalette.accent.amber : d3roPalette.text.inactive,
|
||||
}}
|
||||
>
|
||||
<SendIcon sx={{ fontSize: 20 }} />
|
||||
</IconButton>
|
||||
|
||||
{/* 세션 종료 */}
|
||||
{isActive && (
|
||||
<PhysicalButton onClick={handleStopSession} sx={{ minWidth: 48, px: 1 }}>
|
||||
<PhosphorText variant="micro">{t('conversation.end')}</PhosphorText>
|
||||
</PhysicalButton>
|
||||
)}
|
||||
</Box>
|
||||
</MetalCard>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue