feat(conversation): OpenAI gpt-realtime-2.1 라이브 음성 대화 Premium 백엔드
- realtime-token Edge Function: 티어 검증 + realtime_session 쿼터 + ephemeral key 발급 - useRealtimeConversation 훅: WebRTC 직결 (마이크 → OpenAI, 오디오 자동 재생) - VoiceConversationPage: conversationBackend 분기 + 연결 실패 시 로컬 fallback - AppConfig.conversationBackend + SettingsModal 음성 대화 엔진 선택 - ErrorCode 799 ConversationRealtimeTokenFailed, i18n ko/en
This commit is contained in:
Yun Chan
parent
983c60cda2
commit
8f7d300b89
15 changed files with 762 additions and 40 deletions
|
|
@ -12,6 +12,7 @@ import CancelIcon from '@mui/icons-material/Cancel'
|
|||
import { MetalCard, PhosphorText, Led, PhysicalButton, ScreenPanel, InstrumentPanel } from '@d3ro/ui/components/ds'
|
||||
import { PageHeader } from '../components/shared'
|
||||
import { VoiceRecordingPanel } from '../components/voice-conversation/VoiceRecordingPanel'
|
||||
import { useRealtimeConversation } from '../hooks/useRealtimeConversation'
|
||||
import { isImeComposingEvent } from '../utils/keyboard'
|
||||
import { d3roPalette, d3roFontMono, d3roTypo } from '@d3ro/ui/theme'
|
||||
import { useI18n } from '@d3ro/i18n'
|
||||
|
|
@ -24,41 +25,68 @@ import type {
|
|||
|
||||
export function VoiceConversationPage(): React.ReactElement {
|
||||
const { t } = useI18n()
|
||||
const [state, setState] = useState<ConversationState>('idle')
|
||||
const [messages, setMessages] = useState<ConversationMessage[]>([])
|
||||
const [isActive, setIsActive] = useState(false)
|
||||
const [streamingText, setStreamingText] = useState('')
|
||||
const [streamingMsgId, setStreamingMsgId] = useState<string | null>(null)
|
||||
const [localState, setLocalState] = useState<ConversationState>('idle')
|
||||
const [localMessages, setLocalMessages] = useState<ConversationMessage[]>([])
|
||||
const [localActive, setLocalActive] = useState(false)
|
||||
const [localStreamingText, setLocalStreamingText] = useState('')
|
||||
const [localStreamingMsgId, setLocalStreamingMsgId] = useState<string | null>(null)
|
||||
const [textInput, setTextInput] = useState('')
|
||||
const [errorBanner, setErrorBanner] = useState<ConversationError | null>(null)
|
||||
const [backend, setBackend] = useState<'local' | 'realtime'>('local')
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
const realtime = useRealtimeConversation()
|
||||
const isRealtime = backend === 'realtime'
|
||||
|
||||
// 표시용 상태 — 백엔드에 따라 로컬 파이프라인(IPC) vs Realtime 훅
|
||||
const state = isRealtime ? realtime.conversationState : localState
|
||||
const messages = isRealtime ? realtime.messages : localMessages
|
||||
const isActive = isRealtime ? realtime.isActive : localActive
|
||||
const streamingText = isRealtime ? realtime.streamingText : localStreamingText
|
||||
const streamingMsgId = isRealtime ? realtime.streamingMsgId : localStreamingMsgId
|
||||
|
||||
// 설정에서 대화 백엔드 로드
|
||||
useEffect(() => {
|
||||
window.electronAPI.config.getAll().then((r) => {
|
||||
if (r.success && r.data.conversationBackend === 'realtime') {
|
||||
setBackend('realtime')
|
||||
}
|
||||
})
|
||||
}, [])
|
||||
|
||||
// Realtime 에러 → 배너
|
||||
useEffect(() => {
|
||||
if (realtime.error) {
|
||||
setErrorBanner({ phase: 'llm', message: realtime.error })
|
||||
}
|
||||
}, [realtime.error])
|
||||
|
||||
const scrollToBottom = useCallback(() => {
|
||||
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' })
|
||||
}, [])
|
||||
|
||||
// IPC 이벤트 구독
|
||||
// IPC 이벤트 구독 (로컬 파이프라인)
|
||||
useEffect(() => {
|
||||
const unsubState = window.electronAPI.voiceConversation.onStateChanged((info) => {
|
||||
setState(info.state)
|
||||
setMessages(info.messages)
|
||||
setIsActive(info.isActive)
|
||||
setLocalState(info.state)
|
||||
setLocalMessages(info.messages)
|
||||
setLocalActive(info.isActive)
|
||||
})
|
||||
const unsubUser = window.electronAPI.voiceConversation.onUserMessage((msg) => {
|
||||
setMessages((prev) => [...prev, msg])
|
||||
setStreamingText('')
|
||||
setStreamingMsgId(null)
|
||||
setLocalMessages((prev) => [...prev, msg])
|
||||
setLocalStreamingText('')
|
||||
setLocalStreamingMsgId(null)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubDelta = window.electronAPI.voiceConversation.onAssistantDelta((data: ConversationAssistantDelta) => {
|
||||
setStreamingMsgId(data.messageId)
|
||||
setStreamingText(data.accumulated)
|
||||
setLocalStreamingMsgId(data.messageId)
|
||||
setLocalStreamingText(data.accumulated)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubComplete = window.electronAPI.voiceConversation.onAssistantMessage((msg) => {
|
||||
setMessages((prev) => [...prev, { id: msg.messageId, role: 'assistant', content: msg.content, timestamp: Date.now() }])
|
||||
setStreamingText('')
|
||||
setStreamingMsgId(null)
|
||||
setLocalMessages((prev) => [...prev, { id: msg.messageId, role: 'assistant', content: msg.content, timestamp: Date.now() }])
|
||||
setLocalStreamingText('')
|
||||
setLocalStreamingMsgId(null)
|
||||
setTimeout(scrollToBottom, 50)
|
||||
})
|
||||
const unsubError = window.electronAPI.voiceConversation.onError((err: ConversationError) => {
|
||||
|
|
@ -69,9 +97,9 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
// 초기 상태 로드
|
||||
window.electronAPI.voiceConversation.getState().then((r) => {
|
||||
if (r.success) {
|
||||
setState(r.data.state)
|
||||
setMessages(r.data.messages)
|
||||
setIsActive(r.data.isActive)
|
||||
setLocalState(r.data.state)
|
||||
setLocalMessages(r.data.messages)
|
||||
setLocalActive(r.data.isActive)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -84,36 +112,78 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
}
|
||||
}, [scrollToBottom])
|
||||
|
||||
// Realtime 메시지 갱신 시 스크롤
|
||||
useEffect(() => {
|
||||
if (isRealtime) setTimeout(scrollToBottom, 50)
|
||||
}, [isRealtime, realtime.messages, realtime.streamingText, scrollToBottom])
|
||||
|
||||
const handleStartSession = useCallback(async () => {
|
||||
if (isRealtime) {
|
||||
const ok = await realtime.start()
|
||||
if (!ok) {
|
||||
// Realtime 연결 실패 → 로컬 파이프라인 자동 fallback
|
||||
setBackend('local')
|
||||
setErrorBanner({ phase: 'llm', message: t('conversation.realtime.fallback') })
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
}
|
||||
return
|
||||
}
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
}, [])
|
||||
}, [isRealtime, realtime, t])
|
||||
|
||||
const handleStopSession = useCallback(async () => {
|
||||
if (isRealtime) {
|
||||
realtime.stop()
|
||||
return
|
||||
}
|
||||
await window.electronAPI.voiceConversation.stopSession()
|
||||
}, [])
|
||||
}, [isRealtime, realtime])
|
||||
|
||||
const handleFinishListening = useCallback(async () => {
|
||||
if (isRealtime) return // server VAD가 자동 처리
|
||||
await window.electronAPI.voiceConversation.finishListening()
|
||||
}, [])
|
||||
}, [isRealtime])
|
||||
|
||||
const handleSendText = useCallback(async () => {
|
||||
if (!textInput.trim()) return
|
||||
const text = textInput.trim()
|
||||
setTextInput('')
|
||||
if (isRealtime) {
|
||||
if (!realtime.isActive) {
|
||||
const ok = await realtime.start()
|
||||
if (!ok) {
|
||||
setBackend('local')
|
||||
setErrorBanner({ phase: 'llm', message: t('conversation.realtime.fallback') })
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
await window.electronAPI.voiceConversation.sendMessage({ text })
|
||||
return
|
||||
}
|
||||
}
|
||||
realtime.sendText(text)
|
||||
return
|
||||
}
|
||||
if (!isActive) {
|
||||
await window.electronAPI.voiceConversation.startSession()
|
||||
}
|
||||
await window.electronAPI.voiceConversation.sendMessage({ text })
|
||||
}, [textInput, isActive])
|
||||
}, [textInput, isActive, isRealtime, realtime, t])
|
||||
|
||||
const handleClearHistory = useCallback(async () => {
|
||||
if (isRealtime) {
|
||||
realtime.clearMessages()
|
||||
return
|
||||
}
|
||||
await window.electronAPI.voiceConversation.clearHistory()
|
||||
setMessages([])
|
||||
}, [])
|
||||
setLocalMessages([])
|
||||
}, [isRealtime, realtime])
|
||||
|
||||
const handleCancelResponse = useCallback(async () => {
|
||||
if (isRealtime) {
|
||||
realtime.cancelResponse()
|
||||
return
|
||||
}
|
||||
await window.electronAPI.voiceConversation.cancelResponse()
|
||||
}, [])
|
||||
}, [isRealtime, realtime])
|
||||
|
||||
const handleTextKeyDown = useCallback((e: React.KeyboardEvent) => {
|
||||
if (isImeComposingEvent(e)) return
|
||||
|
|
@ -130,6 +200,11 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
speaking: t('conversation.speaking'),
|
||||
}
|
||||
|
||||
const displayStateLabel =
|
||||
isRealtime && realtime.connectionState === 'connecting'
|
||||
? t('conversation.realtime.connecting')
|
||||
: stateLabel[state]
|
||||
|
||||
const formatErrorMessage = useCallback((err: ConversationError): string => {
|
||||
// Bug 13의 전형적 STT 메시지는 i18n 키로 매핑 — 그 외는 서비스 원문 + phase label.
|
||||
if (err.phase === 'stt' && err.message.toLowerCase().startsWith('no speech')) {
|
||||
|
|
@ -152,9 +227,23 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
title={t('conversation.title').toUpperCase()}
|
||||
action={
|
||||
<Box sx={{ display: 'flex', gap: 1, alignItems: 'center' }}>
|
||||
{isRealtime && (
|
||||
<PhosphorText
|
||||
variant="micro"
|
||||
sx={{
|
||||
color: d3roPalette.accent.amber,
|
||||
border: `1px solid ${d3roPalette.accent.amber}`,
|
||||
borderRadius: '3px',
|
||||
px: 0.5,
|
||||
lineHeight: 1.6,
|
||||
}}
|
||||
>
|
||||
{t('conversation.realtime.badge')}
|
||||
</PhosphorText>
|
||||
)}
|
||||
<Led color={stateLedColor[state]} pulse={state === 'listening' || state === 'thinking'} size={8} />
|
||||
<PhosphorText variant="label" sx={{ color: d3roPalette.text.dimLabel }}>
|
||||
{stateLabel[state].toUpperCase()}
|
||||
{displayStateLabel.toUpperCase()}
|
||||
</PhosphorText>
|
||||
{messages.length > 0 && (
|
||||
<Tooltip title={t('conversation.clearHistory')}>
|
||||
|
|
@ -167,8 +256,8 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
}
|
||||
/>
|
||||
|
||||
{/* 메시지 영역 — state 분기: listening=몰입 패널, 그 외=메시지 리스트 */}
|
||||
{state === 'listening' ? (
|
||||
{/* 메시지 영역 — 로컬 파이프라인의 listening만 몰입 패널 (Realtime은 항상 채팅 뷰) */}
|
||||
{!isRealtime && state === 'listening' ? (
|
||||
<Box
|
||||
sx={{
|
||||
flex: 1,
|
||||
|
|
@ -292,7 +381,11 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
<MicIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
) : state === 'listening' ? (
|
||||
<PhysicalButton selected onClick={handleFinishListening} sx={{ minWidth: 48, px: 2 }}>
|
||||
<PhysicalButton
|
||||
selected
|
||||
onClick={isRealtime ? handleStopSession : handleFinishListening}
|
||||
sx={{ minWidth: 48, px: 2 }}
|
||||
>
|
||||
<StopIcon sx={{ fontSize: 20 }} />
|
||||
</PhysicalButton>
|
||||
) : state === 'thinking' || state === 'speaking' ? (
|
||||
|
|
@ -305,8 +398,8 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
</PhysicalButton>
|
||||
)}
|
||||
|
||||
{/* 텍스트 입력 — listening 시 숨김, thinking/speaking 시 disabled */}
|
||||
{state !== 'listening' && (
|
||||
{/* 텍스트 입력 — 로컬 listening 시 숨김 (Realtime은 항상 표시), thinking/speaking 시 disabled */}
|
||||
{(isRealtime || state !== 'listening') && (
|
||||
<TextField
|
||||
value={state === 'thinking' || state === 'speaking' ? '' : textInput}
|
||||
onChange={(e) => setTextInput(e.target.value)}
|
||||
|
|
@ -335,8 +428,8 @@ export function VoiceConversationPage(): React.ReactElement {
|
|||
/>
|
||||
)}
|
||||
|
||||
{/* 전송 버튼 — listening 시 숨김 */}
|
||||
{state !== 'listening' && (
|
||||
{/* 전송 버튼 — 로컬 listening 시 숨김 */}
|
||||
{(isRealtime || state !== 'listening') && (
|
||||
<IconButton
|
||||
onClick={handleSendText}
|
||||
disabled={!textInput.trim() || state === 'thinking' || state === 'speaking'}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue