feat(conversation): OpenAI gpt-realtime-2.1 라이브 음성 대화 Premium 백엔드
- realtime-token Edge Function: 티어 검증 + realtime_session 쿼터 + ephemeral key 발급 - useRealtimeConversation 훅: WebRTC 직결 (마이크 → OpenAI, 오디오 자동 재생) - VoiceConversationPage: conversationBackend 분기 + 연결 실패 시 로컬 fallback - AppConfig.conversationBackend + SettingsModal 음성 대화 엔진 선택 - ErrorCode 799 ConversationRealtimeTokenFailed, i18n ko/en
This commit is contained in:
Yun Chan
parent
983c60cda2
commit
8f7d300b89
15 changed files with 762 additions and 40 deletions
306
apps/desktop/src/renderer/hooks/useRealtimeConversation.ts
Normal file
306
apps/desktop/src/renderer/hooks/useRealtimeConversation.ts
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
// src/renderer/hooks/useRealtimeConversation.ts
|
||||
// OpenAI Realtime API (gpt-realtime-2.1) 라이브 음성 대화 훅.
|
||||
// Supabase realtime-token(IPC 경유)으로 ephemeral key를 받아
|
||||
// 렌더러가 OpenAI와 직접 WebRTC 연결한다 (마이크 캡처 + 오디오 재생 자동).
|
||||
//
|
||||
// 이벤트 매핑 (data channel 'oai-events'):
|
||||
// response.created → thinking
|
||||
// response.(output_)audio_transcript.delta → speaking + 스트리밍 텍스트
|
||||
// conversation.item.input_audio_transcription.completed → 유저 메시지
|
||||
// response.done → 어시스턴트 메시지 확정 + listening 복귀
|
||||
|
||||
import { useState, useRef, useCallback, useEffect } from 'react'
|
||||
import type { ConversationState, ConversationMessage } from '@d3ro/core/types'
|
||||
|
||||
const REALTIME_CALLS_URL = 'https://api.openai.com/v1/realtime/calls'
|
||||
const DATA_CHANNEL_NAME = 'oai-events'
|
||||
|
||||
export type RealtimeConnectionState = 'idle' | 'connecting' | 'live' | 'error'
|
||||
|
||||
interface RealtimeServerEvent {
|
||||
type: string
|
||||
delta?: string
|
||||
transcript?: string
|
||||
response?: { id?: string }
|
||||
item_id?: string
|
||||
error?: { message?: string }
|
||||
}
|
||||
|
||||
export interface UseRealtimeConversationResult {
|
||||
connectionState: RealtimeConnectionState
|
||||
conversationState: ConversationState
|
||||
messages: ConversationMessage[]
|
||||
isActive: boolean
|
||||
streamingText: string
|
||||
streamingMsgId: string | null
|
||||
error: string | null
|
||||
/** 연결 시작. 실패 시 false (로컬 파이프라인 fallback 유도) */
|
||||
start: () => Promise<boolean>
|
||||
stop: () => void
|
||||
sendText: (text: string) => void
|
||||
cancelResponse: () => void
|
||||
clearMessages: () => void
|
||||
}
|
||||
|
||||
let messageSeq = 0
|
||||
function nextMessageId(prefix: string): string {
|
||||
messageSeq += 1
|
||||
return `rt-${prefix}-${Date.now()}-${messageSeq}`
|
||||
}
|
||||
|
||||
export function useRealtimeConversation(): UseRealtimeConversationResult {
|
||||
const [connectionState, setConnectionState] = useState<RealtimeConnectionState>('idle')
|
||||
const [conversationState, setConversationState] = useState<ConversationState>('idle')
|
||||
const [messages, setMessages] = useState<ConversationMessage[]>([])
|
||||
const [streamingText, setStreamingText] = useState('')
|
||||
const [streamingMsgId, setStreamingMsgId] = useState<string | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const pcRef = useRef<RTCPeerConnection | null>(null)
|
||||
const dcRef = useRef<RTCDataChannel | null>(null)
|
||||
const micStreamRef = useRef<MediaStream | null>(null)
|
||||
const audioElRef = useRef<HTMLAudioElement | null>(null)
|
||||
const assistantBufferRef = useRef('')
|
||||
const assistantMsgIdRef = useRef<string | null>(null)
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
dcRef.current?.close()
|
||||
dcRef.current = null
|
||||
pcRef.current?.close()
|
||||
pcRef.current = null
|
||||
micStreamRef.current?.getTracks().forEach((track) => track.stop())
|
||||
micStreamRef.current = null
|
||||
if (audioElRef.current) {
|
||||
audioElRef.current.srcObject = null
|
||||
audioElRef.current.remove()
|
||||
audioElRef.current = null
|
||||
}
|
||||
assistantBufferRef.current = ''
|
||||
assistantMsgIdRef.current = null
|
||||
}, [])
|
||||
|
||||
// 언마운트 시 연결 정리
|
||||
useEffect(() => cleanup, [cleanup])
|
||||
|
||||
const flushAssistantMessage = useCallback(() => {
|
||||
const content = assistantBufferRef.current.trim()
|
||||
const msgId = assistantMsgIdRef.current
|
||||
assistantBufferRef.current = ''
|
||||
assistantMsgIdRef.current = null
|
||||
setStreamingText('')
|
||||
setStreamingMsgId(null)
|
||||
if (content && msgId) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ id: msgId, role: 'assistant', content, timestamp: Date.now() },
|
||||
])
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleServerEvent = useCallback(
|
||||
(event: RealtimeServerEvent) => {
|
||||
switch (event.type) {
|
||||
case 'response.created': {
|
||||
assistantBufferRef.current = ''
|
||||
assistantMsgIdRef.current = nextMessageId('assistant')
|
||||
setConversationState('thinking')
|
||||
break
|
||||
}
|
||||
|
||||
// GA/신규 이벤트명 모두 수용
|
||||
case 'response.output_audio_transcript.delta':
|
||||
case 'response.audio_transcript.delta':
|
||||
case 'response.output_text.delta':
|
||||
case 'response.text.delta': {
|
||||
if (event.delta) {
|
||||
assistantBufferRef.current += event.delta
|
||||
setStreamingMsgId(assistantMsgIdRef.current)
|
||||
setStreamingText(assistantBufferRef.current)
|
||||
setConversationState('speaking')
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
case 'conversation.item.input_audio_transcription.completed': {
|
||||
const transcript = (event.transcript ?? '').trim()
|
||||
if (transcript) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: nextMessageId('user'),
|
||||
role: 'user',
|
||||
content: transcript,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
])
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
case 'response.done': {
|
||||
flushAssistantMessage()
|
||||
setConversationState('listening')
|
||||
break
|
||||
}
|
||||
|
||||
case 'error': {
|
||||
setError(event.error?.message ?? 'Realtime error')
|
||||
break
|
||||
}
|
||||
|
||||
default:
|
||||
break
|
||||
}
|
||||
},
|
||||
[flushAssistantMessage],
|
||||
)
|
||||
|
||||
const start = useCallback(async (): Promise<boolean> => {
|
||||
if (pcRef.current) return true
|
||||
|
||||
setError(null)
|
||||
setConnectionState('connecting')
|
||||
|
||||
try {
|
||||
// 1) ephemeral token (Supabase realtime-token → OpenAI client_secrets)
|
||||
const tokenResult = await window.electronAPI.voiceConversation.getRealtimeToken({})
|
||||
if (!tokenResult.success) {
|
||||
throw new Error(tokenResult.error?.message ?? 'token request failed')
|
||||
}
|
||||
const { value: ephemeralKey, model } = tokenResult.data
|
||||
|
||||
// 2) 마이크 + peer connection
|
||||
const micStream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
micStreamRef.current = micStream
|
||||
|
||||
const pc = new RTCPeerConnection()
|
||||
pcRef.current = pc
|
||||
const micTrack = micStream.getAudioTracks()[0]
|
||||
if (micTrack) pc.addTrack(micTrack, micStream)
|
||||
|
||||
// 3) 원격 오디오 재생
|
||||
const audioEl = document.createElement('audio')
|
||||
audioEl.autoplay = true
|
||||
audioElRef.current = audioEl
|
||||
pc.ontrack = (e) => {
|
||||
audioEl.srcObject = e.streams[0] ?? null
|
||||
}
|
||||
|
||||
pc.onconnectionstatechange = () => {
|
||||
const cs = pc.connectionState
|
||||
if (cs === 'failed' || cs === 'disconnected' || cs === 'closed') {
|
||||
setConnectionState((prev) => (prev === 'live' ? 'error' : prev))
|
||||
}
|
||||
}
|
||||
|
||||
// 4) 이벤트 데이터 채널
|
||||
const dc = pc.createDataChannel(DATA_CHANNEL_NAME)
|
||||
dcRef.current = dc
|
||||
dc.addEventListener('message', (e: MessageEvent) => {
|
||||
try {
|
||||
handleServerEvent(JSON.parse(e.data as string) as RealtimeServerEvent)
|
||||
} catch {
|
||||
// 파싱 불가 이벤트 무시
|
||||
}
|
||||
})
|
||||
dc.addEventListener('open', () => {
|
||||
// 유저 발화 전사 활성화 — 미지원 모델이면 error 이벤트만 오고 대화는 유지됨
|
||||
dc.send(
|
||||
JSON.stringify({
|
||||
type: 'session.update',
|
||||
session: {
|
||||
type: 'realtime',
|
||||
audio: {
|
||||
input: { transcription: { model: 'whisper-1' } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
// 5) SDP 교환
|
||||
const offer = await pc.createOffer()
|
||||
await pc.setLocalDescription(offer)
|
||||
|
||||
const sdpResponse = await fetch(`${REALTIME_CALLS_URL}?model=${encodeURIComponent(model)}`, {
|
||||
method: 'POST',
|
||||
body: offer.sdp,
|
||||
headers: {
|
||||
Authorization: `Bearer ${ephemeralKey}`,
|
||||
'Content-Type': 'application/sdp',
|
||||
},
|
||||
})
|
||||
if (!sdpResponse.ok) {
|
||||
const text = await sdpResponse.text()
|
||||
throw new Error(`SDP exchange failed (${sdpResponse.status}): ${text.slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const answerSdp = await sdpResponse.text()
|
||||
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp })
|
||||
|
||||
setConnectionState('live')
|
||||
setConversationState('listening')
|
||||
return true
|
||||
} catch (err) {
|
||||
cleanup()
|
||||
setConnectionState('error')
|
||||
setConversationState('idle')
|
||||
setError(err instanceof Error ? err.message : String(err))
|
||||
return false
|
||||
}
|
||||
}, [cleanup, handleServerEvent])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
flushAssistantMessage()
|
||||
cleanup()
|
||||
setConnectionState('idle')
|
||||
setConversationState('idle')
|
||||
}, [cleanup, flushAssistantMessage])
|
||||
|
||||
const sendText = useCallback((text: string) => {
|
||||
const dc = dcRef.current
|
||||
if (!dc || dc.readyState !== 'open') return
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ id: nextMessageId('user'), role: 'user', content: text, timestamp: Date.now() },
|
||||
])
|
||||
dc.send(
|
||||
JSON.stringify({
|
||||
type: 'conversation.item.create',
|
||||
item: {
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [{ type: 'input_text', text }],
|
||||
},
|
||||
}),
|
||||
)
|
||||
dc.send(JSON.stringify({ type: 'response.create' }))
|
||||
}, [])
|
||||
|
||||
const cancelResponse = useCallback(() => {
|
||||
const dc = dcRef.current
|
||||
if (!dc || dc.readyState !== 'open') return
|
||||
dc.send(JSON.stringify({ type: 'response.cancel' }))
|
||||
flushAssistantMessage()
|
||||
setConversationState('listening')
|
||||
}, [flushAssistantMessage])
|
||||
|
||||
const clearMessages = useCallback(() => {
|
||||
setMessages([])
|
||||
}, [])
|
||||
|
||||
return {
|
||||
connectionState,
|
||||
conversationState,
|
||||
messages,
|
||||
isActive: connectionState === 'connecting' || connectionState === 'live',
|
||||
streamingText,
|
||||
streamingMsgId,
|
||||
error,
|
||||
start,
|
||||
stop,
|
||||
sendText,
|
||||
cancelResponse,
|
||||
clearMessages,
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue