"""Voice routes for the OpenAI STT/TTS cascade over WebSocket. Client sends JSON controls plus binary audio chunks: audio_start -> binary audio chunks -> audio_end Server emits: ready -> state(listening) -> state(thinking) -> transcript -> reply -> state(speaking) -> tts_chunk + binary audio chunks -> tts_end -> state(idle) When voice is not configured, the route reports degraded state and closes cleanly instead of crashing. """ from __future__ import annotations import json import hashlib import time from typing import Optional from fastapi import APIRouter, WebSocket, WebSocketDisconnect from fastapi.responses import JSONResponse from starlette.websockets import WebSocketState from .. import session_persistence, turn_runtime from ..auth_sessions import get_session from ..config import settings from ..deps import Principal, Role from ..engine_client import EngineError, engine_client from ..persona_repository import get_catalog_persona from ..runtime_policy import require_runtime_fallback_allowed from ..services import evaluator, orchestrator, state_machine from ..services import voice as voice_svc from ..services.voice import VoicePreset, VoiceUnavailable, resolve_voice, voice_service from ..store import InProcSession, TurnRecord, store router = APIRouter(prefix="/voice", tags=["voice"]) # WebSocket close codes. WS_CLOSE_DEGRADED = 1011 WS_CLOSE_BAD_REQUEST = 1008 WS_CLOSE_UNAUTHORIZED = 1008 # Per-utterance audio cap to avoid unbounded memory growth. _MAX_AUDIO_BYTES = 10 * 1024 * 1024 @router.get("/health") async def voice_health() -> JSONResponse: """Return voice service readiness.""" available = voice_service.is_available() tts_provider = voice_service.tts_provider() body = { "status": "ok" if available else "degraded", "available": available, "stt_model": voice_svc.STT_MODEL, "tts_model": voice_svc.TTS_MODEL, "tts_provider": tts_provider, "reason": None if available else "OPENAI_API_KEY is not configured", } return JSONResponse(body, status_code=200 if available else 503) @router.websocket("/ws") async def voice_ws(websocket: WebSocket) -> None: """Run one authenticated learner voice cascade.""" await websocket.accept() # Authenticate the same server-side browser session used by REST routes. principal = await _principal_from_websocket(websocket) if principal is None: await _safe_send_json(websocket, {"type": "error", "detail": "not authenticated"}) await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED) return if principal.role != Role.LEARNER: await _safe_send_json(websocket, {"type": "error", "detail": "only learners can use voice"}) await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED) return # Bind to an existing session first. persona_code creation is dev-only. session_id, voice_preset, err, bind_meta = await _bind_session(websocket, principal) if err is not None: await _safe_send_json(websocket, {"type": "error", "detail": err}) await _safe_close(websocket, WS_CLOSE_BAD_REQUEST) return assert session_id is not None and voice_preset is not None if bind_meta.get("degraded"): await _safe_send_json( websocket, { "type": "degraded", "reason": bind_meta.get("degraded_reason", "voice session binding degraded"), **bind_meta, }, ) # Voice misconfiguration is reported explicitly and then closed cleanly. if not voice_service.is_available(): await _safe_send_json( websocket, {"type": "degraded", "reason": "OPENAI_API_KEY is not configured"}, ) await _safe_close(websocket, WS_CLOSE_DEGRADED) return await _safe_send_json( websocket, { "type": "ready", "session_id": session_id, "voice": voice_preset.openai_voice, "preset": voice_preset.preset, "tts_provider": voice_service.tts_provider(), "state": "idle", **bind_meta, }, ) audio_buf = bytearray() receiving = False audio_started_at: float | None = None last_audio_end_at: float | None = None try: while True: msg = await websocket.receive() mtype = msg.get("type") if mtype == "websocket.disconnect": break # Binary frames are audio chunks. if msg.get("bytes") is not None: if not receiving: # Be tolerant when audio arrives before audio_start. receiving = True audio_started_at = time.monotonic() audio_buf.clear() await _safe_send_json(websocket, {"type": "state", "state": "listening"}) audio_buf.extend(msg["bytes"]) if len(audio_buf) > _MAX_AUDIO_BYTES: await _safe_send_json( websocket, {"type": "error", "detail": "audio too large; please send a shorter utterance"}, ) audio_buf.clear() receiving = False continue # Text frames are JSON controls. text = msg.get("text") if text is None: continue try: ctrl = json.loads(text) except (json.JSONDecodeError, TypeError): await _safe_send_json(websocket, {"type": "error", "detail": "invalid control json"}) continue ctype = ctrl.get("type") if ctype == "audio_start": receiving = True audio_started_at = time.monotonic() audio_buf.clear() await _safe_send_json(websocket, {"type": "state", "state": "listening"}) elif ctype == "audio_end": receiving = False audio_ended_at = time.monotonic() silence_ms = _safe_int(ctrl.get("silence_ms")) if silence_ms is None and last_audio_end_at is not None and audio_started_at is not None: silence_ms = max(0, int((audio_started_at - last_audio_end_at) * 1000)) await _handle_utterance( websocket, session_id=session_id, principal=principal, voice_preset=voice_preset, audio=bytes(audio_buf), fmt=ctrl.get("format"), audio_started_at=audio_started_at, audio_ended_at=audio_ended_at, silence_ms=silence_ms, barge_in=_safe_bool(ctrl.get("barge_in")), ) last_audio_end_at = audio_ended_at audio_started_at = None audio_buf.clear() elif ctype == "text_turn": # Text-only path for accessibility and deterministic tests. receiving = False audio_buf.clear() learner_text = (ctrl.get("text") or "").strip() if learner_text: await _run_turn_and_speak( websocket, session_id=session_id, principal=principal, voice_preset=voice_preset, learner_text=learner_text, ) elif ctype == "ping": await _safe_send_json(websocket, {"type": "pong"}) elif ctype == "close": break except WebSocketDisconnect: pass except Exception as e: await _safe_send_json(websocket, {"type": "error", "detail": f"voice ws error: {e}"}) finally: await _safe_close(websocket) async def _handle_utterance( websocket: WebSocket, *, session_id: str, principal: Principal, voice_preset: VoicePreset, audio: bytes, fmt: Optional[str], audio_started_at: float | None = None, audio_ended_at: float | None = None, silence_ms: int | None = None, barge_in: bool | None = None, ) -> None: """Transcribe one utterance, generate the client reply, then synthesize TTS.""" if not audio: await _safe_send_json(websocket, {"type": "transcript", "text": "", "final": True}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return # STT begins after the learner stops speaking. await _safe_send_json(websocket, {"type": "state", "state": "thinking"}) filename, content_type = _audio_meta(fmt) try: stt = await voice_service.transcribe( audio, filename=filename, content_type=content_type ) except VoiceUnavailable as e: await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return except Exception as e: await _safe_send_json(websocket, {"type": "error", "detail": f"STT failed: {e}"}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return learner_text = stt.text audio_ref = _voice_audio_ref(audio, fmt) duration_s = stt.duration or _elapsed_seconds(audio_started_at, audio_ended_at) speech_rate = _estimate_speech_rate(learner_text, duration_s) await _safe_send_json( websocket, {"type": "transcript", "text": learner_text, "final": True, "speaker": "counselor"}, ) if not learner_text: await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return await _run_turn_and_speak( websocket, session_id=session_id, principal=principal, voice_preset=voice_preset, learner_text=learner_text, audio_ref=audio_ref, silence_ms=silence_ms, speech_rate=speech_rate, barge_in=barge_in, ) async def _run_turn_and_speak( websocket: WebSocket, *, session_id: str, principal: Principal, voice_preset: VoicePreset, learner_text: str, audio_ref: str | None = None, silence_ms: int | None = None, speech_rate: float | None = None, barge_in: bool | None = None, ) -> None: """Run one counseling turn and stream synthesized client speech.""" sess, err = await _load_voice_session(session_id, principal) if sess is None: await _safe_send_json(websocket, {"type": "error", "detail": err or "session not found or ended"}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return from . import sessions as session_routes recall = await session_routes.ensure_recall_context(sess) kb_cues = session_routes._KB_CUES_CACHE.get(session_id) or [] ctx = orchestrator.prepare_turn( session_id=session_id, case_id=sess.case_id, card=sess.persona, state=sess.state, learner_text=learner_text, recall_summary=recall.recall_summary, pinned_facts=recall.pinned_facts, recent_turns=sess.recent_turns(visible_to="client"), kb_behavior_cues=kb_cues, theory_mode=sess.theory_mode, ) assert ctx.state_after is not None # Voice needs the full client reply before TTS starts. try: result = await orchestrator.run_turn_generate( ctx, engine_client, eval_hook=evaluator.make_eval_hook( engine_client, audit_hook=session_persistence.record_llm_call_audit, ), audit_hook=session_persistence.record_llm_call_audit, ) except EngineError as e: await _safe_send_json(websocket, {"type": "error", "detail": f"engine unavailable: {e}"}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return reply = result.client_reply or "" # Persist only after the client reply has been generated. A failed AI turn # must not leave a learner-only transcript in review or history. await turn_runtime.record_completed_turn( sess, ctx, result, context_prefix="voice session", counselor_turn=TurnRecord( turn_seq=ctx.state_after.turn_seq, speaker="counselor", stage=turn_runtime.stage_label(ctx.state_after.stage), text=learner_text, text_masked=ctx.learner_text_masked, audio_ref=audio_ref, silence_ms=silence_ms, speech_rate=speech_rate, barge_in=barge_in, evaluation=result.evaluation, ), ) await turn_runtime.record_safety_event(sess, ctx, result) # Send the final client text before audio playback. await _safe_send_json( websocket, { "type": "reply", "text": reply, "speaker": "client", "stage": result.stage, "effective_openness": round(result.effective_openness, 4), "turn_seq": result.turn_seq, "safety_flagged": result.safety_flagged, "crisis_kind": result.crisis_kind, "crisis_resource": result.crisis_resource, "conversation_stopped": result.conversation_stopped, }, ) if not reply: await _safe_send_json(websocket, {"type": "state", "state": "idle"}) return # TTS speaking state comes before chunk metadata and binary audio. await _safe_send_json( websocket, { "type": "state", "state": "speaking", "voice": voice_preset.openai_voice, "tts_provider": voice_service.tts_provider(), }, ) try: n = 0 async for ck in voice_service.synthesize_stream(reply, voice_preset): # 바이너리 오디오 청크만 송신(프론트가 Web Audio AnalyserNode로 립싱크 자체 산출). await _safe_send_bytes(websocket, ck.audio) n += 1 await _safe_send_json(websocket, {"type": "tts_end", "chunks": n}) except VoiceUnavailable as e: await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)}) except Exception as e: await _safe_send_json(websocket, {"type": "error", "detail": f"TTS failed: {e}"}) await _safe_send_json(websocket, {"type": "state", "state": "idle"}) async def _load_voice_session( session_id: str, principal: Principal, ) -> tuple[InProcSession | None, str | None]: sess, err = await turn_runtime.load_owned_session(session_id, principal) if err == turn_runtime.SessionAccessError.NOT_FOUND: return None, f"unknown session {session_id}" if err == turn_runtime.SessionAccessError.FORBIDDEN: return None, "session does not belong to user" if err == turn_runtime.SessionAccessError.ENDED: return None, "session already ended" assert sess is not None return sess, None async def _principal_from_websocket(websocket: WebSocket) -> Principal | None: """Restore the same server-side browser session used by REST routes.""" raw_cookie = websocket.cookies.get(settings.cookie_name) if raw_cookie is None and settings.environment == "dev": raw_cookie = websocket.cookies.get("vignette_sid") session = await get_session(raw_cookie) if session is None: return None try: role = Role(session.role) except ValueError: return None return Principal( user_id=session.user_id, role=role, cohort_ids=session.cohort_ids, email=session.email, display_name=session.display_name, ) async def _bind_session( websocket: WebSocket, principal: Principal, ) -> tuple[Optional[str], Optional[VoicePreset], Optional[str], dict[str, object]]: """Bind an existing session or create a dev-only voice session.""" qp = websocket.query_params explicit_preset = qp.get("preset") session_id = qp.get("session_id") if session_id: sess, err = await _load_voice_session(session_id, principal) if sess is None: return None, None, err or f"unknown session {session_id}", {} vp = resolve_voice(persona_code=sess.persona.code, preset=explicit_preset) return session_id, vp, None, {"degraded": False, "persona_catalog_source": "session"} # persona_code session creation is local-dev only. Production uses REST start. if settings.environment != "dev": return None, None, "session_id required", {} persona_code = qp.get("persona_code") if not persona_code: return None, None, "session_id or persona_code query required", {} try: catalog_persona = await get_catalog_persona(persona_code) except Exception: return None, None, "persona catalog database unavailable", {} if catalog_persona is None: return None, None, f"unknown persona {persona_code}", {} card = catalog_persona.card st = state_machine.init_state( params=card.openness_params(), ) sess = await session_persistence.create_session( learner_id=principal.user_id, card=card, theory_mode="humanistic", state=st, session_no=1, carry_rapport=st.rapport_credit, persona_id=catalog_persona.persona_id, persona_version=catalog_persona.version, ) session_source = "database" if sess is None: require_runtime_fallback_allowed("voice session creation") sess = store.create( learner_id=principal.user_id, persona=card, theory_mode="humanistic", state=st, session_no=1, carry_rapport=st.rapport_credit, ) session_source = "runtime" else: store.put(sess) vp = resolve_voice(persona_code=card.code, preset=explicit_preset) degraded_reasons: list[str] = [] if catalog_persona.degraded: degraded_reasons.append("카탈로그 원본을 확인하지 못해 음성 회기를 시작하지 않습니다") if session_source == "runtime": degraded_reasons.append("세션 저장소 연결 전까지 비영구 개발 런타임 기록을 사용합니다") bind_meta = { "degraded": bool(degraded_reasons), "degraded_reason": "; ".join(degraded_reasons) if degraded_reasons else None, "persona_catalog_source": catalog_persona.source, "session_source": session_source, } return sess.session_id, vp, None, bind_meta def _audio_meta(fmt: Optional[str]) -> tuple[str, str]: """Map the browser audio format to upload metadata.""" f = (fmt or "webm").lower().lstrip(".") table = { "webm": ("audio.webm", "audio/webm"), "ogg": ("audio.ogg", "audio/ogg"), "opus": ("audio.ogg", "audio/ogg"), "wav": ("audio.wav", "audio/wav"), "mp3": ("audio.mp3", "audio/mpeg"), "mp4": ("audio.mp4", "audio/mp4"), "m4a": ("audio.m4a", "audio/mp4"), "pcm": ("audio.wav", "audio/wav"), } return table.get(f, ("audio.webm", "audio/webm")) def _voice_audio_ref(audio: bytes, fmt: Optional[str]) -> str | None: if not audio: return None f = (fmt or "webm").lower().lstrip(".") or "webm" digest = hashlib.sha256(audio).hexdigest()[:24] return f"voice:{f}:sha256:{digest}" def _elapsed_seconds(started_at: float | None, ended_at: float | None) -> float | None: if started_at is None or ended_at is None: return None return max(0.001, ended_at - started_at) def _estimate_speech_rate(text: str, duration_s: float | None) -> float | None: if not text or not duration_s or duration_s <= 0: return None units = sum(1 for ch in text if not ch.isspace()) if units <= 0: return None return round((units / duration_s) * 60.0, 2) def _safe_int(value: object) -> int | None: if value is None: return None try: return int(value) except (TypeError, ValueError): return None def _safe_bool(value: object) -> bool | None: if value is None: return None if isinstance(value, bool): return value if isinstance(value, str): normalized = value.strip().lower() if normalized in {"1", "true", "yes", "y"}: return True if normalized in {"0", "false", "no", "n"}: return False return bool(value) async def _safe_send_json(websocket: WebSocket, payload: dict) -> None: if websocket.client_state != WebSocketState.CONNECTED: return try: await websocket.send_text(json.dumps(payload, ensure_ascii=False)) except Exception: pass async def _safe_send_bytes(websocket: WebSocket, data: bytes) -> None: if websocket.client_state != WebSocketState.CONNECTED: return try: await websocket.send_bytes(data) except Exception: pass async def _safe_close(websocket: WebSocket, code: int = 1000) -> None: if websocket.client_state == WebSocketState.DISCONNECTED: return try: await websocket.close(code=code) except Exception: pass __all__ = ["router"]