602 lines
21 KiB
Python
602 lines
21 KiB
Python
"""Voice routes for the OpenAI STT/TTS cascade over WebSocket.
|
|
|
|
Client sends JSON controls plus binary audio chunks:
|
|
audio_start -> binary audio chunks -> audio_end
|
|
|
|
Server emits:
|
|
ready -> state(listening) -> state(thinking) -> transcript -> reply
|
|
-> state(speaking) -> tts_chunk + binary audio chunks -> tts_end -> state(idle)
|
|
|
|
When voice is not configured, the route reports degraded state and closes
|
|
cleanly instead of crashing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import hashlib
|
|
import time
|
|
from typing import Optional
|
|
|
|
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
|
from fastapi.responses import JSONResponse
|
|
from starlette.websockets import WebSocketState
|
|
|
|
from .. import session_persistence, turn_runtime
|
|
from ..auth_sessions import get_session, user_has_consent
|
|
from ..config import settings
|
|
from ..deps import Principal, Role
|
|
from ..engine_client import EngineError, engine_client
|
|
from ..persona_repository import get_catalog_persona
|
|
from ..runtime_policy import require_runtime_fallback_allowed
|
|
from ..services import evaluator, orchestrator, state_machine
|
|
from ..services import voice as voice_svc
|
|
from ..services.voice import VoicePreset, VoiceUnavailable, resolve_voice, voice_service
|
|
from ..store import InProcSession, TurnRecord, store
|
|
|
|
router = APIRouter(prefix="/voice", tags=["voice"])
|
|
|
|
# WebSocket close codes.
|
|
WS_CLOSE_DEGRADED = 1011
|
|
WS_CLOSE_BAD_REQUEST = 1008
|
|
WS_CLOSE_UNAUTHORIZED = 1008
|
|
|
|
# Per-utterance audio cap to avoid unbounded memory growth.
|
|
_MAX_AUDIO_BYTES = 10 * 1024 * 1024
|
|
|
|
|
|
@router.get("/health")
|
|
async def voice_health() -> JSONResponse:
|
|
"""Return voice service readiness."""
|
|
available = voice_service.is_available()
|
|
tts_provider = voice_service.tts_provider()
|
|
body = {
|
|
"status": "ok" if available else "degraded",
|
|
"available": available,
|
|
"stt_model": voice_svc.STT_MODEL,
|
|
"tts_model": voice_svc.TTS_MODEL,
|
|
"tts_provider": tts_provider,
|
|
"reason": None if available else "OPENAI_API_KEY is not configured",
|
|
}
|
|
return JSONResponse(body, status_code=200 if available else 503)
|
|
|
|
|
|
@router.websocket("/ws")
|
|
async def voice_ws(websocket: WebSocket) -> None:
|
|
"""Run one authenticated learner voice cascade."""
|
|
await websocket.accept()
|
|
|
|
# Authenticate the same server-side browser session used by REST routes.
|
|
principal = await _principal_from_websocket(websocket)
|
|
if principal is None:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": "not authenticated"})
|
|
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
|
|
return
|
|
if principal.role != Role.LEARNER:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": "only learners can use voice"})
|
|
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
|
|
return
|
|
|
|
# Bind to an existing session first. persona_code creation is dev-only.
|
|
session_id, voice_preset, err, bind_meta = await _bind_session(websocket, principal)
|
|
if err is not None:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": err})
|
|
await _safe_close(websocket, WS_CLOSE_BAD_REQUEST)
|
|
return
|
|
assert session_id is not None and voice_preset is not None
|
|
if bind_meta.get("degraded"):
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "degraded",
|
|
"reason": bind_meta.get("degraded_reason", "voice session binding degraded"),
|
|
**bind_meta,
|
|
},
|
|
)
|
|
|
|
# Voice misconfiguration is reported explicitly and then closed cleanly.
|
|
if not voice_service.is_available():
|
|
await _safe_send_json(
|
|
websocket,
|
|
{"type": "degraded", "reason": "OPENAI_API_KEY is not configured"},
|
|
)
|
|
await _safe_close(websocket, WS_CLOSE_DEGRADED)
|
|
return
|
|
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "ready",
|
|
"session_id": session_id,
|
|
"voice": voice_preset.openai_voice,
|
|
"preset": voice_preset.preset,
|
|
"tts_provider": voice_service.tts_provider(),
|
|
"state": "idle",
|
|
**bind_meta,
|
|
},
|
|
)
|
|
|
|
audio_buf = bytearray()
|
|
receiving = False
|
|
audio_started_at: float | None = None
|
|
last_audio_end_at: float | None = None
|
|
|
|
try:
|
|
while True:
|
|
msg = await websocket.receive()
|
|
mtype = msg.get("type")
|
|
if mtype == "websocket.disconnect":
|
|
break
|
|
|
|
# Binary frames are audio chunks.
|
|
if msg.get("bytes") is not None:
|
|
if not receiving:
|
|
# Be tolerant when audio arrives before audio_start.
|
|
receiving = True
|
|
audio_started_at = time.monotonic()
|
|
audio_buf.clear()
|
|
await _safe_send_json(websocket, {"type": "state", "state": "listening"})
|
|
audio_buf.extend(msg["bytes"])
|
|
if len(audio_buf) > _MAX_AUDIO_BYTES:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{"type": "error", "detail": "audio too large; please send a shorter utterance"},
|
|
)
|
|
audio_buf.clear()
|
|
receiving = False
|
|
continue
|
|
|
|
# Text frames are JSON controls.
|
|
text = msg.get("text")
|
|
if text is None:
|
|
continue
|
|
try:
|
|
ctrl = json.loads(text)
|
|
except (json.JSONDecodeError, TypeError):
|
|
await _safe_send_json(websocket, {"type": "error", "detail": "invalid control json"})
|
|
continue
|
|
|
|
ctype = ctrl.get("type")
|
|
if ctype == "audio_start":
|
|
receiving = True
|
|
audio_started_at = time.monotonic()
|
|
audio_buf.clear()
|
|
await _safe_send_json(websocket, {"type": "state", "state": "listening"})
|
|
|
|
elif ctype == "audio_end":
|
|
receiving = False
|
|
audio_ended_at = time.monotonic()
|
|
silence_ms = _safe_int(ctrl.get("silence_ms"))
|
|
if silence_ms is None and last_audio_end_at is not None and audio_started_at is not None:
|
|
silence_ms = max(0, int((audio_started_at - last_audio_end_at) * 1000))
|
|
await _handle_utterance(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
voice_preset=voice_preset,
|
|
audio=bytes(audio_buf),
|
|
fmt=ctrl.get("format"),
|
|
audio_started_at=audio_started_at,
|
|
audio_ended_at=audio_ended_at,
|
|
silence_ms=silence_ms,
|
|
barge_in=_safe_bool(ctrl.get("barge_in")),
|
|
)
|
|
last_audio_end_at = audio_ended_at
|
|
audio_started_at = None
|
|
audio_buf.clear()
|
|
|
|
elif ctype == "text_turn":
|
|
# Text-only path for accessibility and deterministic tests.
|
|
receiving = False
|
|
audio_buf.clear()
|
|
learner_text = (ctrl.get("text") or "").strip()
|
|
if learner_text:
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
voice_preset=voice_preset,
|
|
learner_text=learner_text,
|
|
)
|
|
|
|
elif ctype == "ping":
|
|
await _safe_send_json(websocket, {"type": "pong"})
|
|
|
|
elif ctype == "close":
|
|
break
|
|
|
|
except WebSocketDisconnect:
|
|
pass
|
|
except Exception as e:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": f"voice ws error: {e}"})
|
|
finally:
|
|
await _safe_close(websocket)
|
|
|
|
|
|
async def _handle_utterance(
|
|
websocket: WebSocket,
|
|
*,
|
|
session_id: str,
|
|
principal: Principal,
|
|
voice_preset: VoicePreset,
|
|
audio: bytes,
|
|
fmt: Optional[str],
|
|
audio_started_at: float | None = None,
|
|
audio_ended_at: float | None = None,
|
|
silence_ms: int | None = None,
|
|
barge_in: bool | None = None,
|
|
) -> None:
|
|
"""Transcribe one utterance, generate the client reply, then synthesize TTS."""
|
|
if not audio:
|
|
await _safe_send_json(websocket, {"type": "transcript", "text": "", "final": True})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
# STT begins after the learner stops speaking.
|
|
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
|
filename, content_type = _audio_meta(fmt)
|
|
try:
|
|
stt = await voice_service.transcribe(
|
|
audio, filename=filename, content_type=content_type
|
|
)
|
|
except VoiceUnavailable as e:
|
|
await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
except Exception as e:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": f"STT failed: {e}"})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
learner_text = stt.text
|
|
audio_ref = _voice_audio_ref(audio, fmt)
|
|
duration_s = stt.duration or _elapsed_seconds(audio_started_at, audio_ended_at)
|
|
speech_rate = _estimate_speech_rate(learner_text, duration_s)
|
|
await _safe_send_json(
|
|
websocket,
|
|
{"type": "transcript", "text": learner_text, "final": True, "speaker": "counselor"},
|
|
)
|
|
if not learner_text:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
voice_preset=voice_preset,
|
|
learner_text=learner_text,
|
|
audio_ref=audio_ref,
|
|
silence_ms=silence_ms,
|
|
speech_rate=speech_rate,
|
|
barge_in=barge_in,
|
|
)
|
|
|
|
|
|
async def _run_turn_and_speak(
|
|
websocket: WebSocket,
|
|
*,
|
|
session_id: str,
|
|
principal: Principal,
|
|
voice_preset: VoicePreset,
|
|
learner_text: str,
|
|
audio_ref: str | None = None,
|
|
silence_ms: int | None = None,
|
|
speech_rate: float | None = None,
|
|
barge_in: bool | None = None,
|
|
) -> None:
|
|
"""Run one counseling turn and stream synthesized client speech."""
|
|
sess, err = await _load_voice_session(session_id, principal)
|
|
if sess is None:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": err or "session not found or ended"})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
from . import sessions as session_routes
|
|
|
|
recall = await session_routes.ensure_recall_context(sess)
|
|
kb_cues = session_routes._KB_CUES_CACHE.get(session_id) or []
|
|
ctx = orchestrator.prepare_turn(
|
|
session_id=session_id,
|
|
case_id=sess.case_id,
|
|
card=sess.persona,
|
|
state=sess.state,
|
|
learner_text=learner_text,
|
|
recall_summary=recall.recall_summary,
|
|
pinned_facts=recall.pinned_facts,
|
|
recent_turns=sess.recent_turns(visible_to="client"),
|
|
kb_behavior_cues=kb_cues,
|
|
theory_mode=sess.theory_mode,
|
|
)
|
|
assert ctx.state_after is not None
|
|
|
|
# Voice needs the full client reply before TTS starts.
|
|
try:
|
|
result = await orchestrator.run_turn_generate(
|
|
ctx,
|
|
engine_client,
|
|
eval_hook=evaluator.make_eval_hook(
|
|
engine_client,
|
|
audit_hook=session_persistence.record_llm_call_audit,
|
|
),
|
|
audit_hook=session_persistence.record_llm_call_audit,
|
|
)
|
|
except EngineError as e:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": f"engine unavailable: {e}"})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
reply = result.client_reply or ""
|
|
# Persist only after the client reply has been generated. A failed AI turn
|
|
# must not leave a learner-only transcript in review or history.
|
|
await turn_runtime.record_completed_turn(
|
|
sess,
|
|
ctx,
|
|
result,
|
|
context_prefix="voice session",
|
|
counselor_turn=TurnRecord(
|
|
turn_seq=ctx.state_after.turn_seq,
|
|
speaker="counselor",
|
|
stage=turn_runtime.stage_label(ctx.state_after.stage),
|
|
text=learner_text,
|
|
text_masked=ctx.learner_text_masked,
|
|
audio_ref=audio_ref,
|
|
silence_ms=silence_ms,
|
|
speech_rate=speech_rate,
|
|
barge_in=barge_in,
|
|
evaluation=result.evaluation,
|
|
),
|
|
)
|
|
await turn_runtime.record_safety_event(sess, ctx, result)
|
|
|
|
# Send the final client text before audio playback.
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "reply",
|
|
"text": reply,
|
|
"speaker": "client",
|
|
"stage": result.stage,
|
|
"effective_openness": round(result.effective_openness, 4),
|
|
"turn_seq": result.turn_seq,
|
|
"safety_flagged": result.safety_flagged,
|
|
"crisis_kind": result.crisis_kind,
|
|
"crisis_resource": result.crisis_resource,
|
|
"conversation_stopped": result.conversation_stopped,
|
|
},
|
|
)
|
|
|
|
if not reply:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
# TTS speaking state comes before chunk metadata and binary audio.
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "state",
|
|
"state": "speaking",
|
|
"voice": voice_preset.openai_voice,
|
|
"tts_provider": voice_service.tts_provider(),
|
|
},
|
|
)
|
|
try:
|
|
n = 0
|
|
async for ck in voice_service.synthesize_stream(reply, voice_preset):
|
|
# 바이너리 오디오 청크만 송신(프론트가 Web Audio AnalyserNode로 립싱크 자체 산출).
|
|
await _safe_send_bytes(websocket, ck.audio)
|
|
n += 1
|
|
await _safe_send_json(websocket, {"type": "tts_end", "chunks": n})
|
|
except VoiceUnavailable as e:
|
|
await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)})
|
|
except Exception as e:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": f"TTS failed: {e}"})
|
|
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
|
|
|
|
async def _load_voice_session(
|
|
session_id: str,
|
|
principal: Principal,
|
|
) -> tuple[InProcSession | None, str | None]:
|
|
sess, err = await turn_runtime.load_owned_session(session_id, principal)
|
|
if err == turn_runtime.SessionAccessError.NOT_FOUND:
|
|
return None, f"unknown session {session_id}"
|
|
if err == turn_runtime.SessionAccessError.FORBIDDEN:
|
|
return None, "session does not belong to user"
|
|
if err == turn_runtime.SessionAccessError.ENDED:
|
|
return None, "session already ended"
|
|
assert sess is not None
|
|
return sess, None
|
|
|
|
|
|
async def _principal_from_websocket(websocket: WebSocket) -> Principal | None:
|
|
"""Restore the same server-side browser session used by REST routes."""
|
|
raw_cookie = websocket.cookies.get(settings.cookie_name)
|
|
if raw_cookie is None and settings.environment == "dev":
|
|
raw_cookie = websocket.cookies.get("vignette_sid")
|
|
|
|
session = await get_session(raw_cookie)
|
|
if session is None:
|
|
return None
|
|
|
|
try:
|
|
role = Role(session.role)
|
|
except ValueError:
|
|
return None
|
|
|
|
return Principal(
|
|
user_id=session.user_id,
|
|
role=role,
|
|
cohort_ids=session.cohort_ids,
|
|
email=session.email,
|
|
display_name=session.display_name,
|
|
consent_at=session.consent_at,
|
|
)
|
|
|
|
|
|
async def _bind_session(
|
|
websocket: WebSocket,
|
|
principal: Principal,
|
|
) -> tuple[Optional[str], Optional[VoicePreset], Optional[str], dict[str, object]]:
|
|
"""Bind an existing session or create a dev-only voice session."""
|
|
qp = websocket.query_params
|
|
explicit_preset = qp.get("preset")
|
|
|
|
session_id = qp.get("session_id")
|
|
if session_id:
|
|
sess, err = await _load_voice_session(session_id, principal)
|
|
if sess is None:
|
|
return None, None, err or f"unknown session {session_id}", {}
|
|
vp = resolve_voice(persona_code=sess.persona.code, preset=explicit_preset)
|
|
return session_id, vp, None, {"degraded": False, "persona_catalog_source": "session"}
|
|
|
|
# persona_code session creation is local-dev only. Production uses REST start.
|
|
if settings.environment != "dev":
|
|
return None, None, "session_id required", {}
|
|
|
|
persona_code = qp.get("persona_code")
|
|
if not persona_code:
|
|
return None, None, "session_id or persona_code query required", {}
|
|
if principal.consent_at is None and not await user_has_consent(principal.user_id):
|
|
return None, None, "consent_required", {}
|
|
try:
|
|
catalog_persona = await get_catalog_persona(persona_code)
|
|
except Exception:
|
|
return None, None, "persona catalog database unavailable", {}
|
|
if catalog_persona is None:
|
|
return None, None, f"unknown persona {persona_code}", {}
|
|
card = catalog_persona.card
|
|
|
|
st = state_machine.init_state(
|
|
params=card.openness_params(),
|
|
)
|
|
sess = await session_persistence.create_session(
|
|
learner_id=principal.user_id,
|
|
card=card,
|
|
theory_mode="humanistic",
|
|
state=st,
|
|
session_no=1,
|
|
carry_rapport=st.rapport_credit,
|
|
persona_id=catalog_persona.persona_id,
|
|
persona_version=catalog_persona.version,
|
|
)
|
|
session_source = "database"
|
|
if sess is None:
|
|
require_runtime_fallback_allowed("voice session creation")
|
|
sess = store.create(
|
|
learner_id=principal.user_id,
|
|
persona=card,
|
|
theory_mode="humanistic",
|
|
state=st,
|
|
session_no=1,
|
|
carry_rapport=st.rapport_credit,
|
|
)
|
|
session_source = "runtime"
|
|
else:
|
|
store.put(sess)
|
|
vp = resolve_voice(persona_code=card.code, preset=explicit_preset)
|
|
degraded_reasons: list[str] = []
|
|
if catalog_persona.degraded:
|
|
degraded_reasons.append("카탈로그 원본을 확인하지 못해 음성 회기를 시작하지 않습니다")
|
|
if session_source == "runtime":
|
|
degraded_reasons.append("세션 저장소 연결 전까지 비영구 개발 런타임 기록을 사용합니다")
|
|
bind_meta = {
|
|
"degraded": bool(degraded_reasons),
|
|
"degraded_reason": "; ".join(degraded_reasons) if degraded_reasons else None,
|
|
"persona_catalog_source": catalog_persona.source,
|
|
"session_source": session_source,
|
|
}
|
|
return sess.session_id, vp, None, bind_meta
|
|
|
|
|
|
def _audio_meta(fmt: Optional[str]) -> tuple[str, str]:
|
|
"""Map the browser audio format to upload metadata."""
|
|
f = (fmt or "webm").lower().lstrip(".")
|
|
table = {
|
|
"webm": ("audio.webm", "audio/webm"),
|
|
"ogg": ("audio.ogg", "audio/ogg"),
|
|
"opus": ("audio.ogg", "audio/ogg"),
|
|
"wav": ("audio.wav", "audio/wav"),
|
|
"mp3": ("audio.mp3", "audio/mpeg"),
|
|
"mp4": ("audio.mp4", "audio/mp4"),
|
|
"m4a": ("audio.m4a", "audio/mp4"),
|
|
"pcm": ("audio.wav", "audio/wav"),
|
|
}
|
|
return table.get(f, ("audio.webm", "audio/webm"))
|
|
|
|
|
|
def _voice_audio_ref(audio: bytes, fmt: Optional[str]) -> str | None:
|
|
if not audio:
|
|
return None
|
|
f = (fmt or "webm").lower().lstrip(".") or "webm"
|
|
digest = hashlib.sha256(audio).hexdigest()[:24]
|
|
return f"voice:{f}:sha256:{digest}"
|
|
|
|
|
|
def _elapsed_seconds(started_at: float | None, ended_at: float | None) -> float | None:
|
|
if started_at is None or ended_at is None:
|
|
return None
|
|
return max(0.001, ended_at - started_at)
|
|
|
|
|
|
def _estimate_speech_rate(text: str, duration_s: float | None) -> float | None:
|
|
if not text or not duration_s or duration_s <= 0:
|
|
return None
|
|
units = sum(1 for ch in text if not ch.isspace())
|
|
if units <= 0:
|
|
return None
|
|
return round((units / duration_s) * 60.0, 2)
|
|
|
|
|
|
def _safe_int(value: object) -> int | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _safe_bool(value: object) -> bool | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
normalized = value.strip().lower()
|
|
if normalized in {"1", "true", "yes", "y"}:
|
|
return True
|
|
if normalized in {"0", "false", "no", "n"}:
|
|
return False
|
|
return bool(value)
|
|
|
|
|
|
async def _safe_send_json(websocket: WebSocket, payload: dict) -> None:
|
|
if websocket.client_state != WebSocketState.CONNECTED:
|
|
return
|
|
try:
|
|
await websocket.send_text(json.dumps(payload, ensure_ascii=False))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def _safe_send_bytes(websocket: WebSocket, data: bytes) -> None:
|
|
if websocket.client_state != WebSocketState.CONNECTED:
|
|
return
|
|
try:
|
|
await websocket.send_bytes(data)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def _safe_close(websocket: WebSocket, code: int = 1000) -> None:
|
|
if websocket.client_state == WebSocketState.DISCONNECTED:
|
|
return
|
|
try:
|
|
await websocket.close(code=code)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
__all__ = ["router"]
|