1925 lines
69 KiB
Python
1925 lines
69 KiB
Python
"""Voice routes for the OpenAI STT/TTS cascade over WebSocket.
|
|
|
|
Client sends JSON controls plus binary audio chunks:
|
|
audio_start -> binary audio chunks -> audio_end
|
|
|
|
Server emits:
|
|
ready -> state(listening) -> state(thinking) -> transcript -> reply
|
|
-> state(speaking) -> binary audio chunks -> tts_end -> state(idle)
|
|
|
|
When voice is not configured, the route reports degraded state and closes
|
|
cleanly instead of crashing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import hashlib
|
|
import hmac
|
|
import time
|
|
from dataclasses import dataclass, field as dataclass_field
|
|
from typing import Optional
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
|
|
from fastapi import APIRouter, WebSocket, WebSocketDisconnect, HTTPException, status
|
|
from fastapi.responses import JSONResponse, Response
|
|
from pydantic import BaseModel, Field
|
|
from starlette.websockets import WebSocketState
|
|
|
|
from .. import session_persistence, turn_runtime
|
|
from ..auth_sessions import get_session, user_has_consent, user_onboarding_complete
|
|
from ..config import settings
|
|
from ..deps import CurrentPrincipal, Principal, Role
|
|
from ..engine_client import EngineError, engine_client
|
|
from ..persona_repository import (
|
|
PersonaVoiceMap,
|
|
get_catalog_persona,
|
|
get_persona_voice_map,
|
|
get_session_voice_map,
|
|
)
|
|
from ..runtime_policy import require_runtime_fallback_allowed
|
|
from ..services import (
|
|
evaluator,
|
|
guardrail,
|
|
multimodal_alliance,
|
|
multimodal_alliance_store,
|
|
orchestrator,
|
|
rupture_scenario_director,
|
|
state_machine,
|
|
)
|
|
from ..services import voice as voice_svc
|
|
from ..services.voice import (
|
|
StreamingTranscriptEvent,
|
|
TranscriptResult,
|
|
VoicePreset,
|
|
VoiceUnavailable,
|
|
resolve_voice,
|
|
voice_service,
|
|
)
|
|
from ..services.voice_runtime import (
|
|
VOICE_AUDIO_BUFFER_MAX_BYTES,
|
|
VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
|
VOICE_UVICORN_WS_MAX_QUEUE,
|
|
voice_runtime_metrics,
|
|
)
|
|
from ..store import InProcSession, TurnRecord, store
|
|
|
|
router = APIRouter(prefix="/voice", tags=["voice"])
|
|
|
|
|
|
class VoiceSpeechRequest(BaseModel):
|
|
"""Request OpenAI TTS for an already-persisted client reply."""
|
|
|
|
session_id: str = Field(min_length=1, max_length=80)
|
|
turn_seq: int = Field(ge=1)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class VoiceSessionContext:
|
|
session_id: str
|
|
principal: Principal
|
|
voice_preset: VoicePreset
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class VoiceProsody:
|
|
audio_ref: str | None = None
|
|
duration_s: float | None = None
|
|
silence_ms: int | None = None
|
|
speech_rate: float | None = None
|
|
barge_in: bool | None = None
|
|
provider_events: list[dict[str, object]] = dataclass_field(default_factory=list)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class VoiceTurnInput:
|
|
learner_text: str
|
|
prosody: VoiceProsody = dataclass_field(default_factory=VoiceProsody)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class VoiceAudioInput:
|
|
audio: bytes
|
|
fmt: str | None
|
|
sample_rate: int | None = None
|
|
channels: int | None = None
|
|
sample_width: int | None = None
|
|
audio_started_at: float | None = None
|
|
audio_ended_at: float | None = None
|
|
prosody: VoiceProsody = dataclass_field(default_factory=VoiceProsody)
|
|
|
|
|
|
# WebSocket close codes.
|
|
WS_CLOSE_DEGRADED = 1011
|
|
WS_CLOSE_BAD_REQUEST = 1008
|
|
WS_CLOSE_UNAUTHORIZED = 1008
|
|
|
|
# Per-utterance audio cap to avoid unbounded memory growth.
|
|
_MAX_AUDIO_BYTES = VOICE_AUDIO_BUFFER_MAX_BYTES
|
|
_STREAMING_EVENT_QUEUE_MAX_ITEMS = VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS
|
|
_STREAMING_CONSENT_RECHECK_SECONDS = 1.0
|
|
_PROVIDER_EVENT_MAX_ITEMS = 12
|
|
_PROVIDER_EVENT_MAX_STRING = 80
|
|
_PROVIDER_EVENT_ALLOWED_KEYS = {
|
|
"category",
|
|
"event_type",
|
|
"type",
|
|
"kind",
|
|
"label",
|
|
"source",
|
|
"provider",
|
|
"model",
|
|
"start_ms",
|
|
"end_ms",
|
|
"duration_ms",
|
|
"confidence",
|
|
"score",
|
|
"is_final",
|
|
}
|
|
_PROVIDER_EVENT_TAXONOMY = {
|
|
"barge_in": ("barge_in", "turn_taking"),
|
|
"interrupt": ("barge_in", "turn_taking"),
|
|
"interruption": ("barge_in", "turn_taking"),
|
|
"overlap": ("barge_in", "turn_taking"),
|
|
"sigh": ("sigh", "paralinguistic"),
|
|
"sighing": ("sigh", "paralinguistic"),
|
|
"sob": ("cry", "paralinguistic"),
|
|
"cry": ("cry", "paralinguistic"),
|
|
"crying": ("cry", "paralinguistic"),
|
|
"weep": ("cry", "paralinguistic"),
|
|
"laugh": ("laugh", "paralinguistic"),
|
|
"laughter": ("laugh", "paralinguistic"),
|
|
"breath": ("breath", "paralinguistic"),
|
|
"breathing": ("breath", "paralinguistic"),
|
|
"voice_activity": ("voice_activity", "speech_activity"),
|
|
"vad": ("voice_activity", "speech_activity"),
|
|
"speech_start": ("speech_start", "speech_activity"),
|
|
"speech_end": ("speech_end", "speech_activity"),
|
|
"speech_final": ("speech_final", "speech_activity"),
|
|
"stt_word": ("stt_word", "timing"),
|
|
"silence": ("silence", "timing"),
|
|
"pause": ("silence", "timing"),
|
|
"long_pause": ("silence", "timing"),
|
|
"speech_rate": ("speech_rate", "prosody"),
|
|
"fast_speech": ("speech_rate", "prosody"),
|
|
"slow_speech": ("speech_rate", "prosody"),
|
|
"pitch": ("pitch", "prosody"),
|
|
"intonation": ("intonation", "prosody"),
|
|
"prosody": ("prosody", "prosody"),
|
|
"noise": ("background_noise", "audio_quality"),
|
|
"background_noise": ("background_noise", "audio_quality"),
|
|
}
|
|
|
|
|
|
def _is_turn_persistence_unavailable(exc: Exception) -> bool:
|
|
if not isinstance(exc, HTTPException) or exc.status_code != 503:
|
|
return False
|
|
detail = str(exc.detail or "")
|
|
return "turn append" in detail and "persistence unavailable" in detail
|
|
|
|
|
|
_PROVIDER_EVENT_TYPE_FIELDS = ("event_type", "type", "kind", "label")
|
|
|
|
|
|
def _append_audio_chunk_with_cap(buffer: bytearray, chunk: bytes) -> bool:
|
|
"""Append only when the route-owned buffer remains within its hard cap."""
|
|
|
|
if len(chunk) > _MAX_AUDIO_BYTES - len(buffer):
|
|
return False
|
|
buffer.extend(chunk)
|
|
return True
|
|
|
|
|
|
@router.get("/health")
|
|
async def voice_health() -> JSONResponse:
|
|
"""Return voice service readiness."""
|
|
available = voice_service.is_available()
|
|
stt_available = voice_service.stt_available()
|
|
tts_available = voice_service.tts_available()
|
|
tts_provider = voice_service.tts_provider()
|
|
body = {
|
|
"status": "ok" if available else "degraded",
|
|
"available": available,
|
|
"stt_available": stt_available,
|
|
"tts_available": tts_available,
|
|
"stt_provider": voice_service.stt_provider(),
|
|
"stt_model": voice_service.stt_model(),
|
|
"stt_batch_fallback_available": voice_service.batch_stt_available(),
|
|
"tts_model": (
|
|
voice_svc.MELOTTS_TTS_MODEL
|
|
if tts_provider == "melotts"
|
|
else (
|
|
voice_svc.HIGGS_TTS_MODEL
|
|
if tts_provider == "higgs"
|
|
else voice_svc.TTS_MODEL
|
|
)
|
|
),
|
|
"tts_provider": tts_provider,
|
|
"limits": {
|
|
"max_utterance_audio_bytes": _MAX_AUDIO_BYTES,
|
|
"streaming_event_queue_max_items": _STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
|
"uvicorn_ws_max_queue": VOICE_UVICORN_WS_MAX_QUEUE,
|
|
},
|
|
"reason": (
|
|
None
|
|
if available
|
|
else "STT 또는 TTS provider가 준비되지 않았습니다"
|
|
),
|
|
}
|
|
return JSONResponse(body, status_code=200 if available else 503)
|
|
|
|
|
|
@router.post("/speech")
|
|
async def voice_speech(
|
|
body: VoiceSpeechRequest, principal: CurrentPrincipal
|
|
) -> Response:
|
|
"""Synthesize the persisted client reply for a completed text turn.
|
|
|
|
The browser sends only session/turn identifiers. The server reloads the
|
|
owned session and speaks the stored client-visible reply, so this endpoint
|
|
cannot be used as an arbitrary paid text-to-speech proxy.
|
|
"""
|
|
learner = principal
|
|
if learner.role != Role.LEARNER and learner.can_access_role(Role.LEARNER):
|
|
learner = learner.with_role(Role.LEARNER)
|
|
if learner.role != Role.LEARNER:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="only learners can use voice",
|
|
)
|
|
|
|
access_error = await _practice_access_error(learner)
|
|
if access_error is not None:
|
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=access_error)
|
|
sess, err = await turn_runtime.load_owned_session(
|
|
body.session_id,
|
|
learner,
|
|
allow_ended=False,
|
|
)
|
|
if sess is None:
|
|
status_code = {
|
|
turn_runtime.SessionAccessError.FORBIDDEN: status.HTTP_403_FORBIDDEN,
|
|
turn_runtime.SessionAccessError.ENDED: status.HTTP_409_CONFLICT,
|
|
}.get(err, status.HTTP_404_NOT_FOUND)
|
|
raise HTTPException(
|
|
status_code=status_code, detail=f"voice session {err or 'not_found'}"
|
|
)
|
|
|
|
text = _client_turn_text_for_speech(sess, body.turn_seq)
|
|
if text is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="client reply not found for turn",
|
|
)
|
|
|
|
voice_preset = await _resolve_session_voice(
|
|
session_id=body.session_id,
|
|
persona_code=sess.persona.code,
|
|
explicit_preset=None,
|
|
)
|
|
if not voice_service.tts_available(voice_preset):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail="TTS provider is not configured",
|
|
)
|
|
try:
|
|
chunks = [
|
|
chunk.audio
|
|
async for chunk in voice_service.synthesize_stream(text, voice_preset)
|
|
]
|
|
except VoiceUnavailable as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=str(exc),
|
|
) from exc
|
|
except Exception as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_502_BAD_GATEWAY,
|
|
detail=f"TTS failed: {exc}",
|
|
) from exc
|
|
|
|
audio = b"".join(chunks)
|
|
if not audio:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="client reply has no speakable text",
|
|
)
|
|
return Response(
|
|
content=audio,
|
|
media_type=voice_service.tts_media_type_for_voice(voice_preset),
|
|
headers={
|
|
"Cache-Control": "no-store",
|
|
"X-Vignette-TTS-Model": voice_service.tts_model_for_voice(voice_preset),
|
|
"X-Vignette-TTS-Provider": voice_service.tts_provider_for_voice(voice_preset),
|
|
},
|
|
)
|
|
|
|
|
|
@router.websocket("/ws")
|
|
async def voice_ws(websocket: WebSocket) -> None:
|
|
"""Run one authenticated learner voice cascade."""
|
|
await websocket.accept()
|
|
|
|
# Authenticate the same server-side browser session used by REST routes.
|
|
principal = await _principal_from_websocket(websocket)
|
|
if principal is None:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": "not authenticated"}
|
|
)
|
|
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
|
|
return
|
|
if principal.role != Role.LEARNER and principal.can_access_role(Role.LEARNER):
|
|
principal = principal.with_role(Role.LEARNER)
|
|
if principal.role != Role.LEARNER:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": "only learners can use voice"}
|
|
)
|
|
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
|
|
return
|
|
|
|
# Bind to an existing session first. persona_code creation is dev-only.
|
|
session_id, voice_preset, err, bind_meta = await _bind_session(websocket, principal)
|
|
if err is not None:
|
|
await _safe_send_json(websocket, {"type": "error", "detail": err})
|
|
await _safe_close(websocket, WS_CLOSE_BAD_REQUEST)
|
|
return
|
|
assert session_id is not None and voice_preset is not None
|
|
if bind_meta.get("degraded"):
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "degraded",
|
|
"reason": bind_meta.get(
|
|
"degraded_reason", "voice session binding degraded"
|
|
),
|
|
**bind_meta,
|
|
},
|
|
)
|
|
|
|
# Voice misconfiguration is reported explicitly and then closed cleanly.
|
|
if not voice_service.is_available():
|
|
await _safe_send_json(
|
|
websocket,
|
|
{"type": "degraded", "reason": "voice STT/TTS is not configured"},
|
|
)
|
|
await _safe_close(websocket, WS_CLOSE_DEGRADED)
|
|
return
|
|
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "ready",
|
|
"session_id": session_id,
|
|
"voice": voice_preset.openai_voice,
|
|
"preset": voice_preset.preset,
|
|
"stt_provider": voice_service.stt_provider(),
|
|
"stt_model": voice_service.stt_model(),
|
|
"stt_batch_fallback_available": voice_service.batch_stt_available(),
|
|
"tts_provider": voice_service.tts_provider_for_voice(voice_preset),
|
|
"tts_model": voice_service.tts_model_for_voice(voice_preset),
|
|
"limits": {
|
|
"max_utterance_audio_bytes": _MAX_AUDIO_BYTES,
|
|
"streaming_event_queue_max_items": _STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
|
"uvicorn_ws_max_queue": VOICE_UVICORN_WS_MAX_QUEUE,
|
|
},
|
|
"state": "idle",
|
|
**bind_meta,
|
|
},
|
|
)
|
|
|
|
runtime_connection_id = voice_runtime_metrics.websocket_opened()
|
|
|
|
audio_buf = bytearray()
|
|
receiving = False
|
|
audio_started_at: float | None = None
|
|
last_audio_end_at: float | None = None
|
|
audio_format: str | None = None
|
|
audio_sample_rate: int | None = None
|
|
audio_channels: int | None = None
|
|
audio_sample_width: int | None = None
|
|
streaming_session: voice_svc.DeepgramStreamingSession | None = None
|
|
streaming_consent_checked_at: float | None = None
|
|
streaming_events: asyncio.Queue[StreamingTranscriptEvent] = asyncio.Queue(
|
|
maxsize=_STREAMING_EVENT_QUEUE_MAX_ITEMS
|
|
)
|
|
discard_audio_until_end = False
|
|
last_stream_transcript: tuple[str, bool] | None = None
|
|
|
|
try:
|
|
while True:
|
|
msg = await websocket.receive()
|
|
mtype = msg.get("type")
|
|
if mtype == "websocket.disconnect":
|
|
break
|
|
|
|
# Binary frames are audio chunks.
|
|
if msg.get("bytes") is not None:
|
|
if discard_audio_until_end:
|
|
continue
|
|
if not receiving:
|
|
# Be tolerant when audio arrives before audio_start.
|
|
receiving = True
|
|
audio_started_at = time.monotonic()
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(
|
|
runtime_connection_id
|
|
)
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "listening"}
|
|
)
|
|
chunk = msg["bytes"]
|
|
if not _append_audio_chunk_with_cap(audio_buf, chunk):
|
|
voice_runtime_metrics.audio_overflow_rejected(
|
|
runtime_connection_id
|
|
)
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"detail": "audio too large; please send a shorter utterance",
|
|
},
|
|
)
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "idle"}
|
|
)
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(
|
|
runtime_connection_id
|
|
)
|
|
receiving = False
|
|
discard_audio_until_end = True
|
|
continue
|
|
voice_runtime_metrics.audio_chunk_received(
|
|
runtime_connection_id,
|
|
current_buffer_bytes=len(audio_buf),
|
|
chunk_bytes=len(chunk),
|
|
)
|
|
if streaming_session is not None:
|
|
now = time.monotonic()
|
|
if (
|
|
streaming_consent_checked_at is None
|
|
or now - streaming_consent_checked_at
|
|
>= _STREAMING_CONSENT_RECHECK_SECONDS
|
|
):
|
|
if not await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
):
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
discard_audio_until_end = True
|
|
receiving = False
|
|
continue
|
|
streaming_consent_checked_at = now
|
|
try:
|
|
await streaming_session.send_audio(chunk)
|
|
await asyncio.sleep(0)
|
|
drained = await _drain_streaming_transcripts(
|
|
websocket, streaming_events
|
|
)
|
|
voice_runtime_metrics.streaming_queue_observed(
|
|
runtime_connection_id,
|
|
queue_items=streaming_events.qsize(),
|
|
)
|
|
if drained is not None:
|
|
last_stream_transcript = drained
|
|
except Exception:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
if voice_service.batch_stt_available():
|
|
voice_runtime_metrics.provider_fallback()
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "degraded",
|
|
"reason": "streaming STT unavailable; using batch fallback",
|
|
},
|
|
)
|
|
else:
|
|
discard_audio_until_end = True
|
|
receiving = False
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "streaming_stt_unavailable",
|
|
"detail": "streaming STT failed and no batch fallback is configured",
|
|
},
|
|
)
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "idle"}
|
|
)
|
|
continue
|
|
|
|
# Text frames are JSON controls.
|
|
text = msg.get("text")
|
|
if text is None:
|
|
continue
|
|
try:
|
|
ctrl = json.loads(text)
|
|
except (json.JSONDecodeError, TypeError):
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": "invalid control json"}
|
|
)
|
|
continue
|
|
|
|
ctype = ctrl.get("type")
|
|
if ctype == "audio_start":
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
receiving = True
|
|
discard_audio_until_end = False
|
|
last_stream_transcript = None
|
|
while not streaming_events.empty():
|
|
streaming_events.get_nowait()
|
|
voice_runtime_metrics.streaming_queue_observed(
|
|
runtime_connection_id,
|
|
queue_items=0,
|
|
)
|
|
audio_started_at = time.monotonic()
|
|
audio_format = _safe_str(ctrl.get("format"))
|
|
audio_sample_rate = _safe_int(ctrl.get("sample_rate"))
|
|
audio_channels = _safe_int(ctrl.get("channels"))
|
|
audio_sample_width = _safe_int(ctrl.get("sample_width"))
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
|
if voice_service.can_stream_audio(
|
|
fmt=audio_format,
|
|
sample_rate=audio_sample_rate,
|
|
channels=audio_channels,
|
|
sample_width=audio_sample_width,
|
|
):
|
|
if not await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
):
|
|
receiving = False
|
|
discard_audio_until_end = True
|
|
else:
|
|
async def queue_streaming_event(
|
|
event: StreamingTranscriptEvent,
|
|
) -> None:
|
|
queue_was_full = streaming_events.full()
|
|
queue_wait_started = time.perf_counter()
|
|
await streaming_events.put(event)
|
|
voice_runtime_metrics.streaming_queue_observed(
|
|
runtime_connection_id,
|
|
queue_items=streaming_events.qsize(),
|
|
saturated=queue_was_full,
|
|
wait_seconds=(
|
|
time.perf_counter() - queue_wait_started
|
|
),
|
|
)
|
|
|
|
try:
|
|
streaming_session = (
|
|
await voice_service.open_streaming_transcription(
|
|
fmt=audio_format,
|
|
sample_rate=audio_sample_rate,
|
|
channels=audio_channels,
|
|
sample_width=audio_sample_width,
|
|
on_event=queue_streaming_event,
|
|
)
|
|
)
|
|
streaming_consent_checked_at = time.monotonic()
|
|
except Exception:
|
|
if voice_service.batch_stt_available():
|
|
voice_runtime_metrics.provider_fallback()
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "degraded",
|
|
"reason": "streaming STT unavailable; using batch fallback",
|
|
},
|
|
)
|
|
else:
|
|
receiving = False
|
|
discard_audio_until_end = True
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "streaming_stt_unavailable",
|
|
"detail": "streaming STT is unavailable and no batch fallback is configured",
|
|
},
|
|
)
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "idle"}
|
|
)
|
|
if not discard_audio_until_end:
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "listening"}
|
|
)
|
|
|
|
elif ctype == "audio_end":
|
|
receiving = False
|
|
audio_ended_at = time.monotonic()
|
|
silence_ms = _safe_int(ctrl.get("silence_ms"))
|
|
if (
|
|
silence_ms is None
|
|
and last_audio_end_at is not None
|
|
and audio_started_at is not None
|
|
):
|
|
silence_ms = max(
|
|
0, int((audio_started_at - last_audio_end_at) * 1000)
|
|
)
|
|
end_format = _safe_str(ctrl.get("format")) or audio_format
|
|
context = VoiceSessionContext(session_id, principal, voice_preset)
|
|
utterance = VoiceAudioInput(
|
|
audio=bytes(audio_buf),
|
|
fmt=end_format,
|
|
sample_rate=_safe_int(ctrl.get("sample_rate"))
|
|
or audio_sample_rate,
|
|
channels=_safe_int(ctrl.get("channels")) or audio_channels,
|
|
sample_width=_safe_int(ctrl.get("sample_width"))
|
|
or audio_sample_width,
|
|
audio_started_at=audio_started_at,
|
|
audio_ended_at=audio_ended_at,
|
|
prosody=VoiceProsody(
|
|
silence_ms=silence_ms,
|
|
barge_in=_safe_bool(ctrl.get("barge_in")),
|
|
provider_events=_safe_provider_events(
|
|
ctrl.get("provider_events")
|
|
),
|
|
),
|
|
)
|
|
if discard_audio_until_end:
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
elif streaming_session is not None:
|
|
if not await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
):
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
else:
|
|
try:
|
|
streaming_result, drained = (
|
|
await _finish_streaming_transcription(
|
|
websocket,
|
|
streaming_session,
|
|
streaming_events,
|
|
)
|
|
)
|
|
voice_runtime_metrics.streaming_queue_observed(
|
|
runtime_connection_id,
|
|
queue_items=streaming_events.qsize(),
|
|
)
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
except Exception:
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
if voice_service.batch_stt_available():
|
|
voice_runtime_metrics.provider_fallback()
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "degraded",
|
|
"reason": "streaming STT unavailable; using batch fallback",
|
|
},
|
|
)
|
|
await _handle_utterance(websocket, context, utterance)
|
|
else:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "streaming_stt_unavailable",
|
|
"detail": "streaming STT finalization failed and no batch fallback is configured",
|
|
},
|
|
)
|
|
await _safe_send_json(
|
|
websocket, {"type": "state", "state": "idle"}
|
|
)
|
|
else:
|
|
if drained is not None:
|
|
last_stream_transcript = drained
|
|
if await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
):
|
|
await _handle_streaming_utterance(
|
|
websocket,
|
|
context,
|
|
utterance,
|
|
streaming_result,
|
|
transcript_already_sent=(
|
|
last_stream_transcript
|
|
== (streaming_result.text, True)
|
|
),
|
|
)
|
|
else:
|
|
await _handle_utterance(websocket, context, utterance)
|
|
last_audio_end_at = audio_ended_at
|
|
audio_started_at = None
|
|
audio_format = None
|
|
audio_sample_rate = None
|
|
audio_channels = None
|
|
audio_sample_width = None
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
|
discard_audio_until_end = False
|
|
last_stream_transcript = None
|
|
|
|
elif ctype == "text_turn":
|
|
# Text-only path for accessibility and deterministic tests.
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
receiving = False
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
|
learner_text = (ctrl.get("text") or "").strip()
|
|
if learner_text:
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
VoiceSessionContext(session_id, principal, voice_preset),
|
|
VoiceTurnInput(learner_text=learner_text),
|
|
)
|
|
|
|
elif ctype == "stt_result":
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
receiving = False
|
|
audio_buf.clear()
|
|
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
|
stt_received_at = time.monotonic()
|
|
await _handle_stt_result_control(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
voice_preset=voice_preset,
|
|
ctrl=ctrl,
|
|
audio_started_at=audio_started_at,
|
|
audio_ended_at=stt_received_at,
|
|
last_audio_end_at=last_audio_end_at,
|
|
)
|
|
last_audio_end_at = stt_received_at
|
|
audio_started_at = None
|
|
audio_format = None
|
|
audio_sample_rate = None
|
|
audio_channels = None
|
|
audio_sample_width = None
|
|
|
|
elif ctype == "ping":
|
|
await _safe_send_json(websocket, {"type": "pong"})
|
|
|
|
elif ctype == "close":
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
streaming_session = None
|
|
streaming_consent_checked_at = None
|
|
break
|
|
|
|
except WebSocketDisconnect:
|
|
pass
|
|
except Exception as e:
|
|
voice_runtime_metrics.websocket_error()
|
|
if _is_turn_persistence_unavailable(e):
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "turn_persistence_unavailable",
|
|
"detail": "voice turn persistence unavailable; retry the utterance",
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
else:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": f"voice ws error: {e}"}
|
|
)
|
|
finally:
|
|
if streaming_session is not None:
|
|
await streaming_session.abort()
|
|
voice_runtime_metrics.websocket_closed(runtime_connection_id)
|
|
await _safe_close(websocket)
|
|
|
|
|
|
async def _multimodal_voice_processing_allowed(
|
|
websocket: WebSocket,
|
|
*,
|
|
session_id: str,
|
|
principal: Principal,
|
|
) -> bool:
|
|
"""Stop before STT/derived processing when G7 consent is not active."""
|
|
|
|
try:
|
|
await multimodal_alliance_store.assert_voice_processing_allowed(
|
|
principal=principal,
|
|
session_id=session_id,
|
|
)
|
|
except multimodal_alliance_store.MultimodalConsentWithdrawnError as exc:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_consent_withdrawn",
|
|
"detail": str(exc),
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return False
|
|
except multimodal_alliance_store.MultimodalConsentRequiredError as exc:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_consent_required",
|
|
"detail": str(exc),
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return False
|
|
except (multimodal_alliance_store.MultimodalAllianceError, RuntimeError):
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_consent_unavailable",
|
|
"detail": "multimodal consent state is unavailable; voice processing is blocked",
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return False
|
|
return True
|
|
|
|
|
|
async def _handle_stt_result_control(
|
|
websocket: WebSocket,
|
|
*,
|
|
session_id: str,
|
|
principal: Principal,
|
|
voice_preset: VoicePreset,
|
|
ctrl: dict[str, object],
|
|
audio_started_at: float | None = None,
|
|
audio_ended_at: float | None = None,
|
|
last_audio_end_at: float | None = None,
|
|
) -> None:
|
|
if not await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=session_id,
|
|
principal=principal,
|
|
):
|
|
return
|
|
learner_text = str(ctrl.get("text") or "").strip()
|
|
transcript_final = _safe_bool(ctrl.get("final"))
|
|
silence_ms = _safe_int(ctrl.get("silence_ms"))
|
|
if (
|
|
silence_ms is None
|
|
and last_audio_end_at is not None
|
|
and audio_started_at is not None
|
|
):
|
|
silence_ms = max(0, int((audio_started_at - last_audio_end_at) * 1000))
|
|
provider_events = _safe_provider_events(ctrl.get("provider_events"))
|
|
decision = voice_svc.assess_end_of_turn(
|
|
transcript_text=learner_text,
|
|
transcript_final=bool(transcript_final),
|
|
silence_ms=silence_ms,
|
|
)
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "eot",
|
|
"ready": decision.ready,
|
|
"reason": decision.reason,
|
|
"silence_ms": decision.silence_ms,
|
|
"threshold_ms": decision.threshold_ms,
|
|
},
|
|
)
|
|
if not decision.ready:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "listening"})
|
|
return
|
|
|
|
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "transcript",
|
|
"text": learner_text,
|
|
"final": True,
|
|
"speaker": "counselor",
|
|
},
|
|
)
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
VoiceSessionContext(session_id, principal, voice_preset),
|
|
VoiceTurnInput(
|
|
learner_text=learner_text,
|
|
prosody=VoiceProsody(
|
|
duration_s=_elapsed_seconds(audio_started_at, audio_ended_at),
|
|
silence_ms=decision.silence_ms,
|
|
barge_in=_safe_bool(ctrl.get("barge_in")),
|
|
provider_events=provider_events,
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
async def _handle_utterance(
|
|
websocket: WebSocket,
|
|
context: VoiceSessionContext,
|
|
utterance: VoiceAudioInput,
|
|
) -> None:
|
|
"""Transcribe one utterance, generate the client reply, then synthesize TTS."""
|
|
if not utterance.audio:
|
|
await _safe_send_json(
|
|
websocket, {"type": "transcript", "text": "", "final": True}
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
if not await _multimodal_voice_processing_allowed(
|
|
websocket,
|
|
session_id=context.session_id,
|
|
principal=context.principal,
|
|
):
|
|
return
|
|
|
|
# STT begins after the learner stops speaking.
|
|
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
|
upload_audio, upload_fmt = _normalize_audio_upload(
|
|
utterance.audio,
|
|
fmt=utterance.fmt,
|
|
sample_rate=utterance.sample_rate,
|
|
channels=utterance.channels,
|
|
sample_width=utterance.sample_width,
|
|
)
|
|
filename, content_type = _audio_meta(upload_fmt)
|
|
try:
|
|
stt = await voice_service.transcribe(
|
|
upload_audio, filename=filename, content_type=content_type
|
|
)
|
|
except VoiceUnavailable as e:
|
|
await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)})
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
except Exception as e:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": f"STT failed: {e}"}
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
learner_text = stt.text
|
|
audio_ref = _voice_audio_ref(upload_audio, upload_fmt)
|
|
duration_s = stt.duration or _elapsed_seconds(
|
|
utterance.audio_started_at, utterance.audio_ended_at
|
|
)
|
|
speech_rate = _estimate_speech_rate(learner_text, duration_s)
|
|
provider_events = _merge_provider_events(
|
|
utterance.prosody.provider_events,
|
|
getattr(stt, "provider_events", []),
|
|
)
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "transcript",
|
|
"text": learner_text,
|
|
"final": True,
|
|
"speaker": "counselor",
|
|
},
|
|
)
|
|
if not learner_text:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
context,
|
|
VoiceTurnInput(
|
|
learner_text=learner_text,
|
|
prosody=VoiceProsody(
|
|
audio_ref=audio_ref,
|
|
duration_s=duration_s,
|
|
silence_ms=utterance.prosody.silence_ms,
|
|
speech_rate=speech_rate,
|
|
barge_in=utterance.prosody.barge_in,
|
|
provider_events=provider_events,
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
async def _drain_streaming_transcripts(
|
|
websocket: WebSocket,
|
|
events: asyncio.Queue[StreamingTranscriptEvent],
|
|
) -> tuple[str, bool] | None:
|
|
"""Relay provider-neutral streaming updates without concurrent ASGI sends."""
|
|
|
|
last: tuple[str, bool] | None = None
|
|
while True:
|
|
try:
|
|
event = events.get_nowait()
|
|
except asyncio.QueueEmpty:
|
|
return last
|
|
await _relay_streaming_transcript(websocket, event)
|
|
last = (event.text, event.speech_final)
|
|
|
|
|
|
async def _finish_streaming_transcription(
|
|
websocket: WebSocket,
|
|
session: voice_svc.DeepgramStreamingSession,
|
|
events: asyncio.Queue[StreamingTranscriptEvent],
|
|
) -> tuple[TranscriptResult, tuple[str, bool] | None]:
|
|
"""Finalize while draining the bounded event queue to avoid producer deadlock."""
|
|
|
|
finish_task = asyncio.create_task(session.finish())
|
|
last: tuple[str, bool] | None = None
|
|
try:
|
|
while not finish_task.done():
|
|
event_task = asyncio.create_task(events.get())
|
|
done, _ = await asyncio.wait(
|
|
{finish_task, event_task},
|
|
return_when=asyncio.FIRST_COMPLETED,
|
|
)
|
|
if event_task in done:
|
|
event = event_task.result()
|
|
await _relay_streaming_transcript(websocket, event)
|
|
last = (event.text, event.speech_final)
|
|
else:
|
|
event_task.cancel()
|
|
try:
|
|
await event_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
result = await finish_task
|
|
drained = await _drain_streaming_transcripts(websocket, events)
|
|
return result, drained or last
|
|
except Exception:
|
|
if not finish_task.done():
|
|
finish_task.cancel()
|
|
raise
|
|
|
|
|
|
async def _relay_streaming_transcript(
|
|
websocket: WebSocket, event: StreamingTranscriptEvent
|
|
) -> None:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "transcript",
|
|
"text": event.text,
|
|
# Deepgram is_final seals one segment; speech_final seals the
|
|
# learner utterance. The browser's `final` contract means the
|
|
# latter so it never stops capture at an intermediate segment.
|
|
"final": event.speech_final,
|
|
"speech_final": event.speech_final,
|
|
"speaker": "counselor",
|
|
},
|
|
)
|
|
|
|
|
|
async def _handle_streaming_utterance(
|
|
websocket: WebSocket,
|
|
context: VoiceSessionContext,
|
|
utterance: VoiceAudioInput,
|
|
stt: TranscriptResult,
|
|
*,
|
|
transcript_already_sent: bool,
|
|
) -> None:
|
|
"""Persist and run one finalized provider-streamed learner utterance."""
|
|
|
|
learner_text = stt.text.strip()
|
|
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
|
if not transcript_already_sent:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "transcript",
|
|
"text": learner_text,
|
|
"final": True,
|
|
"speech_final": True,
|
|
"speaker": "counselor",
|
|
},
|
|
)
|
|
if not learner_text:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
audio_ref = _voice_audio_ref(utterance.audio, utterance.fmt)
|
|
duration_s = stt.duration or _elapsed_seconds(
|
|
utterance.audio_started_at, utterance.audio_ended_at
|
|
)
|
|
if stt.words:
|
|
duration_s = max(duration_s or 0.0, max(word.end for word in stt.words))
|
|
speech_rate = _estimate_speech_rate(learner_text, duration_s)
|
|
provider_events = _merge_provider_events(
|
|
[
|
|
{
|
|
"type": "stt_metadata",
|
|
"provider": "deepgram",
|
|
"model": stt.model,
|
|
"source": "streaming_stt",
|
|
"is_final": True,
|
|
}
|
|
],
|
|
utterance.prosody.provider_events,
|
|
stt.provider_events,
|
|
)
|
|
try:
|
|
await _persist_streaming_timeline(
|
|
context=context,
|
|
utterance=utterance,
|
|
stt=stt,
|
|
audio_ref=audio_ref,
|
|
duration_s=duration_s,
|
|
)
|
|
except multimodal_alliance_store.MultimodalConsentWithdrawnError as exc:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_consent_withdrawn",
|
|
"detail": str(exc),
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
except multimodal_alliance_store.MultimodalConsentRequiredError as exc:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_consent_required",
|
|
"detail": str(exc),
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
except multimodal_alliance_store.MultimodalAllianceError:
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "error",
|
|
"code": "multimodal_timeline_unavailable",
|
|
"detail": "multimodal timeline persistence is unavailable; voice turn is blocked",
|
|
},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
await _run_turn_and_speak(
|
|
websocket,
|
|
context,
|
|
VoiceTurnInput(
|
|
learner_text=learner_text,
|
|
prosody=VoiceProsody(
|
|
audio_ref=audio_ref,
|
|
duration_s=duration_s,
|
|
silence_ms=utterance.prosody.silence_ms,
|
|
speech_rate=speech_rate,
|
|
barge_in=utterance.prosody.barge_in,
|
|
provider_events=provider_events,
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
async def _persist_streaming_timeline(
|
|
*,
|
|
context: VoiceSessionContext,
|
|
utterance: VoiceAudioInput,
|
|
stt: TranscriptResult,
|
|
audio_ref: str,
|
|
duration_s: float | None,
|
|
) -> dict[str, object] | None:
|
|
audio_sha256 = hashlib.sha256(utterance.audio).hexdigest()
|
|
submission_id = uuid5(
|
|
NAMESPACE_URL,
|
|
f"vignette:g7:streaming-stt:{context.session_id}:{audio_sha256}:{stt.model}",
|
|
)
|
|
duration_ms = max(
|
|
1,
|
|
round((duration_s or 0.0) * 1000),
|
|
*(round(word.end * 1000) for word in stt.words),
|
|
)
|
|
words = []
|
|
for index, word in enumerate(sorted(stt.words, key=lambda item: item.start)):
|
|
start_ms = max(0, min(duration_ms - 1, round(word.start * 1000)))
|
|
end_ms = max(start_ms + 1, min(duration_ms, round(word.end * 1000)))
|
|
words.append(
|
|
{
|
|
"word_index": index,
|
|
"start_ms": start_ms,
|
|
"end_ms": end_ms,
|
|
"speaker": "learner",
|
|
# Common counselling words are dictionary-attackable when stored
|
|
# as plain SHA-256. Bind the pseudonym to this deployment and
|
|
# submission so the timeline remains useful without creating a
|
|
# reusable transcript fingerprint.
|
|
"token_hash": hmac.new(
|
|
settings.session_secret.encode("utf-8"),
|
|
(
|
|
f"{context.session_id}:{submission_id}:"
|
|
f"{word.word.casefold()}"
|
|
).encode("utf-8"),
|
|
hashlib.sha256,
|
|
).hexdigest(),
|
|
}
|
|
)
|
|
|
|
events = []
|
|
for index, event in enumerate(_safe_provider_events(stt.provider_events)):
|
|
provider_type = str(event.get("event_type") or "")
|
|
event_type = _g7_event_type(provider_type)
|
|
if event_type is None:
|
|
continue
|
|
start_ms = max(0, _safe_int(event.get("start_ms")) or 0)
|
|
end_ms = _safe_int(event.get("end_ms"))
|
|
if end_ms is None:
|
|
end_ms = start_ms + max(0, _safe_int(event.get("duration_ms")) or 0)
|
|
start_ms = min(start_ms, duration_ms - 1)
|
|
end_ms = min(duration_ms, max(start_ms + 1, end_ms))
|
|
confidence = _safe_float(event.get("confidence"))
|
|
uncertainty = 0.5 if confidence is None else max(0.0, min(1.0, 1.0 - confidence))
|
|
events.append(
|
|
{
|
|
"event_id": f"oas-g7-event-{submission_id.hex}-{index}",
|
|
"event_type": event_type,
|
|
"start_ms": start_ms,
|
|
"end_ms": end_ms,
|
|
"actor": "learner",
|
|
"observed_feature": f"provider observed {provider_type}",
|
|
"uncertainty": uncertainty,
|
|
"source": "stt_word_timestamps",
|
|
}
|
|
)
|
|
|
|
timeline = multimodal_alliance.align_voice_timeline(
|
|
audio_duration_ms=duration_ms,
|
|
words=words,
|
|
events=events,
|
|
)
|
|
_, media_type = _audio_meta(utterance.fmt)
|
|
return await multimodal_alliance_store.append_runtime_timeline(
|
|
session_id=context.session_id,
|
|
submission_id=submission_id,
|
|
timeline=timeline,
|
|
audio_asset={
|
|
"audio_ref": audio_ref,
|
|
"audio_sha256": audio_sha256,
|
|
"media_type": media_type,
|
|
"byte_size": len(utterance.audio),
|
|
},
|
|
)
|
|
|
|
|
|
def _g7_event_type(provider_type: str) -> str | None:
|
|
if provider_type in {"speech_final", "speech_end", "voice_activity", "speech_rate"}:
|
|
return "pace"
|
|
if provider_type in {"barge_in", "interrupt", "interruption"}:
|
|
return "interruption"
|
|
if provider_type == "overlap":
|
|
return "overlap"
|
|
if provider_type in {"silence", "pause", "long_pause"}:
|
|
return "silence"
|
|
if provider_type in {"background_noise", "noise"}:
|
|
return "audio_quality"
|
|
if provider_type in {"sigh", "cry", "laugh", "breath", "pitch", "intonation", "prosody"}:
|
|
return "prosody"
|
|
return None
|
|
|
|
|
|
async def _run_turn_and_speak(
|
|
websocket: WebSocket,
|
|
context: VoiceSessionContext,
|
|
turn: VoiceTurnInput,
|
|
) -> None:
|
|
"""Run one counseling turn and stream synthesized client speech."""
|
|
learner_text = turn.learner_text
|
|
prosody = turn.prosody
|
|
speech_rate = prosody.speech_rate
|
|
if speech_rate is None:
|
|
speech_rate = _estimate_speech_rate(learner_text, prosody.duration_s)
|
|
sess, err = await _load_voice_session(context.session_id, context.principal)
|
|
if sess is None:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": err or "session not found or ended"}
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
from . import sessions as session_routes
|
|
|
|
if session_routes.session_time_over(sess):
|
|
await _safe_send_json(
|
|
websocket,
|
|
{"type": "error", "detail": "session_time_over"},
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
ctx = await _prepare_voice_turn_context(
|
|
session_id=context.session_id,
|
|
sess=sess,
|
|
learner_text=learner_text,
|
|
)
|
|
assert ctx.state_after is not None
|
|
|
|
# Voice needs the full client reply before TTS starts.
|
|
try:
|
|
result = await orchestrator.run_turn_generate(
|
|
ctx,
|
|
engine_client,
|
|
eval_hook=evaluator.make_eval_hook(
|
|
engine_client,
|
|
audit_hook=session_persistence.record_llm_call_audit,
|
|
),
|
|
audit_hook=session_persistence.record_llm_call_audit,
|
|
)
|
|
except EngineError as e:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": f"engine unavailable: {e}"}
|
|
)
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
reply = result.client_reply or ""
|
|
# Persist only after the client reply has been generated. A failed AI turn
|
|
# must not leave a learner-only transcript in review or history.
|
|
await turn_runtime.finalize_completed_turn(
|
|
sess,
|
|
ctx,
|
|
result,
|
|
context_prefix="voice session",
|
|
counselor_turn=TurnRecord(
|
|
turn_seq=ctx.state_after.turn_seq,
|
|
speaker="counselor",
|
|
stage=turn_runtime.stage_label(ctx.state_after.stage),
|
|
text=learner_text,
|
|
text_masked=ctx.learner_text_masked,
|
|
audio_ref=prosody.audio_ref,
|
|
silence_ms=prosody.silence_ms,
|
|
speech_rate=speech_rate,
|
|
barge_in=prosody.barge_in,
|
|
provider_events=prosody.provider_events,
|
|
evaluation=result.evaluation,
|
|
),
|
|
)
|
|
|
|
# Send the final client text before audio playback.
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "reply",
|
|
"text": reply,
|
|
"speaker": "client",
|
|
"stage": result.stage,
|
|
"effective_openness": round(result.effective_openness, 4),
|
|
"turn_seq": result.turn_seq,
|
|
"safety_flagged": result.safety_flagged,
|
|
"crisis_kind": result.crisis_kind,
|
|
"crisis_resource": result.crisis_resource,
|
|
"conversation_stopped": result.conversation_stopped,
|
|
"progress": session_routes.build_session_progress(
|
|
result.state_after,
|
|
prev_rapport_credit=sess.prev_rapport_credit,
|
|
goal_stages=list(sess.goal_stages or []),
|
|
).model_dump(),
|
|
},
|
|
)
|
|
|
|
if not reply:
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
return
|
|
|
|
# TTS speaking state comes before chunk metadata and binary audio.
|
|
await _safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "state",
|
|
"state": "speaking",
|
|
"voice": context.voice_preset.openai_voice,
|
|
"tts_provider": voice_service.tts_provider_for_voice(context.voice_preset),
|
|
},
|
|
)
|
|
try:
|
|
n = 0
|
|
async for ck in voice_service.synthesize_stream(reply, context.voice_preset):
|
|
# 바이너리 오디오 청크만 송신(프론트가 Web Audio AnalyserNode로 립싱크 자체 산출).
|
|
await _safe_send_bytes(websocket, ck.audio)
|
|
n += 1
|
|
await _safe_send_json(websocket, {"type": "tts_end", "chunks": n})
|
|
except VoiceUnavailable as e:
|
|
await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)})
|
|
except Exception as e:
|
|
await _safe_send_json(
|
|
websocket, {"type": "error", "detail": f"TTS failed: {e}"}
|
|
)
|
|
|
|
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
|
|
|
|
|
async def _prepare_voice_turn_context(
|
|
*,
|
|
session_id: str,
|
|
sess: InProcSession,
|
|
learner_text: str,
|
|
) -> orchestrator.TurnContext:
|
|
"""Build voice turn context with the same fail-closed G3 ledger projection."""
|
|
|
|
from . import sessions as session_routes
|
|
|
|
recall = await session_routes.ensure_recall_context(sess)
|
|
kb_cues = session_routes.cached_kb_cues(session_id)
|
|
scenario_context = (
|
|
await rupture_scenario_director.load_stored_scenario_context(
|
|
session_id=session_id,
|
|
case_id=sess.case_id,
|
|
)
|
|
)
|
|
return orchestrator.prepare_turn(
|
|
session_id=session_id,
|
|
case_id=sess.case_id,
|
|
card=sess.persona,
|
|
state=sess.state,
|
|
learner_text=learner_text,
|
|
memory=orchestrator.TurnMemory(
|
|
recall_summary=recall.recall_summary,
|
|
pinned_facts=recall.pinned_facts,
|
|
recent_turns=sess.recent_turns(visible_to="client"),
|
|
kb_behavior_cues=kb_cues,
|
|
),
|
|
theory_mode=sess.theory_mode,
|
|
scenario_context=scenario_context,
|
|
)
|
|
|
|
|
|
async def _load_voice_session(
|
|
session_id: str,
|
|
principal: Principal,
|
|
) -> tuple[InProcSession | None, str | None]:
|
|
sess, err = await turn_runtime.load_owned_session(session_id, principal)
|
|
if err == turn_runtime.SessionAccessError.NOT_FOUND:
|
|
return None, f"unknown session {session_id}"
|
|
if err == turn_runtime.SessionAccessError.FORBIDDEN:
|
|
return None, "session does not belong to user"
|
|
if err == turn_runtime.SessionAccessError.ENDED:
|
|
return None, "session already ended"
|
|
assert sess is not None
|
|
return sess, None
|
|
|
|
|
|
def _client_turn_text_for_speech(sess: InProcSession, turn_seq: int) -> str | None:
|
|
"""완료된 상담 턴 번호에 대응하는 client-visible 응답을 반환한다.
|
|
|
|
degraded 인메모리 미러는 상담자/내담자 한 쌍이 같은 논리 turn_seq를 쓰지만,
|
|
DB의 app.turns.seq는 발화마다 1씩 증가한다. DB 스냅샷에서 논리 1턴을
|
|
그대로 seq=1로 찾으면 상담자 발화만 잡혀 TTS가 404가 되므로 두 저장 형태를
|
|
명시적으로 구분한다.
|
|
"""
|
|
visible_clients = [
|
|
turn
|
|
for turn in sess.turns
|
|
if turn.speaker == "client" and turn.is_visible_to("client")
|
|
]
|
|
counselor_sequences = {
|
|
turn.turn_seq for turn in sess.turns if turn.speaker == "counselor"
|
|
}
|
|
paired_sequences = counselor_sequences.intersection(
|
|
turn.turn_seq for turn in visible_clients
|
|
)
|
|
if paired_sequences or not counselor_sequences:
|
|
match = next(
|
|
(turn for turn in reversed(visible_clients) if turn.turn_seq == turn_seq),
|
|
None,
|
|
)
|
|
else:
|
|
index = turn_seq - 1
|
|
match = visible_clients[index] if 0 <= index < len(visible_clients) else None
|
|
if match is not None:
|
|
text = (match.text_masked or match.text).strip()
|
|
return guardrail.humanize_pii_placeholders(text) or None
|
|
return None
|
|
|
|
|
|
async def _principal_from_websocket(websocket: WebSocket) -> Principal | None:
|
|
"""Restore the same server-side browser session used by REST routes."""
|
|
raw_cookie = websocket.cookies.get(settings.cookie_name)
|
|
if raw_cookie is None and settings.environment == "dev":
|
|
raw_cookie = websocket.cookies.get("vignette_sid")
|
|
|
|
session = await get_session(raw_cookie)
|
|
if session is None:
|
|
return None
|
|
if session.account_status != "approved":
|
|
return None
|
|
|
|
try:
|
|
role = Role(session.role)
|
|
except ValueError:
|
|
return None
|
|
|
|
return Principal(
|
|
user_id=session.user_id,
|
|
role=role,
|
|
admin_access=session.admin_access,
|
|
super_admin=session.super_admin,
|
|
account_status=session.account_status,
|
|
cohort_ids=session.cohort_ids,
|
|
email=session.email,
|
|
display_name=session.display_name,
|
|
consent_at=session.consent_at,
|
|
profile_completed_at=session.profile_completed_at,
|
|
)
|
|
|
|
|
|
async def _practice_access_error(principal: Principal) -> str | None:
|
|
if principal.profile_completed_at is None and not await user_onboarding_complete(
|
|
principal.user_id
|
|
):
|
|
return "onboarding_required"
|
|
if principal.consent_at is None and not await user_has_consent(principal.user_id):
|
|
return "consent_required"
|
|
return None
|
|
|
|
|
|
async def _bind_session(
|
|
websocket: WebSocket,
|
|
principal: Principal,
|
|
) -> tuple[Optional[str], Optional[VoicePreset], Optional[str], dict[str, object]]:
|
|
"""Bind an existing session or create a dev-only voice session."""
|
|
qp = websocket.query_params
|
|
explicit_preset = qp.get("preset")
|
|
access_error = await _practice_access_error(principal)
|
|
if access_error is not None:
|
|
return None, None, access_error, {}
|
|
|
|
session_id = qp.get("session_id")
|
|
if session_id:
|
|
sess, err = await _load_voice_session(session_id, principal)
|
|
if sess is None:
|
|
return None, None, err or f"unknown session {session_id}", {}
|
|
vp = await _resolve_session_voice(
|
|
session_id=session_id,
|
|
persona_code=sess.persona.code,
|
|
explicit_preset=explicit_preset,
|
|
)
|
|
return (
|
|
session_id,
|
|
vp,
|
|
None,
|
|
{"degraded": False, "persona_catalog_source": "session"},
|
|
)
|
|
|
|
# persona_code session creation is local-dev only. Production uses REST start.
|
|
if settings.environment != "dev":
|
|
return None, None, "session_id required", {}
|
|
|
|
persona_code = qp.get("persona_code")
|
|
if not persona_code:
|
|
return None, None, "session_id or persona_code query required", {}
|
|
try:
|
|
catalog_persona = await get_catalog_persona(persona_code)
|
|
except Exception:
|
|
return None, None, "persona catalog database unavailable", {}
|
|
if catalog_persona is None:
|
|
return None, None, f"unknown persona {persona_code}", {}
|
|
card = catalog_persona.card
|
|
|
|
st = state_machine.init_state(
|
|
params=card.openness_params(),
|
|
)
|
|
sess = await session_persistence.create_session(
|
|
learner_id=principal.user_id,
|
|
card=card,
|
|
theory_mode="humanistic",
|
|
state=st,
|
|
session_no=1,
|
|
carry_rapport=st.rapport_credit,
|
|
persona_id=catalog_persona.persona_id,
|
|
persona_version=catalog_persona.version,
|
|
)
|
|
session_source = "database"
|
|
if sess is None:
|
|
require_runtime_fallback_allowed("voice session creation")
|
|
sess = store.create(
|
|
learner_id=principal.user_id,
|
|
persona=card,
|
|
theory_mode="humanistic",
|
|
state=st,
|
|
session_no=1,
|
|
carry_rapport=st.rapport_credit,
|
|
)
|
|
session_source = "runtime"
|
|
else:
|
|
store.put(sess)
|
|
vp = await _resolve_catalog_voice(
|
|
persona_id=catalog_persona.persona_id,
|
|
version=catalog_persona.version,
|
|
persona_code=card.code,
|
|
explicit_preset=explicit_preset,
|
|
)
|
|
degraded_reasons: list[str] = []
|
|
if catalog_persona.degraded:
|
|
degraded_reasons.append(
|
|
"카탈로그 원본을 확인하지 못해 음성 회기를 시작하지 않습니다"
|
|
)
|
|
if session_source == "runtime":
|
|
degraded_reasons.append(
|
|
"세션 저장소 연결 전까지 비영구 개발 런타임 기록을 사용합니다"
|
|
)
|
|
bind_meta = {
|
|
"degraded": bool(degraded_reasons),
|
|
"degraded_reason": "; ".join(degraded_reasons) if degraded_reasons else None,
|
|
"persona_catalog_source": catalog_persona.source,
|
|
"session_source": session_source,
|
|
}
|
|
return sess.session_id, vp, None, bind_meta
|
|
|
|
|
|
async def _resolve_session_voice(
|
|
*,
|
|
session_id: str,
|
|
persona_code: str,
|
|
explicit_preset: str | None,
|
|
) -> VoicePreset:
|
|
fallback = resolve_voice(persona_code=persona_code, preset=explicit_preset)
|
|
if explicit_preset:
|
|
return fallback
|
|
try:
|
|
voice_map = await get_session_voice_map(session_id)
|
|
except Exception:
|
|
return fallback
|
|
return _voice_from_map(voice_map, persona_code=persona_code) or fallback
|
|
|
|
|
|
async def _resolve_catalog_voice(
|
|
*,
|
|
persona_id: str | None,
|
|
version: int | None,
|
|
persona_code: str,
|
|
explicit_preset: str | None,
|
|
) -> VoicePreset:
|
|
fallback = resolve_voice(persona_code=persona_code, preset=explicit_preset)
|
|
if explicit_preset:
|
|
return fallback
|
|
try:
|
|
voice_map = await get_persona_voice_map(persona_id=persona_id, version=version)
|
|
except Exception:
|
|
return fallback
|
|
return _voice_from_map(voice_map, persona_code=persona_code) or fallback
|
|
|
|
|
|
def _voice_from_map(
|
|
voice_map: PersonaVoiceMap | None,
|
|
*,
|
|
persona_code: str,
|
|
) -> VoicePreset | None:
|
|
if voice_map is None:
|
|
return None
|
|
return voice_svc.resolve_voice_from_map(
|
|
provider=voice_map.provider,
|
|
voice_id=voice_map.voice_id,
|
|
base_params=voice_map.base_params,
|
|
persona_code=persona_code,
|
|
)
|
|
|
|
|
|
def _audio_meta(fmt: Optional[str]) -> tuple[str, str]:
|
|
"""Map the browser audio format to upload metadata."""
|
|
f = (fmt or "webm").lower().lstrip(".")
|
|
table = {
|
|
"webm": ("audio.webm", "audio/webm"),
|
|
"ogg": ("audio.ogg", "audio/ogg"),
|
|
"opus": ("audio.ogg", "audio/ogg"),
|
|
"wav": ("audio.wav", "audio/wav"),
|
|
"mp3": ("audio.mp3", "audio/mpeg"),
|
|
"mp4": ("audio.mp4", "audio/mp4"),
|
|
"m4a": ("audio.m4a", "audio/mp4"),
|
|
"pcm": ("audio.wav", "audio/wav"),
|
|
}
|
|
return table.get(f, ("audio.webm", "audio/webm"))
|
|
|
|
|
|
def _normalize_audio_upload(
|
|
audio: bytes,
|
|
*,
|
|
fmt: Optional[str],
|
|
sample_rate: int | None = None,
|
|
channels: int | None = None,
|
|
sample_width: int | None = None,
|
|
) -> tuple[bytes, str]:
|
|
f = (fmt or "webm").lower().lstrip(".") or "webm"
|
|
if f != "pcm":
|
|
return audio, f
|
|
if sample_width not in (None, 2):
|
|
raise ValueError("pcm sample_width must be 2 bytes")
|
|
return _wav_from_pcm16(
|
|
audio,
|
|
sample_rate=_bounded_int(
|
|
sample_rate, default=48000, minimum=8000, maximum=96000
|
|
),
|
|
channels=_bounded_int(channels, default=1, minimum=1, maximum=2),
|
|
), "wav"
|
|
|
|
|
|
def _bounded_int(value: int | None, *, default: int, minimum: int, maximum: int) -> int:
|
|
if value is None:
|
|
return default
|
|
return min(maximum, max(minimum, value))
|
|
|
|
|
|
def _wav_from_pcm16(pcm: bytes, *, sample_rate: int, channels: int) -> bytes:
|
|
byte_rate = sample_rate * channels * 2
|
|
block_align = channels * 2
|
|
data_size = len(pcm)
|
|
header = b"".join(
|
|
[
|
|
b"RIFF",
|
|
(36 + data_size).to_bytes(4, "little"),
|
|
b"WAVE",
|
|
b"fmt ",
|
|
(16).to_bytes(4, "little"),
|
|
(1).to_bytes(2, "little"),
|
|
channels.to_bytes(2, "little"),
|
|
sample_rate.to_bytes(4, "little"),
|
|
byte_rate.to_bytes(4, "little"),
|
|
block_align.to_bytes(2, "little"),
|
|
(16).to_bytes(2, "little"),
|
|
b"data",
|
|
data_size.to_bytes(4, "little"),
|
|
]
|
|
)
|
|
return header + pcm
|
|
|
|
|
|
def _voice_audio_ref(audio: bytes, fmt: Optional[str]) -> str | None:
|
|
if not audio:
|
|
return None
|
|
f = (fmt or "webm").lower().lstrip(".") or "webm"
|
|
digest = hashlib.sha256(audio).hexdigest()[:24]
|
|
return f"voice:{f}:sha256:{digest}"
|
|
|
|
|
|
def _elapsed_seconds(started_at: float | None, ended_at: float | None) -> float | None:
|
|
if started_at is None or ended_at is None:
|
|
return None
|
|
return max(0.001, ended_at - started_at)
|
|
|
|
|
|
def _estimate_speech_rate(text: str, duration_s: float | None) -> float | None:
|
|
if not text or not duration_s or duration_s <= 0:
|
|
return None
|
|
units = sum(1 for ch in text if not ch.isspace())
|
|
if units <= 0:
|
|
return None
|
|
return round((units / duration_s) * 60.0, 2)
|
|
|
|
|
|
def _safe_int(value: object) -> int | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _safe_float(value: object) -> float | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _safe_str(value: object) -> str | None:
|
|
if isinstance(value, str):
|
|
text = value.strip()
|
|
return text or None
|
|
return None
|
|
|
|
|
|
def _safe_bool(value: object) -> bool | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
normalized = value.strip().lower()
|
|
if normalized in {"1", "true", "yes", "y"}:
|
|
return True
|
|
if normalized in {"0", "false", "no", "n"}:
|
|
return False
|
|
return bool(value)
|
|
|
|
|
|
def _provider_event_slug(value: object) -> str:
|
|
text = str(value or "").strip().lower()
|
|
if not text:
|
|
return ""
|
|
chars = []
|
|
previous_underscore = False
|
|
for char in text:
|
|
if char.isalnum():
|
|
chars.append(char)
|
|
previous_underscore = False
|
|
elif not previous_underscore:
|
|
chars.append("_")
|
|
previous_underscore = True
|
|
return "".join(chars).strip("_")[:_PROVIDER_EVENT_MAX_STRING]
|
|
|
|
|
|
def _provider_event_taxonomy(event: dict[str, object]) -> tuple[str, str]:
|
|
for field in _PROVIDER_EVENT_TYPE_FIELDS:
|
|
slug = _provider_event_slug(event.get(field))
|
|
if not slug:
|
|
continue
|
|
canonical = _PROVIDER_EVENT_TAXONOMY.get(slug)
|
|
if canonical:
|
|
return canonical
|
|
return slug, "unknown"
|
|
return "", ""
|
|
|
|
|
|
def _safe_provider_events(value: object) -> list[dict[str, object]]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
events: list[dict[str, object]] = []
|
|
for item in value[:_PROVIDER_EVENT_MAX_ITEMS]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
safe: dict[str, object] = {}
|
|
for key in _PROVIDER_EVENT_ALLOWED_KEYS:
|
|
raw = item.get(key)
|
|
if isinstance(raw, bool):
|
|
safe[key] = raw
|
|
elif isinstance(raw, (int, float)):
|
|
safe[key] = raw
|
|
elif isinstance(raw, str):
|
|
text = raw.strip()
|
|
if text:
|
|
safe[key] = text[:_PROVIDER_EVENT_MAX_STRING]
|
|
if safe:
|
|
event_type, category = _provider_event_taxonomy(safe)
|
|
if event_type:
|
|
safe["event_type"] = event_type
|
|
if category:
|
|
safe["category"] = category
|
|
events.append(safe)
|
|
return events
|
|
|
|
|
|
def _merge_provider_events(*values: object) -> list[dict[str, object]]:
|
|
merged: list[dict[str, object]] = []
|
|
for value in values:
|
|
merged.extend(_safe_provider_events(value))
|
|
if len(merged) >= _PROVIDER_EVENT_MAX_ITEMS:
|
|
return merged[:_PROVIDER_EVENT_MAX_ITEMS]
|
|
return merged
|
|
|
|
|
|
async def _safe_send_json(websocket: WebSocket, payload: dict) -> None:
|
|
if websocket.client_state != WebSocketState.CONNECTED:
|
|
return
|
|
try:
|
|
await websocket.send_text(json.dumps(payload, ensure_ascii=False))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def _safe_send_bytes(websocket: WebSocket, data: bytes) -> None:
|
|
if websocket.client_state != WebSocketState.CONNECTED:
|
|
return
|
|
try:
|
|
await websocket.send_bytes(data)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def _safe_close(websocket: WebSocket, code: int = 1000) -> None:
|
|
if websocket.client_state == WebSocketState.DISCONNECTED:
|
|
return
|
|
try:
|
|
await websocket.close(code=code)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
__all__ = ["router"]
|