G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
|
|
@ -13,11 +13,14 @@ cleanly instead of crashing.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
from dataclasses import dataclass, field as dataclass_field
|
||||
from typing import Optional
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect, HTTPException, status
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
|
@ -36,9 +39,29 @@ from ..persona_repository import (
|
|||
get_session_voice_map,
|
||||
)
|
||||
from ..runtime_policy import require_runtime_fallback_allowed
|
||||
from ..services import evaluator, orchestrator, state_machine
|
||||
from ..services import (
|
||||
evaluator,
|
||||
multimodal_alliance,
|
||||
multimodal_alliance_store,
|
||||
orchestrator,
|
||||
rupture_scenario_director,
|
||||
state_machine,
|
||||
)
|
||||
from ..services import voice as voice_svc
|
||||
from ..services.voice import VoicePreset, VoiceUnavailable, resolve_voice, voice_service
|
||||
from ..services.voice import (
|
||||
StreamingTranscriptEvent,
|
||||
TranscriptResult,
|
||||
VoicePreset,
|
||||
VoiceUnavailable,
|
||||
resolve_voice,
|
||||
voice_service,
|
||||
)
|
||||
from ..services.voice_runtime import (
|
||||
VOICE_AUDIO_BUFFER_MAX_BYTES,
|
||||
VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
||||
VOICE_UVICORN_WS_MAX_QUEUE,
|
||||
voice_runtime_metrics,
|
||||
)
|
||||
from ..store import InProcSession, TurnRecord, store
|
||||
|
||||
router = APIRouter(prefix="/voice", tags=["voice"])
|
||||
|
|
@ -92,7 +115,9 @@ WS_CLOSE_BAD_REQUEST = 1008
|
|||
WS_CLOSE_UNAUTHORIZED = 1008
|
||||
|
||||
# Per-utterance audio cap to avoid unbounded memory growth.
|
||||
_MAX_AUDIO_BYTES = 10 * 1024 * 1024
|
||||
_MAX_AUDIO_BYTES = VOICE_AUDIO_BUFFER_MAX_BYTES
|
||||
_STREAMING_EVENT_QUEUE_MAX_ITEMS = VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS
|
||||
_STREAMING_CONSENT_RECHECK_SECONDS = 1.0
|
||||
_PROVIDER_EVENT_MAX_ITEMS = 12
|
||||
_PROVIDER_EVENT_MAX_STRING = 80
|
||||
_PROVIDER_EVENT_ALLOWED_KEYS = {
|
||||
|
|
@ -103,6 +128,7 @@ _PROVIDER_EVENT_ALLOWED_KEYS = {
|
|||
"label",
|
||||
"source",
|
||||
"provider",
|
||||
"model",
|
||||
"start_ms",
|
||||
"end_ms",
|
||||
"duration_ms",
|
||||
|
|
@ -130,6 +156,7 @@ _PROVIDER_EVENT_TAXONOMY = {
|
|||
"speech_start": ("speech_start", "speech_activity"),
|
||||
"speech_end": ("speech_end", "speech_activity"),
|
||||
"speech_final": ("speech_final", "speech_activity"),
|
||||
"stt_word": ("stt_word", "timing"),
|
||||
"silence": ("silence", "timing"),
|
||||
"pause": ("silence", "timing"),
|
||||
"long_pause": ("silence", "timing"),
|
||||
|
|
@ -154,6 +181,15 @@ def _is_turn_persistence_unavailable(exc: Exception) -> bool:
|
|||
_PROVIDER_EVENT_TYPE_FIELDS = ("event_type", "type", "kind", "label")
|
||||
|
||||
|
||||
def _append_audio_chunk_with_cap(buffer: bytearray, chunk: bytes) -> bool:
|
||||
"""Append only when the route-owned buffer remains within its hard cap."""
|
||||
|
||||
if len(chunk) > _MAX_AUDIO_BYTES - len(buffer):
|
||||
return False
|
||||
buffer.extend(chunk)
|
||||
return True
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def voice_health() -> JSONResponse:
|
||||
"""Return voice service readiness."""
|
||||
|
|
@ -166,13 +202,20 @@ async def voice_health() -> JSONResponse:
|
|||
"available": available,
|
||||
"stt_available": stt_available,
|
||||
"tts_available": tts_available,
|
||||
"stt_model": voice_svc.STT_MODEL,
|
||||
"stt_provider": voice_service.stt_provider(),
|
||||
"stt_model": voice_service.stt_model(),
|
||||
"stt_batch_fallback_available": voice_service.batch_stt_available(),
|
||||
"tts_model": (
|
||||
voice_svc.HIGGS_TTS_MODEL
|
||||
if tts_provider == "higgs"
|
||||
else voice_svc.TTS_MODEL
|
||||
),
|
||||
"tts_provider": tts_provider,
|
||||
"limits": {
|
||||
"max_utterance_audio_bytes": _MAX_AUDIO_BYTES,
|
||||
"streaming_event_queue_max_items": _STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
||||
"uvicorn_ws_max_queue": VOICE_UVICORN_WS_MAX_QUEUE,
|
||||
},
|
||||
"reason": (
|
||||
None
|
||||
if available
|
||||
|
|
@ -313,7 +356,7 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
if not voice_service.is_available():
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{"type": "degraded", "reason": "OPENAI_API_KEY is not configured"},
|
||||
{"type": "degraded", "reason": "voice STT/TTS is not configured"},
|
||||
)
|
||||
await _safe_close(websocket, WS_CLOSE_DEGRADED)
|
||||
return
|
||||
|
|
@ -325,12 +368,23 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
"session_id": session_id,
|
||||
"voice": voice_preset.openai_voice,
|
||||
"preset": voice_preset.preset,
|
||||
"stt_provider": voice_service.stt_provider(),
|
||||
"stt_model": voice_service.stt_model(),
|
||||
"stt_batch_fallback_available": voice_service.batch_stt_available(),
|
||||
"tts_provider": voice_service.tts_provider_for_voice(voice_preset),
|
||||
"tts_model": voice_service.tts_model_for_voice(voice_preset),
|
||||
"limits": {
|
||||
"max_utterance_audio_bytes": _MAX_AUDIO_BYTES,
|
||||
"streaming_event_queue_max_items": _STREAMING_EVENT_QUEUE_MAX_ITEMS,
|
||||
"uvicorn_ws_max_queue": VOICE_UVICORN_WS_MAX_QUEUE,
|
||||
},
|
||||
"state": "idle",
|
||||
**bind_meta,
|
||||
},
|
||||
)
|
||||
|
||||
runtime_connection_id = voice_runtime_metrics.websocket_opened()
|
||||
|
||||
audio_buf = bytearray()
|
||||
receiving = False
|
||||
audio_started_at: float | None = None
|
||||
|
|
@ -339,6 +393,13 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
audio_sample_rate: int | None = None
|
||||
audio_channels: int | None = None
|
||||
audio_sample_width: int | None = None
|
||||
streaming_session: voice_svc.DeepgramStreamingSession | None = None
|
||||
streaming_consent_checked_at: float | None = None
|
||||
streaming_events: asyncio.Queue[StreamingTranscriptEvent] = asyncio.Queue(
|
||||
maxsize=_STREAMING_EVENT_QUEUE_MAX_ITEMS
|
||||
)
|
||||
discard_audio_until_end = False
|
||||
last_stream_transcript: tuple[str, bool] | None = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
|
|
@ -349,16 +410,28 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
|
||||
# Binary frames are audio chunks.
|
||||
if msg.get("bytes") is not None:
|
||||
if discard_audio_until_end:
|
||||
continue
|
||||
if not receiving:
|
||||
# Be tolerant when audio arrives before audio_start.
|
||||
receiving = True
|
||||
audio_started_at = time.monotonic()
|
||||
audio_buf.clear()
|
||||
voice_runtime_metrics.audio_buffer_cleared(
|
||||
runtime_connection_id
|
||||
)
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "listening"}
|
||||
)
|
||||
audio_buf.extend(msg["bytes"])
|
||||
if len(audio_buf) > _MAX_AUDIO_BYTES:
|
||||
chunk = msg["bytes"]
|
||||
if not _append_audio_chunk_with_cap(audio_buf, chunk):
|
||||
voice_runtime_metrics.audio_overflow_rejected(
|
||||
runtime_connection_id
|
||||
)
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
|
|
@ -366,8 +439,79 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
"detail": "audio too large; please send a shorter utterance",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "idle"}
|
||||
)
|
||||
audio_buf.clear()
|
||||
voice_runtime_metrics.audio_buffer_cleared(
|
||||
runtime_connection_id
|
||||
)
|
||||
receiving = False
|
||||
discard_audio_until_end = True
|
||||
continue
|
||||
voice_runtime_metrics.audio_chunk_received(
|
||||
runtime_connection_id,
|
||||
current_buffer_bytes=len(audio_buf),
|
||||
chunk_bytes=len(chunk),
|
||||
)
|
||||
if streaming_session is not None:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
streaming_consent_checked_at is None
|
||||
or now - streaming_consent_checked_at
|
||||
>= _STREAMING_CONSENT_RECHECK_SECONDS
|
||||
):
|
||||
if not await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=session_id,
|
||||
principal=principal,
|
||||
):
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
discard_audio_until_end = True
|
||||
receiving = False
|
||||
continue
|
||||
streaming_consent_checked_at = now
|
||||
try:
|
||||
await streaming_session.send_audio(chunk)
|
||||
await asyncio.sleep(0)
|
||||
drained = await _drain_streaming_transcripts(
|
||||
websocket, streaming_events
|
||||
)
|
||||
voice_runtime_metrics.streaming_queue_observed(
|
||||
runtime_connection_id,
|
||||
queue_items=streaming_events.qsize(),
|
||||
)
|
||||
if drained is not None:
|
||||
last_stream_transcript = drained
|
||||
except Exception:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
if voice_service.batch_stt_available():
|
||||
voice_runtime_metrics.provider_fallback()
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "degraded",
|
||||
"reason": "streaming STT unavailable; using batch fallback",
|
||||
},
|
||||
)
|
||||
else:
|
||||
discard_audio_until_end = True
|
||||
receiving = False
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "streaming_stt_unavailable",
|
||||
"detail": "streaming STT failed and no batch fallback is configured",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "idle"}
|
||||
)
|
||||
continue
|
||||
|
||||
# Text frames are JSON controls.
|
||||
|
|
@ -384,16 +528,94 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
|
||||
ctype = ctrl.get("type")
|
||||
if ctype == "audio_start":
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
receiving = True
|
||||
discard_audio_until_end = False
|
||||
last_stream_transcript = None
|
||||
while not streaming_events.empty():
|
||||
streaming_events.get_nowait()
|
||||
voice_runtime_metrics.streaming_queue_observed(
|
||||
runtime_connection_id,
|
||||
queue_items=0,
|
||||
)
|
||||
audio_started_at = time.monotonic()
|
||||
audio_format = _safe_str(ctrl.get("format"))
|
||||
audio_sample_rate = _safe_int(ctrl.get("sample_rate"))
|
||||
audio_channels = _safe_int(ctrl.get("channels"))
|
||||
audio_sample_width = _safe_int(ctrl.get("sample_width"))
|
||||
audio_buf.clear()
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "listening"}
|
||||
)
|
||||
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
||||
if voice_service.can_stream_audio(
|
||||
fmt=audio_format,
|
||||
sample_rate=audio_sample_rate,
|
||||
channels=audio_channels,
|
||||
sample_width=audio_sample_width,
|
||||
):
|
||||
if not await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=session_id,
|
||||
principal=principal,
|
||||
):
|
||||
receiving = False
|
||||
discard_audio_until_end = True
|
||||
else:
|
||||
async def queue_streaming_event(
|
||||
event: StreamingTranscriptEvent,
|
||||
) -> None:
|
||||
queue_was_full = streaming_events.full()
|
||||
queue_wait_started = time.perf_counter()
|
||||
await streaming_events.put(event)
|
||||
voice_runtime_metrics.streaming_queue_observed(
|
||||
runtime_connection_id,
|
||||
queue_items=streaming_events.qsize(),
|
||||
saturated=queue_was_full,
|
||||
wait_seconds=(
|
||||
time.perf_counter() - queue_wait_started
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
streaming_session = (
|
||||
await voice_service.open_streaming_transcription(
|
||||
fmt=audio_format,
|
||||
sample_rate=audio_sample_rate,
|
||||
channels=audio_channels,
|
||||
sample_width=audio_sample_width,
|
||||
on_event=queue_streaming_event,
|
||||
)
|
||||
)
|
||||
streaming_consent_checked_at = time.monotonic()
|
||||
except Exception:
|
||||
if voice_service.batch_stt_available():
|
||||
voice_runtime_metrics.provider_fallback()
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "degraded",
|
||||
"reason": "streaming STT unavailable; using batch fallback",
|
||||
},
|
||||
)
|
||||
else:
|
||||
receiving = False
|
||||
discard_audio_until_end = True
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "streaming_stt_unavailable",
|
||||
"detail": "streaming STT is unavailable and no batch fallback is configured",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "idle"}
|
||||
)
|
||||
if not discard_audio_until_end:
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "listening"}
|
||||
)
|
||||
|
||||
elif ctype == "audio_end":
|
||||
receiving = False
|
||||
|
|
@ -408,28 +630,101 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
0, int((audio_started_at - last_audio_end_at) * 1000)
|
||||
)
|
||||
end_format = _safe_str(ctrl.get("format")) or audio_format
|
||||
await _handle_utterance(
|
||||
websocket,
|
||||
VoiceSessionContext(session_id, principal, voice_preset),
|
||||
VoiceAudioInput(
|
||||
audio=bytes(audio_buf),
|
||||
fmt=end_format,
|
||||
sample_rate=_safe_int(ctrl.get("sample_rate"))
|
||||
or audio_sample_rate,
|
||||
channels=_safe_int(ctrl.get("channels")) or audio_channels,
|
||||
sample_width=_safe_int(ctrl.get("sample_width"))
|
||||
or audio_sample_width,
|
||||
audio_started_at=audio_started_at,
|
||||
audio_ended_at=audio_ended_at,
|
||||
prosody=VoiceProsody(
|
||||
silence_ms=silence_ms,
|
||||
barge_in=_safe_bool(ctrl.get("barge_in")),
|
||||
provider_events=_safe_provider_events(
|
||||
ctrl.get("provider_events")
|
||||
),
|
||||
context = VoiceSessionContext(session_id, principal, voice_preset)
|
||||
utterance = VoiceAudioInput(
|
||||
audio=bytes(audio_buf),
|
||||
fmt=end_format,
|
||||
sample_rate=_safe_int(ctrl.get("sample_rate"))
|
||||
or audio_sample_rate,
|
||||
channels=_safe_int(ctrl.get("channels")) or audio_channels,
|
||||
sample_width=_safe_int(ctrl.get("sample_width"))
|
||||
or audio_sample_width,
|
||||
audio_started_at=audio_started_at,
|
||||
audio_ended_at=audio_ended_at,
|
||||
prosody=VoiceProsody(
|
||||
silence_ms=silence_ms,
|
||||
barge_in=_safe_bool(ctrl.get("barge_in")),
|
||||
provider_events=_safe_provider_events(
|
||||
ctrl.get("provider_events")
|
||||
),
|
||||
),
|
||||
)
|
||||
if discard_audio_until_end:
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
elif streaming_session is not None:
|
||||
if not await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=session_id,
|
||||
principal=principal,
|
||||
):
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
else:
|
||||
try:
|
||||
streaming_result, drained = (
|
||||
await _finish_streaming_transcription(
|
||||
websocket,
|
||||
streaming_session,
|
||||
streaming_events,
|
||||
)
|
||||
)
|
||||
voice_runtime_metrics.streaming_queue_observed(
|
||||
runtime_connection_id,
|
||||
queue_items=streaming_events.qsize(),
|
||||
)
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
except Exception:
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
if voice_service.batch_stt_available():
|
||||
voice_runtime_metrics.provider_fallback()
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "degraded",
|
||||
"reason": "streaming STT unavailable; using batch fallback",
|
||||
},
|
||||
)
|
||||
await _handle_utterance(websocket, context, utterance)
|
||||
else:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "streaming_stt_unavailable",
|
||||
"detail": "streaming STT finalization failed and no batch fallback is configured",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(
|
||||
websocket, {"type": "state", "state": "idle"}
|
||||
)
|
||||
else:
|
||||
if drained is not None:
|
||||
last_stream_transcript = drained
|
||||
if await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=session_id,
|
||||
principal=principal,
|
||||
):
|
||||
await _handle_streaming_utterance(
|
||||
websocket,
|
||||
context,
|
||||
utterance,
|
||||
streaming_result,
|
||||
transcript_already_sent=(
|
||||
last_stream_transcript
|
||||
== (streaming_result.text, True)
|
||||
),
|
||||
)
|
||||
else:
|
||||
await _handle_utterance(websocket, context, utterance)
|
||||
last_audio_end_at = audio_ended_at
|
||||
audio_started_at = None
|
||||
audio_format = None
|
||||
|
|
@ -437,11 +732,19 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
audio_channels = None
|
||||
audio_sample_width = None
|
||||
audio_buf.clear()
|
||||
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
||||
discard_audio_until_end = False
|
||||
last_stream_transcript = None
|
||||
|
||||
elif ctype == "text_turn":
|
||||
# Text-only path for accessibility and deterministic tests.
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
receiving = False
|
||||
audio_buf.clear()
|
||||
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
||||
learner_text = (ctrl.get("text") or "").strip()
|
||||
if learner_text:
|
||||
await _run_turn_and_speak(
|
||||
|
|
@ -451,8 +754,13 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
)
|
||||
|
||||
elif ctype == "stt_result":
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
receiving = False
|
||||
audio_buf.clear()
|
||||
voice_runtime_metrics.audio_buffer_cleared(runtime_connection_id)
|
||||
stt_received_at = time.monotonic()
|
||||
await _handle_stt_result_control(
|
||||
websocket,
|
||||
|
|
@ -475,11 +783,16 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
await _safe_send_json(websocket, {"type": "pong"})
|
||||
|
||||
elif ctype == "close":
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
streaming_session = None
|
||||
streaming_consent_checked_at = None
|
||||
break
|
||||
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception as e:
|
||||
voice_runtime_metrics.websocket_error()
|
||||
if _is_turn_persistence_unavailable(e):
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
|
|
@ -495,9 +808,61 @@ async def voice_ws(websocket: WebSocket) -> None:
|
|||
websocket, {"type": "error", "detail": f"voice ws error: {e}"}
|
||||
)
|
||||
finally:
|
||||
if streaming_session is not None:
|
||||
await streaming_session.abort()
|
||||
voice_runtime_metrics.websocket_closed(runtime_connection_id)
|
||||
await _safe_close(websocket)
|
||||
|
||||
|
||||
async def _multimodal_voice_processing_allowed(
|
||||
websocket: WebSocket,
|
||||
*,
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
) -> bool:
|
||||
"""Stop before STT/derived processing when G7 consent is not active."""
|
||||
|
||||
try:
|
||||
await multimodal_alliance_store.assert_voice_processing_allowed(
|
||||
principal=principal,
|
||||
session_id=session_id,
|
||||
)
|
||||
except multimodal_alliance_store.MultimodalConsentWithdrawnError as exc:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_consent_withdrawn",
|
||||
"detail": str(exc),
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return False
|
||||
except multimodal_alliance_store.MultimodalConsentRequiredError as exc:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_consent_required",
|
||||
"detail": str(exc),
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return False
|
||||
except (multimodal_alliance_store.MultimodalAllianceError, RuntimeError):
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_consent_unavailable",
|
||||
"detail": "multimodal consent state is unavailable; voice processing is blocked",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def _handle_stt_result_control(
|
||||
websocket: WebSocket,
|
||||
*,
|
||||
|
|
@ -509,6 +874,12 @@ async def _handle_stt_result_control(
|
|||
audio_ended_at: float | None = None,
|
||||
last_audio_end_at: float | None = None,
|
||||
) -> None:
|
||||
if not await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=session_id,
|
||||
principal=principal,
|
||||
):
|
||||
return
|
||||
learner_text = str(ctrl.get("text") or "").strip()
|
||||
transcript_final = _safe_bool(ctrl.get("final"))
|
||||
silence_ms = _safe_int(ctrl.get("silence_ms"))
|
||||
|
|
@ -576,6 +947,13 @@ async def _handle_utterance(
|
|||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
|
||||
if not await _multimodal_voice_processing_allowed(
|
||||
websocket,
|
||||
session_id=context.session_id,
|
||||
principal=context.principal,
|
||||
):
|
||||
return
|
||||
|
||||
# STT begins after the learner stops speaking.
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
||||
upload_audio, upload_fmt = _normalize_audio_upload(
|
||||
|
|
@ -641,6 +1019,286 @@ async def _handle_utterance(
|
|||
)
|
||||
|
||||
|
||||
async def _drain_streaming_transcripts(
|
||||
websocket: WebSocket,
|
||||
events: asyncio.Queue[StreamingTranscriptEvent],
|
||||
) -> tuple[str, bool] | None:
|
||||
"""Relay provider-neutral streaming updates without concurrent ASGI sends."""
|
||||
|
||||
last: tuple[str, bool] | None = None
|
||||
while True:
|
||||
try:
|
||||
event = events.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
return last
|
||||
await _relay_streaming_transcript(websocket, event)
|
||||
last = (event.text, event.speech_final)
|
||||
|
||||
|
||||
async def _finish_streaming_transcription(
|
||||
websocket: WebSocket,
|
||||
session: voice_svc.DeepgramStreamingSession,
|
||||
events: asyncio.Queue[StreamingTranscriptEvent],
|
||||
) -> tuple[TranscriptResult, tuple[str, bool] | None]:
|
||||
"""Finalize while draining the bounded event queue to avoid producer deadlock."""
|
||||
|
||||
finish_task = asyncio.create_task(session.finish())
|
||||
last: tuple[str, bool] | None = None
|
||||
try:
|
||||
while not finish_task.done():
|
||||
event_task = asyncio.create_task(events.get())
|
||||
done, _ = await asyncio.wait(
|
||||
{finish_task, event_task},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
if event_task in done:
|
||||
event = event_task.result()
|
||||
await _relay_streaming_transcript(websocket, event)
|
||||
last = (event.text, event.speech_final)
|
||||
else:
|
||||
event_task.cancel()
|
||||
try:
|
||||
await event_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
result = await finish_task
|
||||
drained = await _drain_streaming_transcripts(websocket, events)
|
||||
return result, drained or last
|
||||
except Exception:
|
||||
if not finish_task.done():
|
||||
finish_task.cancel()
|
||||
raise
|
||||
|
||||
|
||||
async def _relay_streaming_transcript(
|
||||
websocket: WebSocket, event: StreamingTranscriptEvent
|
||||
) -> None:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "transcript",
|
||||
"text": event.text,
|
||||
# Deepgram is_final seals one segment; speech_final seals the
|
||||
# learner utterance. The browser's `final` contract means the
|
||||
# latter so it never stops capture at an intermediate segment.
|
||||
"final": event.speech_final,
|
||||
"speech_final": event.speech_final,
|
||||
"speaker": "counselor",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _handle_streaming_utterance(
|
||||
websocket: WebSocket,
|
||||
context: VoiceSessionContext,
|
||||
utterance: VoiceAudioInput,
|
||||
stt: TranscriptResult,
|
||||
*,
|
||||
transcript_already_sent: bool,
|
||||
) -> None:
|
||||
"""Persist and run one finalized provider-streamed learner utterance."""
|
||||
|
||||
learner_text = stt.text.strip()
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
|
||||
if not transcript_already_sent:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "transcript",
|
||||
"text": learner_text,
|
||||
"final": True,
|
||||
"speech_final": True,
|
||||
"speaker": "counselor",
|
||||
},
|
||||
)
|
||||
if not learner_text:
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
|
||||
audio_ref = _voice_audio_ref(utterance.audio, utterance.fmt)
|
||||
duration_s = stt.duration or _elapsed_seconds(
|
||||
utterance.audio_started_at, utterance.audio_ended_at
|
||||
)
|
||||
if stt.words:
|
||||
duration_s = max(duration_s or 0.0, max(word.end for word in stt.words))
|
||||
speech_rate = _estimate_speech_rate(learner_text, duration_s)
|
||||
provider_events = _merge_provider_events(
|
||||
[
|
||||
{
|
||||
"type": "stt_metadata",
|
||||
"provider": "deepgram",
|
||||
"model": stt.model,
|
||||
"source": "streaming_stt",
|
||||
"is_final": True,
|
||||
}
|
||||
],
|
||||
utterance.prosody.provider_events,
|
||||
stt.provider_events,
|
||||
)
|
||||
try:
|
||||
await _persist_streaming_timeline(
|
||||
context=context,
|
||||
utterance=utterance,
|
||||
stt=stt,
|
||||
audio_ref=audio_ref,
|
||||
duration_s=duration_s,
|
||||
)
|
||||
except multimodal_alliance_store.MultimodalConsentWithdrawnError as exc:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_consent_withdrawn",
|
||||
"detail": str(exc),
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
except multimodal_alliance_store.MultimodalConsentRequiredError as exc:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_consent_required",
|
||||
"detail": str(exc),
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
except multimodal_alliance_store.MultimodalAllianceError:
|
||||
await _safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "error",
|
||||
"code": "multimodal_timeline_unavailable",
|
||||
"detail": "multimodal timeline persistence is unavailable; voice turn is blocked",
|
||||
},
|
||||
)
|
||||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
|
||||
await _run_turn_and_speak(
|
||||
websocket,
|
||||
context,
|
||||
VoiceTurnInput(
|
||||
learner_text=learner_text,
|
||||
prosody=VoiceProsody(
|
||||
audio_ref=audio_ref,
|
||||
duration_s=duration_s,
|
||||
silence_ms=utterance.prosody.silence_ms,
|
||||
speech_rate=speech_rate,
|
||||
barge_in=utterance.prosody.barge_in,
|
||||
provider_events=provider_events,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _persist_streaming_timeline(
|
||||
*,
|
||||
context: VoiceSessionContext,
|
||||
utterance: VoiceAudioInput,
|
||||
stt: TranscriptResult,
|
||||
audio_ref: str,
|
||||
duration_s: float | None,
|
||||
) -> dict[str, object] | None:
|
||||
audio_sha256 = hashlib.sha256(utterance.audio).hexdigest()
|
||||
submission_id = uuid5(
|
||||
NAMESPACE_URL,
|
||||
f"vignette:g7:streaming-stt:{context.session_id}:{audio_sha256}:{stt.model}",
|
||||
)
|
||||
duration_ms = max(
|
||||
1,
|
||||
round((duration_s or 0.0) * 1000),
|
||||
*(round(word.end * 1000) for word in stt.words),
|
||||
)
|
||||
words = []
|
||||
for index, word in enumerate(sorted(stt.words, key=lambda item: item.start)):
|
||||
start_ms = max(0, min(duration_ms - 1, round(word.start * 1000)))
|
||||
end_ms = max(start_ms + 1, min(duration_ms, round(word.end * 1000)))
|
||||
words.append(
|
||||
{
|
||||
"word_index": index,
|
||||
"start_ms": start_ms,
|
||||
"end_ms": end_ms,
|
||||
"speaker": "learner",
|
||||
# Common counselling words are dictionary-attackable when stored
|
||||
# as plain SHA-256. Bind the pseudonym to this deployment and
|
||||
# submission so the timeline remains useful without creating a
|
||||
# reusable transcript fingerprint.
|
||||
"token_hash": hmac.new(
|
||||
settings.session_secret.encode("utf-8"),
|
||||
(
|
||||
f"{context.session_id}:{submission_id}:"
|
||||
f"{word.word.casefold()}"
|
||||
).encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest(),
|
||||
}
|
||||
)
|
||||
|
||||
events = []
|
||||
for index, event in enumerate(_safe_provider_events(stt.provider_events)):
|
||||
provider_type = str(event.get("event_type") or "")
|
||||
event_type = _g7_event_type(provider_type)
|
||||
if event_type is None:
|
||||
continue
|
||||
start_ms = max(0, _safe_int(event.get("start_ms")) or 0)
|
||||
end_ms = _safe_int(event.get("end_ms"))
|
||||
if end_ms is None:
|
||||
end_ms = start_ms + max(0, _safe_int(event.get("duration_ms")) or 0)
|
||||
start_ms = min(start_ms, duration_ms - 1)
|
||||
end_ms = min(duration_ms, max(start_ms + 1, end_ms))
|
||||
confidence = _safe_float(event.get("confidence"))
|
||||
uncertainty = 0.5 if confidence is None else max(0.0, min(1.0, 1.0 - confidence))
|
||||
events.append(
|
||||
{
|
||||
"event_id": f"oas-g7-event-{submission_id.hex}-{index}",
|
||||
"event_type": event_type,
|
||||
"start_ms": start_ms,
|
||||
"end_ms": end_ms,
|
||||
"actor": "learner",
|
||||
"observed_feature": f"provider observed {provider_type}",
|
||||
"uncertainty": uncertainty,
|
||||
"source": "stt_word_timestamps",
|
||||
}
|
||||
)
|
||||
|
||||
timeline = multimodal_alliance.align_voice_timeline(
|
||||
audio_duration_ms=duration_ms,
|
||||
words=words,
|
||||
events=events,
|
||||
)
|
||||
_, media_type = _audio_meta(utterance.fmt)
|
||||
return await multimodal_alliance_store.append_runtime_timeline(
|
||||
session_id=context.session_id,
|
||||
submission_id=submission_id,
|
||||
timeline=timeline,
|
||||
audio_asset={
|
||||
"audio_ref": audio_ref,
|
||||
"audio_sha256": audio_sha256,
|
||||
"media_type": media_type,
|
||||
"byte_size": len(utterance.audio),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _g7_event_type(provider_type: str) -> str | None:
|
||||
if provider_type in {"speech_final", "speech_end", "voice_activity", "speech_rate"}:
|
||||
return "pace"
|
||||
if provider_type in {"barge_in", "interrupt", "interruption"}:
|
||||
return "interruption"
|
||||
if provider_type == "overlap":
|
||||
return "overlap"
|
||||
if provider_type in {"silence", "pause", "long_pause"}:
|
||||
return "silence"
|
||||
if provider_type in {"background_noise", "noise"}:
|
||||
return "audio_quality"
|
||||
if provider_type in {"sigh", "cry", "laugh", "breath", "pitch", "intonation", "prosody"}:
|
||||
return "prosody"
|
||||
return None
|
||||
|
||||
|
||||
async def _run_turn_and_speak(
|
||||
websocket: WebSocket,
|
||||
context: VoiceSessionContext,
|
||||
|
|
@ -670,21 +1328,10 @@ async def _run_turn_and_speak(
|
|||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
return
|
||||
|
||||
recall = await session_routes.ensure_recall_context(sess)
|
||||
kb_cues = session_routes.cached_kb_cues(context.session_id)
|
||||
ctx = orchestrator.prepare_turn(
|
||||
ctx = await _prepare_voice_turn_context(
|
||||
session_id=context.session_id,
|
||||
case_id=sess.case_id,
|
||||
card=sess.persona,
|
||||
state=sess.state,
|
||||
sess=sess,
|
||||
learner_text=learner_text,
|
||||
memory=orchestrator.TurnMemory(
|
||||
recall_summary=recall.recall_summary,
|
||||
pinned_facts=recall.pinned_facts,
|
||||
recent_turns=sess.recent_turns(visible_to="client"),
|
||||
kb_behavior_cues=kb_cues,
|
||||
),
|
||||
theory_mode=sess.theory_mode,
|
||||
)
|
||||
assert ctx.state_after is not None
|
||||
|
||||
|
|
@ -782,6 +1429,41 @@ async def _run_turn_and_speak(
|
|||
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
|
||||
|
||||
|
||||
async def _prepare_voice_turn_context(
|
||||
*,
|
||||
session_id: str,
|
||||
sess: InProcSession,
|
||||
learner_text: str,
|
||||
) -> orchestrator.TurnContext:
|
||||
"""Build voice turn context with the same fail-closed G3 ledger projection."""
|
||||
|
||||
from . import sessions as session_routes
|
||||
|
||||
recall = await session_routes.ensure_recall_context(sess)
|
||||
kb_cues = session_routes.cached_kb_cues(session_id)
|
||||
scenario_context = (
|
||||
await rupture_scenario_director.load_stored_scenario_context(
|
||||
session_id=session_id,
|
||||
case_id=sess.case_id,
|
||||
)
|
||||
)
|
||||
return orchestrator.prepare_turn(
|
||||
session_id=session_id,
|
||||
case_id=sess.case_id,
|
||||
card=sess.persona,
|
||||
state=sess.state,
|
||||
learner_text=learner_text,
|
||||
memory=orchestrator.TurnMemory(
|
||||
recall_summary=recall.recall_summary,
|
||||
pinned_facts=recall.pinned_facts,
|
||||
recent_turns=sess.recent_turns(visible_to="client"),
|
||||
kb_behavior_cues=kb_cues,
|
||||
),
|
||||
theory_mode=sess.theory_mode,
|
||||
scenario_context=scenario_context,
|
||||
)
|
||||
|
||||
|
||||
async def _load_voice_session(
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
|
|
@ -1113,6 +1795,15 @@ def _safe_int(value: object) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _safe_float(value: object) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _safe_str(value: object) -> str | None:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue