Stabilize runtime auth and E2E coverage

This commit is contained in:
Yun Chan 2026-06-26 14:47:00 +09:00
parent 6a3e3b541c
commit 188e899394
133 changed files with 55987 additions and 6775 deletions

View file

@ -1,22 +1,14 @@
"""음성 라우트 — OpenAI STT/TTS 캐스케이드 + WSS 실시간 턴테이킹.
"""Voice routes for the OpenAI STT/TTS cascade over WebSocket.
한신대 요구 '음성 필수'. 학습자가 마이크로 말하면 STT orchestrator 상담 1
내담자 텍스트 TTS 오디오 + 립싱크 힌트(설계 §4.3 RMS) 역방향으로 흘린다.
Client sends JSON controls plus binary audio chunks:
audio_start -> binary audio chunks -> audio_end
캐스케이드(설계 §5.2 음성 오브 4상태 listeningthinkingspeakingidle):
[클라] audio_start(JSON) 바이너리 오디오 청크들 audio_end(JSON)
[서버] state(listening) STT transcript(JSON) state(thinking)
orchestrator.run_turn(가드레일·상태머신·페르소나·내담자AI·출력가드)
reply(JSON, 내담자 텍스트 + stage/openness) state(speaking)
[tts_chunk(JSON: seq/rms) + 바이너리 오디오] × N tts_end(JSON) state(idle)
Server emits:
ready -> state(listening) -> state(thinking) -> transcript -> reply
-> state(speaking) -> tts_chunk + binary audio chunks -> tts_end -> state(idle)
프로토콜(JSON 제어 + 바이너리 오디오 혼합, 단일 WS):
- 클라서버 텍스트 = JSON 제어({"type": ...}); 클라서버 바이너리 = 오디오 청크
- 서버클라 텍스트 = JSON 이벤트; 서버클라 바이너리 = TTS 오디오 청크
- TTS 바이너리 청크 *직전* 메타 JSON(tts_chunk: seq, rms) 보내 프론트가 짝짓는다.
음성 미설정(OPENAI_API_KEY 없음): GET /voice/health 503 degraded,
WS 핸드셰이크 직후 degraded 이벤트 + close(1011). 절대 크래시 금지.
When voice is not configured, the route reports degraded state and closes
cleanly instead of crashing.
"""
from __future__ import annotations
@ -28,67 +20,84 @@ from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse
from starlette.websockets import WebSocketState
from .. import session_persistence
from ..auth_sessions import get_session
from ..config import settings
from ..deps import Principal, Role
from ..engine_client import EngineError, engine_client
from ..services import memory, orchestrator, persona
from ..persona_repository import get_catalog_persona
from ..runtime_policy import require_runtime_fallback_allowed, runtime_fallback_allowed
from ..services import memory, orchestrator, state_machine
from ..services import voice as voice_svc
from ..services.voice import VoicePreset, VoiceUnavailable, resolve_voice, voice_service
from ..store import TurnRecord, store
from ..store import InProcSession, TurnRecord, store
router = APIRouter(prefix="/voice", tags=["voice"])
# WS close 코드(섹션별 의미 명시)
WS_CLOSE_DEGRADED = 1011 # 서버측 음성 미설정/장애
WS_CLOSE_BAD_REQUEST = 1008 # 프로토콜 위반(세션 누락 등)
# WebSocket close codes.
WS_CLOSE_DEGRADED = 1011
WS_CLOSE_BAD_REQUEST = 1008
WS_CLOSE_UNAUTHORIZED = 1008
# 한 발화당 누적 오디오 상한(메모리 방어, ~10MB)
# Per-utterance audio cap to avoid unbounded memory growth.
_MAX_AUDIO_BYTES = 10 * 1024 * 1024
# ════════════════════════════════════════════════════════════════════════════
# 헬스 — 음성 가용성(키 설정) 노출
# ════════════════════════════════════════════════════════════════════════════
@router.get("/health")
async def voice_health() -> JSONResponse:
"""음성 라우터 헬스. 키 미설정이면 503 degraded(시연 투명성)."""
"""Return voice service readiness."""
available = voice_service.is_available()
body = {
"status": "ok" if available else "degraded",
"available": available,
"stt_model": voice_svc.STT_MODEL,
"tts_model": voice_svc.TTS_MODEL,
"reason": None if available else "OPENAI_API_KEY 미설정",
"reason": None if available else "OPENAI_API_KEY is not configured",
}
return JSONResponse(body, status_code=200 if available else 503)
# ════════════════════════════════════════════════════════════════════════════
# WebSocket — 실시간 음성 캐스케이드
# ════════════════════════════════════════════════════════════════════════════
@router.websocket("/ws")
async def voice_ws(websocket: WebSocket) -> None:
"""음성 실시간 턴 캐스케이드.
쿼리: ?session_id=<hex> (없으면 persona_code 일회용 in-proc 세션 생성 시연용)
오디오 in(바이너리) STT 상담 1 TTS out(바이너리) + 립싱크 힌트.
"""
"""Run one authenticated learner voice cascade."""
await websocket.accept()
# 1) 음성 미설정 → degraded 알리고 정상 종료(크래시 금지)
if not voice_service.is_available():
await _safe_send_json(
websocket,
{"type": "degraded", "reason": "OPENAI_API_KEY 미설정 — 음성 기능 비활성"},
)
await _safe_close(websocket, WS_CLOSE_DEGRADED)
# Authenticate the same server-side browser session used by REST routes.
principal = await _principal_from_websocket(websocket)
if principal is None:
await _safe_send_json(websocket, {"type": "error", "detail": "not authenticated"})
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
return
if principal.role != Role.LEARNER:
await _safe_send_json(websocket, {"type": "error", "detail": "only learners can use voice"})
await _safe_close(websocket, WS_CLOSE_UNAUTHORIZED)
return
# 2) 세션 바인딩 — session_id 우선, 없으면 persona_code 로 시연 세션 생성
session_id, voice_preset, err = _bind_session(websocket)
# Bind to an existing session first. persona_code creation is dev-only.
session_id, voice_preset, err, bind_meta = await _bind_session(websocket, principal)
if err is not None:
await _safe_send_json(websocket, {"type": "error", "detail": err})
await _safe_close(websocket, WS_CLOSE_BAD_REQUEST)
return
assert session_id is not None and voice_preset is not None
if bind_meta.get("degraded"):
await _safe_send_json(
websocket,
{
"type": "degraded",
"reason": bind_meta.get("degraded_reason", "voice session binding degraded"),
**bind_meta,
},
)
# Voice misconfiguration is reported explicitly and then closed cleanly.
if not voice_service.is_available():
await _safe_send_json(
websocket,
{"type": "degraded", "reason": "OPENAI_API_KEY is not configured"},
)
await _safe_close(websocket, WS_CLOSE_DEGRADED)
return
await _safe_send_json(
websocket,
@ -98,6 +107,7 @@ async def voice_ws(websocket: WebSocket) -> None:
"voice": voice_preset.openai_voice,
"preset": voice_preset.preset,
"state": "idle",
**bind_meta,
},
)
@ -111,10 +121,10 @@ async def voice_ws(websocket: WebSocket) -> None:
if mtype == "websocket.disconnect":
break
# ── 바이너리 = 오디오 청크 누적 ──
# Binary frames are audio chunks.
if msg.get("bytes") is not None:
if not receiving:
# audio_start 없이 들어온 바이너리 — 관용적으로 자동 시작
# Be tolerant when audio arrives before audio_start.
receiving = True
audio_buf.clear()
await _safe_send_json(websocket, {"type": "state", "state": "listening"})
@ -122,13 +132,13 @@ async def voice_ws(websocket: WebSocket) -> None:
if len(audio_buf) > _MAX_AUDIO_BYTES:
await _safe_send_json(
websocket,
{"type": "error", "detail": "audio too large — 발화를 짧게 끊어 주세요"},
{"type": "error", "detail": "audio too large; please send a shorter utterance"},
)
audio_buf.clear()
receiving = False
continue
# ── 텍스트 = JSON 제어 ──
# Text frames are JSON controls.
text = msg.get("text")
if text is None:
continue
@ -149,6 +159,7 @@ async def voice_ws(websocket: WebSocket) -> None:
await _handle_utterance(
websocket,
session_id=session_id,
principal=principal,
voice_preset=voice_preset,
audio=bytes(audio_buf),
fmt=ctrl.get("format"),
@ -156,7 +167,7 @@ async def voice_ws(websocket: WebSocket) -> None:
audio_buf.clear()
elif ctype == "text_turn":
# 음성 없이 텍스트만 보내는 경로(접근성/디버그): STT 건너뛰고 바로 턴.
# Text-only path for accessibility and deterministic tests.
receiving = False
audio_buf.clear()
learner_text = (ctrl.get("text") or "").strip()
@ -164,6 +175,7 @@ async def voice_ws(websocket: WebSocket) -> None:
await _run_turn_and_speak(
websocket,
session_id=session_id,
principal=principal,
voice_preset=voice_preset,
learner_text=learner_text,
)
@ -176,30 +188,28 @@ async def voice_ws(websocket: WebSocket) -> None:
except WebSocketDisconnect:
pass
except Exception as e: # 어떤 예외도 WS 를 깨끗이 닫고 알린다(크래시 금지)
except Exception as e:
await _safe_send_json(websocket, {"type": "error", "detail": f"voice ws error: {e}"})
finally:
await _safe_close(websocket)
# ════════════════════════════════════════════════════════════════════════════
# 발화 1건 처리 — STT → 턴 → TTS
# ════════════════════════════════════════════════════════════════════════════
async def _handle_utterance(
websocket: WebSocket,
*,
session_id: str,
principal: Principal,
voice_preset: VoicePreset,
audio: bytes,
fmt: Optional[str],
) -> None:
"""오디오 1발화 → STT → 상담 턴 → TTS 캐스케이드."""
"""Transcribe one utterance, generate the client reply, then synthesize TTS."""
if not audio:
await _safe_send_json(websocket, {"type": "transcript", "text": "", "final": True})
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
# 1) STT (thinking 진입)
# STT begins after the learner stops speaking.
await _safe_send_json(websocket, {"type": "state", "state": "thinking"})
filename, content_type = _audio_meta(fmt)
try:
@ -211,7 +221,7 @@ async def _handle_utterance(
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
except Exception as e:
await _safe_send_json(websocket, {"type": "error", "detail": f"STT 실패: {e}"})
await _safe_send_json(websocket, {"type": "error", "detail": f"STT failed: {e}"})
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
@ -221,13 +231,13 @@ async def _handle_utterance(
{"type": "transcript", "text": learner_text, "final": True, "speaker": "counselor"},
)
if not learner_text:
# 무음/인식 실패 — 턴 진행 안 함
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
await _run_turn_and_speak(
websocket,
session_id=session_id,
principal=principal,
voice_preset=voice_preset,
learner_text=learner_text,
)
@ -237,13 +247,14 @@ async def _run_turn_and_speak(
websocket: WebSocket,
*,
session_id: str,
principal: Principal,
voice_preset: VoicePreset,
learner_text: str,
) -> None:
"""상담 1턴(orchestrator) → 내담자 텍스트 → TTS 오디오/립싱크 힌트 역방향 전송."""
sess = store.get(session_id)
if sess is None or sess.ended:
await _safe_send_json(websocket, {"type": "error", "detail": "세션 없음/종료됨"})
"""Run one counseling turn and stream synthesized client speech."""
sess, err = await _load_voice_session(session_id, principal)
if sess is None:
await _safe_send_json(websocket, {"type": "error", "detail": err or "session not found or ended"})
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
@ -260,19 +271,7 @@ async def _run_turn_and_speak(
)
assert ctx.state_after is not None
# 학습자 발화 로깅(마스킹본) — sessions.py 패턴과 동일
store.append_turn(
session_id,
TurnRecord(
turn_seq=ctx.state_after.turn_seq,
speaker="counselor",
stage=ctx.state_after.stage.value,
text=learner_text,
text_masked=ctx.learner_text_masked,
),
)
# 2) 내담자 AI 1턴(동기 — 음성은 TTS 전 전체 텍스트가 필요)
# Voice needs the full client reply before TTS starts.
try:
result = await orchestrator.run_turn_generate(ctx, engine_client)
except EngineError as e:
@ -281,10 +280,22 @@ async def _run_turn_and_speak(
return
reply = result.client_reply or ""
# 내담자 응답 로깅 + 상태 체크포인트
# Persist only after the client reply has been generated. A failed AI turn
# must not leave a learner-only transcript in review or history.
await _append_voice_turn(
sess,
TurnRecord(
turn_seq=ctx.state_after.turn_seq,
speaker="counselor",
stage=ctx.state_after.stage.value,
text=learner_text,
text_masked=ctx.learner_text_masked,
),
)
if reply:
store.append_turn(
session_id,
# Persist the generated client reply before TTS playback.
await _append_voice_turn(
sess,
TurnRecord(
turn_seq=result.turn_seq,
speaker="client",
@ -293,9 +304,9 @@ async def _run_turn_and_speak(
text_masked=reply,
),
)
store.update_state(session_id, result.state_after)
await _update_voice_state(sess, result.state_after)
# 내담자 텍스트 이벤트(설계 §5.3 자막 — partial 없이 final)
# Send the final client text before audio playback.
await _safe_send_json(
websocket,
{
@ -314,7 +325,7 @@ async def _run_turn_and_speak(
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
return
# 3) TTS (speaking) — 청크별 메타 JSON(립싱크 rms) + 바이너리 오디오
# TTS speaking state comes before chunk metadata and binary audio.
await _safe_send_json(
websocket,
{"type": "state", "state": "speaking", "voice": voice_preset.openai_voice},
@ -322,7 +333,7 @@ async def _run_turn_and_speak(
try:
n = 0
async for ck in voice_service.synthesize_stream(reply, voice_preset):
# 메타 먼저(프론트가 직후 바이너리와 짝지음) — 설계 §4.3 RMS 1채널
# Metadata precedes the binary chunk so the client can pair them.
await _safe_send_json(
websocket, {"type": "tts_chunk", "seq": ck.seq, "rms": round(ck.rms, 4)}
)
@ -332,46 +343,112 @@ async def _run_turn_and_speak(
except VoiceUnavailable as e:
await _safe_send_json(websocket, {"type": "degraded", "reason": str(e)})
except Exception as e:
await _safe_send_json(websocket, {"type": "error", "detail": f"TTS 실패: {e}"})
await _safe_send_json(websocket, {"type": "error", "detail": f"TTS failed: {e}"})
await _safe_send_json(websocket, {"type": "state", "state": "idle"})
# ════════════════════════════════════════════════════════════════════════════
# 세션 바인딩 / 메타 헬퍼
# ════════════════════════════════════════════════════════════════════════════
def _bind_session(
websocket: WebSocket,
) -> tuple[Optional[str], Optional[VoicePreset], Optional[str]]:
"""쿼리에서 세션을 바인딩(또는 시연 세션 생성)하고 voice preset 을 해석.
async def _load_voice_session(
session_id: str,
principal: Principal,
) -> tuple[InProcSession | None, str | None]:
sess = await session_persistence.load_session(session_id, principal, allow_ended=True)
if sess is not None:
store.put(sess)
elif runtime_fallback_allowed():
sess = store.get(session_id)
if sess is None:
return None, f"unknown session {session_id}"
if sess.learner_id != principal.user_id:
return None, "session does not belong to user"
if sess.ended:
return None, "session already ended"
return sess, None
우선순위:
?session_id=<hex> 기존 세션(REST 시작된) 음성 부착
?persona_code=P1[&preset=] in-proc 시연 세션 생성(DB off 폴백)
반환 (session_id, voice_preset, error).
"""
async def _append_voice_turn(sess: InProcSession, turn: TurnRecord) -> None:
if await session_persistence.append_turn(
session_id=sess.session_id,
learner_id=sess.learner_id,
turn=turn,
):
sess.turns.append(turn)
store.put(sess)
return
require_runtime_fallback_allowed("voice session turn append")
store.append_turn(sess.session_id, turn)
async def _update_voice_state(
sess: InProcSession,
state: state_machine.SessionState,
) -> None:
if await session_persistence.update_state(
session_id=sess.session_id,
learner_id=sess.learner_id,
state=state,
):
sess.state = state
store.put(sess)
return
require_runtime_fallback_allowed("voice session state update")
store.update_state(sess.session_id, state)
async def _principal_from_websocket(websocket: WebSocket) -> Principal | None:
"""Restore the same server-side browser session used by REST routes."""
raw_cookie = websocket.cookies.get(settings.cookie_name)
if raw_cookie is None and settings.environment == "dev":
raw_cookie = websocket.cookies.get("vignette_sid")
session = await get_session(raw_cookie)
if session is None:
return None
try:
role = Role(session.role)
except ValueError:
return None
return Principal(
user_id=session.user_id,
role=role,
cohort_ids=session.cohort_ids,
email=session.email,
display_name=session.display_name,
)
async def _bind_session(
websocket: WebSocket,
principal: Principal,
) -> tuple[Optional[str], Optional[VoicePreset], Optional[str], dict[str, object]]:
"""Bind an existing session or create a dev-only voice session."""
qp = websocket.query_params
explicit_preset = qp.get("preset")
session_id = qp.get("session_id")
if session_id:
sess = store.get(session_id)
sess, err = await _load_voice_session(session_id, principal)
if sess is None:
return None, None, f"unknown session {session_id}"
if sess.ended:
return None, None, "session already ended"
return None, None, err or f"unknown session {session_id}", {}
vp = resolve_voice(persona_code=sess.persona.code, preset=explicit_preset)
return session_id, vp, None
return session_id, vp, None, {"degraded": False, "persona_catalog_source": "session"}
# persona_code session creation is local-dev only. Production uses REST start.
if settings.environment != "dev":
return None, None, "session_id required", {}
# persona_code 로 시연 세션 생성(REST 미경유 음성 단독 데모)
persona_code = qp.get("persona_code")
if not persona_code:
return None, None, "session_id 또는 persona_code 쿼리 필요"
card = persona.get_seed_persona(persona_code)
if card is None:
return None, None, f"unknown persona {persona_code}"
from ..services import state_machine
return None, None, "session_id or persona_code query required", {}
try:
catalog_persona = await get_catalog_persona(persona_code)
except Exception:
return None, None, "persona catalog database unavailable", {}
if catalog_persona is None:
return None, None, f"unknown persona {persona_code}", {}
card = catalog_persona.card
st = state_machine.init_state(
base_resistance=card.base_resistance(),
@ -379,19 +456,47 @@ def _bind_session(
decay_floor=card.decay_floor(),
ideation_baseline=card.ideation_baseline(),
)
sess = store.create(
learner_id="dev-learner-voice",
persona=card,
sess = await session_persistence.create_session(
learner_id=principal.user_id,
card=card,
theory_mode="humanistic",
state=st,
session_no=1,
carry_rapport=st.rapport_credit,
persona_id=catalog_persona.persona_id,
persona_version=catalog_persona.version,
)
session_source = "database"
if sess is None:
require_runtime_fallback_allowed("voice session creation")
sess = store.create(
learner_id=principal.user_id,
persona=card,
theory_mode="humanistic",
state=st,
session_no=1,
carry_rapport=st.rapport_credit,
)
session_source = "runtime"
else:
store.put(sess)
vp = resolve_voice(persona_code=card.code, preset=explicit_preset)
return sess.session_id, vp, None
degraded_reasons: list[str] = []
if catalog_persona.degraded:
degraded_reasons.append("카탈로그 원본을 확인하지 못해 음성 회기를 시작하지 않습니다")
if session_source == "runtime":
degraded_reasons.append("세션 저장소 연결 전까지 비영구 개발 런타임 기록을 사용합니다")
bind_meta = {
"degraded": bool(degraded_reasons),
"degraded_reason": "; ".join(degraded_reasons) if degraded_reasons else None,
"persona_catalog_source": catalog_persona.source,
"session_source": session_source,
}
return sess.session_id, vp, None, bind_meta
def _audio_meta(fmt: Optional[str]) -> tuple[str, str]:
"""클라가 알려준 포맷 → (filename, content_type). 기본 webm/opus."""
"""Map the browser audio format to upload metadata."""
f = (fmt or "webm").lower().lstrip(".")
table = {
"webm": ("audio.webm", "audio/webm"),
@ -406,7 +511,6 @@ def _audio_meta(fmt: Optional[str]) -> tuple[str, str]:
return table.get(f, ("audio.webm", "audio/webm"))
# ── 안전 송수신(연결 끊김 시 조용히 무시) ───────────────────────────────────
async def _safe_send_json(websocket: WebSocket, payload: dict) -> None:
if websocket.client_state != WebSocketState.CONNECTED:
return