G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
900
scripts/soak-voice-websocket.py
Normal file
900
scripts/soak-voice-websocket.py
Normal file
|
|
@ -0,0 +1,900 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Deterministic loopback soak for the real ``/voice/ws`` ASGI route.
|
||||
|
||||
This runner starts an ephemeral Uvicorn server on 127.0.0.1, includes the
|
||||
production voice router, and drives it through a real WebSocket transport. It
|
||||
does not use a physical microphone, public WSS, a real database, or paid
|
||||
STT/TTS/LLM providers. Those boundaries are replaced at explicit seams while
|
||||
the route framing, consent preflight, audio normalization, byte cap, and close
|
||||
codes remain active.
|
||||
|
||||
Exit codes:
|
||||
0 = every deterministic contract and timed soak cycle passed
|
||||
1 = a route/transport contract failed
|
||||
2 = invalid CLI usage (argparse)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import ctypes
|
||||
import gc
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from websockets.asyncio.client import ClientConnection, connect
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1] / "apps" / "api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.deps import Principal, Role # noqa: E402 - API_ROOT is injected above
|
||||
from app.routes import voice as voice_routes # noqa: E402 - API_ROOT is injected above
|
||||
from app.services import multimodal_alliance_store # noqa: E402 - API_ROOT is injected above
|
||||
from app.services.voice import VoicePreset # noqa: E402 - API_ROOT is injected above
|
||||
|
||||
|
||||
DEFAULT_DURATION_SECONDS = 8.0
|
||||
DEFAULT_CYCLE_INTERVAL_MS = 100
|
||||
DEFAULT_WS_MAX_QUEUE = 4
|
||||
DEFAULT_TEST_AUDIO_CAP_BYTES = 4_096
|
||||
DEFAULT_CHUNK_BYTES = 1_024
|
||||
FRAME_TIMEOUT_SECONDS = 3.0
|
||||
VOICE_PRESET = VoicePreset(preset="neutral", openai_voice="sage")
|
||||
|
||||
|
||||
class SoakFailure(RuntimeError):
|
||||
"""A deterministic WebSocket contract was violated."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SoakStats:
|
||||
schema_version: str = "vignette.g7-voice-ws-soak.v1"
|
||||
status: str = "running"
|
||||
route: str = "/voice/ws"
|
||||
transport: str = "loopback_uvicorn_websocket"
|
||||
requested_duration_seconds: float = 0.0
|
||||
timed_soak_elapsed_seconds: float = 0.0
|
||||
ws_max_queue: int = 0
|
||||
test_audio_cap_bytes: int = 0
|
||||
production_audio_cap_bytes: int = 0
|
||||
physical_microphone_used: bool = False
|
||||
public_wss_used: bool = False
|
||||
real_database_used: bool = False
|
||||
paid_provider_calls: int = 0
|
||||
connections_opened: int = 0
|
||||
connections_completed: int = 0
|
||||
reconnect_successes: int = 0
|
||||
normal_close_1000: int = 0
|
||||
unauthorized_close_1008: int = 0
|
||||
degraded_close_1011: int = 0
|
||||
successful_utterances: int = 0
|
||||
synthetic_stt_calls: int = 0
|
||||
synthetic_tts_frames: int = 0
|
||||
consent_required_blocks: int = 0
|
||||
withdrawal_blocks: int = 0
|
||||
overflow_rejections: int = 0
|
||||
overflow_recoveries: int = 0
|
||||
burst_frames_sent: int = 0
|
||||
max_normalized_audio_bytes_seen: int = 0
|
||||
streaming_event_queue_max_items: int = 0
|
||||
production_cap_buffer_high_water_bytes: int = 0
|
||||
production_cap_overflow_rejected: bool = False
|
||||
rss_before_production_cap_probe_bytes: int = 0
|
||||
rss_at_production_cap_bytes: int = 0
|
||||
rss_after_production_cap_clear_bytes: int = 0
|
||||
peak_working_set_delta_bytes: int = 0
|
||||
unexpected_errors: list[str] = field(default_factory=list)
|
||||
overrides: list[str] = field(
|
||||
default_factory=lambda: [
|
||||
"_principal_from_websocket -> deterministic learner principal",
|
||||
"_bind_session -> deterministic existing-session binding",
|
||||
"voice_service.transcribe -> local synthetic transcript",
|
||||
"_run_turn_and_speak -> local deterministic reply/TTS frames",
|
||||
"multimodal assert_voice_processing_allowed -> in-memory consent gate",
|
||||
"voice_service.is_available -> controllable local readiness",
|
||||
"_MAX_AUDIO_BYTES -> reduced test cap for fast overflow proof",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class ConsentGate:
|
||||
def __init__(self) -> None:
|
||||
self._states: dict[str, str] = {}
|
||||
|
||||
def grant(self, session_id: str) -> None:
|
||||
self._states[session_id] = "granted"
|
||||
|
||||
def withdraw(self, session_id: str) -> None:
|
||||
self._states[session_id] = "withdrawn"
|
||||
|
||||
def clear(self, session_id: str) -> None:
|
||||
self._states.pop(session_id, None)
|
||||
|
||||
async def assert_allowed(self, *, principal: Principal, session_id: str) -> None:
|
||||
del principal
|
||||
state = self._states.get(session_id)
|
||||
if state == "withdrawn":
|
||||
raise multimodal_alliance_store.MultimodalConsentWithdrawnError(
|
||||
"multimodal voice consent was withdrawn"
|
||||
)
|
||||
if state != "granted":
|
||||
raise multimodal_alliance_store.MultimodalConsentRequiredError(
|
||||
"active multimodal voice consent is required"
|
||||
)
|
||||
|
||||
|
||||
class SyntheticOverrides:
|
||||
def __init__(self, gate: ConsentGate, stats: SoakStats) -> None:
|
||||
self.gate = gate
|
||||
self.stats = stats
|
||||
self.voice_available = True
|
||||
self.principal = Principal(
|
||||
user_id="77000000-0000-4000-8000-000000000201",
|
||||
role=Role.LEARNER,
|
||||
cohort_ids=["g7-loopback-soak"],
|
||||
email="g7-loopback-soak@invalid.local",
|
||||
display_name="G7 Loopback Soak",
|
||||
consent_at=1.0,
|
||||
profile_completed_at=1.0,
|
||||
)
|
||||
|
||||
async def principal_from_websocket(self, websocket: Any) -> Principal | None:
|
||||
if websocket.query_params.get("auth") == "denied":
|
||||
return None
|
||||
return self.principal
|
||||
|
||||
async def bind_session(
|
||||
self,
|
||||
websocket: Any,
|
||||
principal: Principal,
|
||||
) -> tuple[str, VoicePreset, None, dict[str, object]]:
|
||||
del principal
|
||||
session_id = websocket.query_params.get("session_id") or "g7-soak-session"
|
||||
return (
|
||||
session_id,
|
||||
VOICE_PRESET,
|
||||
None,
|
||||
{"degraded": False, "persona_catalog_source": "synthetic_override"},
|
||||
)
|
||||
|
||||
async def transcribe(
|
||||
self,
|
||||
audio: bytes,
|
||||
*,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
) -> SimpleNamespace:
|
||||
if not audio.startswith(b"RIFF") or audio[8:12] != b"WAVE":
|
||||
raise SoakFailure("PCM audio did not traverse production WAV normalization")
|
||||
if filename != "audio.wav" or content_type != "audio/wav":
|
||||
raise SoakFailure("normalized audio metadata drifted from WAV")
|
||||
self.stats.synthetic_stt_calls += 1
|
||||
self.stats.max_normalized_audio_bytes_seen = max(
|
||||
self.stats.max_normalized_audio_bytes_seen,
|
||||
len(audio),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
text="합성 음성 회기 응답",
|
||||
duration=0.032,
|
||||
provider_events=[
|
||||
{
|
||||
"event_type": "speech_final",
|
||||
"category": "speech_activity",
|
||||
"confidence": 1.0,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
async def run_turn_and_speak(
|
||||
self,
|
||||
websocket: Any,
|
||||
context: voice_routes.VoiceSessionContext,
|
||||
turn: voice_routes.VoiceTurnInput,
|
||||
) -> None:
|
||||
if not turn.learner_text:
|
||||
raise SoakFailure("synthetic turn reached reply seam without transcript")
|
||||
await voice_routes._safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "reply",
|
||||
"text": "결정적 합성 내담자 응답",
|
||||
"speaker": "client",
|
||||
"stage": "초기",
|
||||
"turn_seq": self.stats.successful_utterances + 1,
|
||||
"synthetic_override": True,
|
||||
},
|
||||
)
|
||||
await voice_routes._safe_send_json(
|
||||
websocket,
|
||||
{
|
||||
"type": "state",
|
||||
"state": "speaking",
|
||||
"voice": context.voice_preset.openai_voice,
|
||||
"tts_provider": "synthetic",
|
||||
},
|
||||
)
|
||||
payload = b"g7-synthetic-tts-frame"
|
||||
await voice_routes._safe_send_bytes(websocket, payload)
|
||||
self.stats.synthetic_tts_frames += 1
|
||||
await voice_routes._safe_send_json(
|
||||
websocket,
|
||||
{"type": "tts_end", "chunks": 1, "synthetic_override": True},
|
||||
)
|
||||
await voice_routes._safe_send_json(
|
||||
websocket,
|
||||
{"type": "state", "state": "idle"},
|
||||
)
|
||||
self.stats.successful_utterances += 1
|
||||
|
||||
|
||||
def synthetic_pcm(chunk_bytes: int = DEFAULT_CHUNK_BYTES) -> bytes:
|
||||
"""Return deterministic 16 kHz mono signed-16 PCM, never microphone input."""
|
||||
|
||||
if chunk_bytes <= 0 or chunk_bytes % 2:
|
||||
raise ValueError("chunk_bytes must be a positive even number")
|
||||
samples = chunk_bytes // 2
|
||||
payload = bytearray()
|
||||
for index in range(samples):
|
||||
value = int(1_200 * math.sin(2 * math.pi * 220 * index / 16_000))
|
||||
payload.extend(struct.pack("<h", value))
|
||||
return bytes(payload)
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise SoakFailure(message)
|
||||
|
||||
|
||||
def process_memory_bytes() -> tuple[int, int]:
|
||||
"""Return current RSS and process peak working set without extra packages."""
|
||||
|
||||
if os.name == "nt":
|
||||
class ProcessMemoryCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("cb", ctypes.c_ulong),
|
||||
("PageFaultCount", ctypes.c_ulong),
|
||||
("PeakWorkingSetSize", ctypes.c_size_t),
|
||||
("WorkingSetSize", ctypes.c_size_t),
|
||||
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("PagefileUsage", ctypes.c_size_t),
|
||||
("PeakPagefileUsage", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
counters = ProcessMemoryCounters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
get_current_process = ctypes.windll.kernel32.GetCurrentProcess
|
||||
get_current_process.restype = ctypes.c_void_p
|
||||
get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo
|
||||
get_process_memory_info.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.POINTER(ProcessMemoryCounters),
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
get_process_memory_info.restype = ctypes.c_int
|
||||
handle = get_current_process()
|
||||
ok = get_process_memory_info(
|
||||
handle,
|
||||
ctypes.byref(counters),
|
||||
counters.cb,
|
||||
)
|
||||
if not ok:
|
||||
raise SoakFailure("GetProcessMemoryInfo failed")
|
||||
return int(counters.WorkingSetSize), int(counters.PeakWorkingSetSize)
|
||||
|
||||
import resource
|
||||
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
peak = int(usage.ru_maxrss)
|
||||
if sys.platform != "darwin":
|
||||
peak *= 1024
|
||||
return peak, peak
|
||||
|
||||
|
||||
def exercise_production_cap_memory(stats: SoakStats) -> None:
|
||||
"""Allocate exactly one production utterance cap and reject one extra byte."""
|
||||
|
||||
gc.collect()
|
||||
rss_before, peak_before = process_memory_bytes()
|
||||
buffer = bytearray()
|
||||
chunk = bytes(64 * 1024)
|
||||
while len(buffer) < voice_routes._MAX_AUDIO_BYTES:
|
||||
remaining = voice_routes._MAX_AUDIO_BYTES - len(buffer)
|
||||
require(
|
||||
voice_routes._append_audio_chunk_with_cap(buffer, chunk[:remaining]),
|
||||
"production-cap probe rejected an in-bound chunk",
|
||||
)
|
||||
stats.production_cap_buffer_high_water_bytes = len(buffer)
|
||||
stats.production_cap_overflow_rejected = not (
|
||||
voice_routes._append_audio_chunk_with_cap(buffer, b"x")
|
||||
)
|
||||
rss_at_cap, peak_at_cap = process_memory_bytes()
|
||||
buffer.clear()
|
||||
del buffer
|
||||
gc.collect()
|
||||
rss_after, peak_after = process_memory_bytes()
|
||||
stats.rss_before_production_cap_probe_bytes = rss_before
|
||||
stats.rss_at_production_cap_bytes = rss_at_cap
|
||||
stats.rss_after_production_cap_clear_bytes = rss_after
|
||||
stats.peak_working_set_delta_bytes = max(
|
||||
0, max(peak_at_cap, peak_after) - peak_before
|
||||
)
|
||||
|
||||
|
||||
def frame_json(raw: str | bytes) -> dict[str, Any] | None:
|
||||
if isinstance(raw, bytes):
|
||||
return None
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SoakFailure("server emitted non-JSON text frame") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise SoakFailure("server emitted non-object JSON frame")
|
||||
return value
|
||||
|
||||
|
||||
async def receive_until(
|
||||
websocket: ClientConnection,
|
||||
predicate,
|
||||
*,
|
||||
timeout: float = FRAME_TIMEOUT_SECONDS,
|
||||
) -> list[str | bytes]:
|
||||
deadline = time.monotonic() + timeout
|
||||
frames: list[str | bytes] = []
|
||||
while time.monotonic() < deadline:
|
||||
remaining = max(0.01, deadline - time.monotonic())
|
||||
try:
|
||||
raw = await asyncio.wait_for(websocket.recv(), timeout=remaining)
|
||||
except TimeoutError as exc:
|
||||
raise SoakFailure("timed out waiting for WebSocket contract frame") from exc
|
||||
frames.append(raw)
|
||||
payload = frame_json(raw)
|
||||
if predicate(raw, payload):
|
||||
return frames
|
||||
raise SoakFailure("WebSocket predicate was not reached before deadline")
|
||||
|
||||
|
||||
async def receive_json_type(
|
||||
websocket: ClientConnection,
|
||||
expected_type: str,
|
||||
) -> tuple[dict[str, Any], list[str | bytes]]:
|
||||
frames = await receive_until(
|
||||
websocket,
|
||||
lambda _raw, payload: payload is not None
|
||||
and payload.get("type") == expected_type,
|
||||
)
|
||||
payload = frame_json(frames[-1])
|
||||
assert payload is not None
|
||||
return payload, frames
|
||||
|
||||
|
||||
async def receive_close_code(websocket: ClientConnection) -> int:
|
||||
try:
|
||||
while True:
|
||||
await asyncio.wait_for(websocket.recv(), timeout=FRAME_TIMEOUT_SECONDS)
|
||||
except ConnectionClosed as exc:
|
||||
return exc.code
|
||||
except TimeoutError as exc:
|
||||
raise SoakFailure("server did not close WebSocket before timeout") from exc
|
||||
|
||||
|
||||
def json_frames(frames: list[str | bytes]) -> list[dict[str, Any]]:
|
||||
return [payload for raw in frames if (payload := frame_json(raw)) is not None]
|
||||
|
||||
|
||||
async def open_ready(uri: str, stats: SoakStats) -> ClientConnection:
|
||||
websocket = await connect(uri, open_timeout=FRAME_TIMEOUT_SECONDS, max_queue=4)
|
||||
stats.connections_opened += 1
|
||||
ready, _ = await receive_json_type(websocket, "ready")
|
||||
require(ready.get("state") == "idle", "ready frame did not start idle")
|
||||
require(
|
||||
ready.get("persona_catalog_source") == "synthetic_override",
|
||||
"route did not expose the explicit synthetic bind override",
|
||||
)
|
||||
return websocket
|
||||
|
||||
|
||||
async def close_normally(websocket: ClientConnection, stats: SoakStats) -> None:
|
||||
await websocket.send(json.dumps({"type": "close"}))
|
||||
code = await receive_close_code(websocket)
|
||||
require(code == 1000, f"expected normal close 1000, got {code}")
|
||||
stats.normal_close_1000 += 1
|
||||
stats.connections_completed += 1
|
||||
|
||||
|
||||
async def send_successful_utterance(
|
||||
websocket: ClientConnection,
|
||||
audio: bytes,
|
||||
) -> list[str | bytes]:
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio_start",
|
||||
"format": "pcm",
|
||||
"sample_rate": 16_000,
|
||||
"channels": 1,
|
||||
"sample_width": 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
listening, _ = await receive_json_type(websocket, "state")
|
||||
require(
|
||||
listening.get("state") == "listening", "audio_start did not enter listening"
|
||||
)
|
||||
await websocket.send(audio)
|
||||
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
||||
frames = await receive_until(
|
||||
websocket,
|
||||
lambda _raw, payload: payload is not None
|
||||
and payload.get("type") == "state"
|
||||
and payload.get("state") == "idle",
|
||||
)
|
||||
payloads = json_frames(frames)
|
||||
types = [item.get("type") for item in payloads]
|
||||
require("transcript" in types, "successful utterance omitted transcript frame")
|
||||
require("reply" in types, "successful utterance omitted reply frame")
|
||||
require("tts_end" in types, "successful utterance omitted tts_end frame")
|
||||
require(
|
||||
any(isinstance(frame, bytes) for frame in frames), "synthetic TTS bytes missing"
|
||||
)
|
||||
return frames
|
||||
|
||||
|
||||
async def send_blocked_utterance(
|
||||
websocket: ClientConnection,
|
||||
audio: bytes,
|
||||
*,
|
||||
expected_code: str,
|
||||
) -> None:
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio_start",
|
||||
"format": "pcm",
|
||||
"sample_rate": 16_000,
|
||||
"channels": 1,
|
||||
"sample_width": 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
listening, _ = await receive_json_type(websocket, "state")
|
||||
require(
|
||||
listening.get("state") == "listening",
|
||||
"blocked utterance did not enter listening",
|
||||
)
|
||||
await websocket.send(audio)
|
||||
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
||||
frames = await receive_until(
|
||||
websocket,
|
||||
lambda _raw, payload: payload is not None
|
||||
and payload.get("type") == "state"
|
||||
and payload.get("state") == "idle",
|
||||
)
|
||||
payloads = json_frames(frames)
|
||||
errors = [item for item in payloads if item.get("type") == "error"]
|
||||
require(
|
||||
any(item.get("code") == expected_code for item in errors),
|
||||
f"expected fail-closed code {expected_code!r}",
|
||||
)
|
||||
require(
|
||||
not any(
|
||||
item.get("type") in {"transcript", "reply", "tts_end"} for item in payloads
|
||||
),
|
||||
"blocked utterance leaked downstream transcript/reply/TTS frames",
|
||||
)
|
||||
|
||||
|
||||
async def exercise_auth_and_degraded_closes(
|
||||
base_uri: str,
|
||||
overrides: SyntheticOverrides,
|
||||
stats: SoakStats,
|
||||
) -> None:
|
||||
unauthorized = await connect(
|
||||
f"{base_uri}?session_id=g7-auth-denied&auth=denied",
|
||||
open_timeout=FRAME_TIMEOUT_SECONDS,
|
||||
)
|
||||
stats.connections_opened += 1
|
||||
error, _ = await receive_json_type(unauthorized, "error")
|
||||
require(error.get("detail") == "not authenticated", "unauthorized detail drifted")
|
||||
code = await receive_close_code(unauthorized)
|
||||
require(
|
||||
code == voice_routes.WS_CLOSE_UNAUTHORIZED, "unauthorized close code drifted"
|
||||
)
|
||||
stats.unauthorized_close_1008 += 1
|
||||
stats.connections_completed += 1
|
||||
|
||||
overrides.voice_available = False
|
||||
degraded = await connect(
|
||||
f"{base_uri}?session_id=g7-provider-degraded",
|
||||
open_timeout=FRAME_TIMEOUT_SECONDS,
|
||||
)
|
||||
stats.connections_opened += 1
|
||||
payload, _ = await receive_json_type(degraded, "degraded")
|
||||
require(
|
||||
payload.get("reason") == "voice STT/TTS is not configured",
|
||||
"degraded reason drifted",
|
||||
)
|
||||
code = await receive_close_code(degraded)
|
||||
require(code == voice_routes.WS_CLOSE_DEGRADED, "degraded close code drifted")
|
||||
stats.degraded_close_1011 += 1
|
||||
stats.connections_completed += 1
|
||||
overrides.voice_available = True
|
||||
|
||||
|
||||
async def exercise_consent_reconnect(
|
||||
base_uri: str,
|
||||
gate: ConsentGate,
|
||||
stats: SoakStats,
|
||||
audio: bytes,
|
||||
) -> None:
|
||||
session_id = "g7-consent-reconnect"
|
||||
gate.grant(session_id)
|
||||
first = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
||||
await send_successful_utterance(first, audio)
|
||||
await close_normally(first, stats)
|
||||
|
||||
second = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
||||
stats.reconnect_successes += 1
|
||||
await send_successful_utterance(second, audio)
|
||||
gate.withdraw(session_id)
|
||||
stt_before = stats.synthetic_stt_calls
|
||||
await send_blocked_utterance(
|
||||
second,
|
||||
audio,
|
||||
expected_code="multimodal_consent_withdrawn",
|
||||
)
|
||||
require(
|
||||
stats.synthetic_stt_calls == stt_before,
|
||||
"withdrawal did not stop before synthetic STT seam",
|
||||
)
|
||||
stats.withdrawal_blocks += 1
|
||||
await close_normally(second, stats)
|
||||
|
||||
third = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
||||
stats.reconnect_successes += 1
|
||||
await send_blocked_utterance(
|
||||
third,
|
||||
audio,
|
||||
expected_code="multimodal_consent_withdrawn",
|
||||
)
|
||||
require(
|
||||
stats.synthetic_stt_calls == stt_before,
|
||||
"withdrawn reconnect reached synthetic STT seam",
|
||||
)
|
||||
stats.withdrawal_blocks += 1
|
||||
await close_normally(third, stats)
|
||||
|
||||
missing_session = "g7-consent-missing"
|
||||
gate.clear(missing_session)
|
||||
missing = await open_ready(f"{base_uri}?session_id={missing_session}", stats)
|
||||
await send_blocked_utterance(
|
||||
missing,
|
||||
audio,
|
||||
expected_code="multimodal_consent_required",
|
||||
)
|
||||
stats.consent_required_blocks += 1
|
||||
await close_normally(missing, stats)
|
||||
|
||||
|
||||
async def exercise_bounded_buffer(
|
||||
base_uri: str,
|
||||
gate: ConsentGate,
|
||||
stats: SoakStats,
|
||||
*,
|
||||
audio_cap_bytes: int,
|
||||
chunk: bytes,
|
||||
) -> None:
|
||||
session_id = "g7-bounded-buffer"
|
||||
gate.grant(session_id)
|
||||
websocket = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio_start",
|
||||
"format": "pcm",
|
||||
"sample_rate": 16_000,
|
||||
"channels": 1,
|
||||
"sample_width": 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
listening, _ = await receive_json_type(websocket, "state")
|
||||
require(listening.get("state") == "listening", "overflow path did not listen")
|
||||
frames_to_overflow = audio_cap_bytes // len(chunk) + 1
|
||||
stt_before = stats.synthetic_stt_calls
|
||||
for _ in range(frames_to_overflow):
|
||||
await websocket.send(chunk)
|
||||
stats.burst_frames_sent += frames_to_overflow
|
||||
frames = await receive_until(
|
||||
websocket,
|
||||
lambda _raw, payload: payload is not None
|
||||
and payload.get("type") == "error"
|
||||
and "audio too large" in str(payload.get("detail")),
|
||||
)
|
||||
require(
|
||||
any(
|
||||
"audio too large" in str(item.get("detail")) for item in json_frames(frames)
|
||||
),
|
||||
"audio cap did not reject burst",
|
||||
)
|
||||
stats.overflow_rejections += 1
|
||||
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
||||
await receive_until(
|
||||
websocket,
|
||||
lambda _raw, payload: payload is not None
|
||||
and payload.get("type") == "state"
|
||||
and payload.get("state") == "idle",
|
||||
)
|
||||
require(
|
||||
stats.synthetic_stt_calls == stt_before,
|
||||
"overflowed audio reached synthetic STT seam",
|
||||
)
|
||||
await send_successful_utterance(websocket, chunk[: min(512, len(chunk))])
|
||||
stats.overflow_recoveries += 1
|
||||
await close_normally(websocket, stats)
|
||||
|
||||
|
||||
async def run_timed_soak(
|
||||
base_uri: str,
|
||||
gate: ConsentGate,
|
||||
stats: SoakStats,
|
||||
*,
|
||||
duration_seconds: float,
|
||||
cycle_interval_ms: int,
|
||||
audio: bytes,
|
||||
) -> None:
|
||||
session_id = "g7-timed-reconnect"
|
||||
gate.grant(session_id)
|
||||
started = time.monotonic()
|
||||
deadline = started + duration_seconds
|
||||
cycles = 0
|
||||
while time.monotonic() < deadline or cycles == 0:
|
||||
websocket = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
||||
if cycles > 0:
|
||||
stats.reconnect_successes += 1
|
||||
await websocket.send(json.dumps({"type": "ping"}))
|
||||
await receive_json_type(websocket, "pong")
|
||||
await send_successful_utterance(websocket, audio)
|
||||
await close_normally(websocket, stats)
|
||||
cycles += 1
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining > 0 and cycle_interval_ms > 0:
|
||||
await asyncio.sleep(min(remaining, cycle_interval_ms / 1000))
|
||||
stats.timed_soak_elapsed_seconds = round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
async def wait_for_server(server: uvicorn.Server, task: asyncio.Task[Any]) -> None:
|
||||
deadline = time.monotonic() + 5.0
|
||||
while not server.started:
|
||||
if task.done():
|
||||
await task
|
||||
raise SoakFailure("loopback Uvicorn server exited before startup")
|
||||
if time.monotonic() >= deadline:
|
||||
raise SoakFailure("loopback Uvicorn server startup timed out")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace, stats: SoakStats) -> None:
|
||||
gate = ConsentGate()
|
||||
overrides = SyntheticOverrides(gate, stats)
|
||||
app = FastAPI()
|
||||
app.include_router(voice_routes.router)
|
||||
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
listener.listen(128)
|
||||
host, port = listener.getsockname()
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=host,
|
||||
port=port,
|
||||
log_level="error",
|
||||
access_log=False,
|
||||
lifespan="off",
|
||||
ws="websockets",
|
||||
ws_max_queue=args.ws_max_queue,
|
||||
)
|
||||
server = uvicorn.Server(config)
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes,
|
||||
"_principal_from_websocket",
|
||||
new=overrides.principal_from_websocket,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(voice_routes, "_bind_session", new=overrides.bind_session)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes.voice_service,
|
||||
"is_available",
|
||||
new=lambda: overrides.voice_available,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes.voice_service,
|
||||
"tts_provider_for_voice",
|
||||
new=lambda _voice: "synthetic",
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes.voice_service,
|
||||
"transcribe",
|
||||
new=overrides.transcribe,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes,
|
||||
"_run_turn_and_speak",
|
||||
new=overrides.run_turn_and_speak,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
voice_routes.multimodal_alliance_store,
|
||||
"assert_voice_processing_allowed",
|
||||
new=gate.assert_allowed,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(voice_routes, "_MAX_AUDIO_BYTES", args.test_audio_cap_bytes)
|
||||
)
|
||||
|
||||
server_task = asyncio.create_task(server.serve(sockets=[listener]))
|
||||
try:
|
||||
await wait_for_server(server, server_task)
|
||||
base_uri = f"ws://127.0.0.1:{port}/voice/ws"
|
||||
audio = synthetic_pcm(args.chunk_bytes)
|
||||
await exercise_auth_and_degraded_closes(base_uri, overrides, stats)
|
||||
await exercise_consent_reconnect(base_uri, gate, stats, audio)
|
||||
await exercise_bounded_buffer(
|
||||
base_uri,
|
||||
gate,
|
||||
stats,
|
||||
audio_cap_bytes=args.test_audio_cap_bytes,
|
||||
chunk=audio,
|
||||
)
|
||||
await run_timed_soak(
|
||||
base_uri,
|
||||
gate,
|
||||
stats,
|
||||
duration_seconds=args.duration_seconds,
|
||||
cycle_interval_ms=args.cycle_interval_ms,
|
||||
audio=audio,
|
||||
)
|
||||
finally:
|
||||
server.should_exit = True
|
||||
try:
|
||||
await asyncio.wait_for(server_task, timeout=5.0)
|
||||
finally:
|
||||
listener.close()
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Loopback synthetic soak for /voice/ws. Use --duration-seconds 3000 "
|
||||
"for the optional 50-minute run."
|
||||
)
|
||||
)
|
||||
result.add_argument(
|
||||
"--duration-seconds",
|
||||
type=float,
|
||||
default=DEFAULT_DURATION_SECONDS,
|
||||
help="wall-clock timed reconnect soak duration; 3000 means 50 minutes",
|
||||
)
|
||||
result.add_argument(
|
||||
"--cycle-interval-ms",
|
||||
type=int,
|
||||
default=DEFAULT_CYCLE_INTERVAL_MS,
|
||||
help="delay between reconnect cycles",
|
||||
)
|
||||
result.add_argument(
|
||||
"--ws-max-queue",
|
||||
type=int,
|
||||
default=DEFAULT_WS_MAX_QUEUE,
|
||||
help="bounded Uvicorn/WebSockets inbound frame queue for this loopback server",
|
||||
)
|
||||
result.add_argument(
|
||||
"--test-audio-cap-bytes",
|
||||
type=int,
|
||||
default=DEFAULT_TEST_AUDIO_CAP_BYTES,
|
||||
help="reduced per-utterance cap override used to prove overflow quickly",
|
||||
)
|
||||
result.add_argument(
|
||||
"--chunk-bytes",
|
||||
type=int,
|
||||
default=DEFAULT_CHUNK_BYTES,
|
||||
help="even-sized deterministic PCM frame payload",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def validate_args(args: argparse.Namespace, cli: argparse.ArgumentParser) -> None:
|
||||
if args.duration_seconds <= 0:
|
||||
cli.error("--duration-seconds must be > 0")
|
||||
if args.cycle_interval_ms < 0:
|
||||
cli.error("--cycle-interval-ms must be >= 0")
|
||||
if args.ws_max_queue < 1:
|
||||
cli.error("--ws-max-queue must be >= 1")
|
||||
if args.chunk_bytes < 2 or args.chunk_bytes % 2:
|
||||
cli.error("--chunk-bytes must be a positive even number")
|
||||
if args.test_audio_cap_bytes < args.chunk_bytes:
|
||||
cli.error("--test-audio-cap-bytes must be >= --chunk-bytes")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
cli = parser()
|
||||
args = cli.parse_args()
|
||||
validate_args(args, cli)
|
||||
stats = SoakStats(
|
||||
requested_duration_seconds=args.duration_seconds,
|
||||
ws_max_queue=args.ws_max_queue,
|
||||
test_audio_cap_bytes=args.test_audio_cap_bytes,
|
||||
production_audio_cap_bytes=voice_routes._MAX_AUDIO_BYTES,
|
||||
streaming_event_queue_max_items=(
|
||||
voice_routes._STREAMING_EVENT_QUEUE_MAX_ITEMS
|
||||
),
|
||||
)
|
||||
try:
|
||||
exercise_production_cap_memory(stats)
|
||||
asyncio.run(run(args, stats))
|
||||
require(stats.paid_provider_calls == 0, "paid provider call counter changed")
|
||||
require(
|
||||
stats.connections_completed == stats.connections_opened, "connection leak"
|
||||
)
|
||||
require(stats.withdrawal_blocks >= 2, "withdrawal reconnect was not proven")
|
||||
require(stats.consent_required_blocks >= 1, "missing consent was not proven")
|
||||
require(stats.overflow_rejections == 1, "bounded buffer rejection missing")
|
||||
require(stats.overflow_recoveries == 1, "post-overflow recovery missing")
|
||||
require(
|
||||
stats.production_cap_buffer_high_water_bytes
|
||||
== stats.production_audio_cap_bytes,
|
||||
"production-cap allocation did not reach the configured boundary",
|
||||
)
|
||||
require(
|
||||
stats.production_cap_overflow_rejected,
|
||||
"production-cap overflow was not rejected before buffer growth",
|
||||
)
|
||||
require(
|
||||
stats.timed_soak_elapsed_seconds >= args.duration_seconds,
|
||||
"timed soak ended early",
|
||||
)
|
||||
stats.status = "passed"
|
||||
print(json.dumps(asdict(stats), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
except Exception as exc:
|
||||
stats.status = "failed"
|
||||
stats.unexpected_errors.append(f"{type(exc).__name__}: {exc}")
|
||||
print(json.dumps(asdict(stats), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue