8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
900 lines
30 KiB
Python
900 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Deterministic loopback soak for the real ``/voice/ws`` ASGI route.
|
|
|
|
This runner starts an ephemeral Uvicorn server on 127.0.0.1, includes the
|
|
production voice router, and drives it through a real WebSocket transport. It
|
|
does not use a physical microphone, public WSS, a real database, or paid
|
|
STT/TTS/LLM providers. Those boundaries are replaced at explicit seams while
|
|
the route framing, consent preflight, audio normalization, byte cap, and close
|
|
codes remain active.
|
|
|
|
Exit codes:
|
|
0 = every deterministic contract and timed soak cycle passed
|
|
1 = a route/transport contract failed
|
|
2 = invalid CLI usage (argparse)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import ctypes
|
|
import gc
|
|
import json
|
|
import math
|
|
import os
|
|
import socket
|
|
import struct
|
|
import sys
|
|
import time
|
|
from contextlib import ExitStack
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
from unittest.mock import patch
|
|
|
|
import uvicorn
|
|
from fastapi import FastAPI
|
|
from websockets.asyncio.client import ClientConnection, connect
|
|
from websockets.exceptions import ConnectionClosed
|
|
|
|
|
|
API_ROOT = Path(__file__).resolve().parents[1] / "apps" / "api"
|
|
if str(API_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(API_ROOT))
|
|
|
|
from app.deps import Principal, Role # noqa: E402 - API_ROOT is injected above
|
|
from app.routes import voice as voice_routes # noqa: E402 - API_ROOT is injected above
|
|
from app.services import multimodal_alliance_store # noqa: E402 - API_ROOT is injected above
|
|
from app.services.voice import VoicePreset # noqa: E402 - API_ROOT is injected above
|
|
|
|
|
|
DEFAULT_DURATION_SECONDS = 8.0
|
|
DEFAULT_CYCLE_INTERVAL_MS = 100
|
|
DEFAULT_WS_MAX_QUEUE = 4
|
|
DEFAULT_TEST_AUDIO_CAP_BYTES = 4_096
|
|
DEFAULT_CHUNK_BYTES = 1_024
|
|
FRAME_TIMEOUT_SECONDS = 3.0
|
|
VOICE_PRESET = VoicePreset(preset="neutral", openai_voice="sage")
|
|
|
|
|
|
class SoakFailure(RuntimeError):
|
|
"""A deterministic WebSocket contract was violated."""
|
|
|
|
|
|
@dataclass
|
|
class SoakStats:
|
|
schema_version: str = "vignette.g7-voice-ws-soak.v1"
|
|
status: str = "running"
|
|
route: str = "/voice/ws"
|
|
transport: str = "loopback_uvicorn_websocket"
|
|
requested_duration_seconds: float = 0.0
|
|
timed_soak_elapsed_seconds: float = 0.0
|
|
ws_max_queue: int = 0
|
|
test_audio_cap_bytes: int = 0
|
|
production_audio_cap_bytes: int = 0
|
|
physical_microphone_used: bool = False
|
|
public_wss_used: bool = False
|
|
real_database_used: bool = False
|
|
paid_provider_calls: int = 0
|
|
connections_opened: int = 0
|
|
connections_completed: int = 0
|
|
reconnect_successes: int = 0
|
|
normal_close_1000: int = 0
|
|
unauthorized_close_1008: int = 0
|
|
degraded_close_1011: int = 0
|
|
successful_utterances: int = 0
|
|
synthetic_stt_calls: int = 0
|
|
synthetic_tts_frames: int = 0
|
|
consent_required_blocks: int = 0
|
|
withdrawal_blocks: int = 0
|
|
overflow_rejections: int = 0
|
|
overflow_recoveries: int = 0
|
|
burst_frames_sent: int = 0
|
|
max_normalized_audio_bytes_seen: int = 0
|
|
streaming_event_queue_max_items: int = 0
|
|
production_cap_buffer_high_water_bytes: int = 0
|
|
production_cap_overflow_rejected: bool = False
|
|
rss_before_production_cap_probe_bytes: int = 0
|
|
rss_at_production_cap_bytes: int = 0
|
|
rss_after_production_cap_clear_bytes: int = 0
|
|
peak_working_set_delta_bytes: int = 0
|
|
unexpected_errors: list[str] = field(default_factory=list)
|
|
overrides: list[str] = field(
|
|
default_factory=lambda: [
|
|
"_principal_from_websocket -> deterministic learner principal",
|
|
"_bind_session -> deterministic existing-session binding",
|
|
"voice_service.transcribe -> local synthetic transcript",
|
|
"_run_turn_and_speak -> local deterministic reply/TTS frames",
|
|
"multimodal assert_voice_processing_allowed -> in-memory consent gate",
|
|
"voice_service.is_available -> controllable local readiness",
|
|
"_MAX_AUDIO_BYTES -> reduced test cap for fast overflow proof",
|
|
]
|
|
)
|
|
|
|
|
|
class ConsentGate:
|
|
def __init__(self) -> None:
|
|
self._states: dict[str, str] = {}
|
|
|
|
def grant(self, session_id: str) -> None:
|
|
self._states[session_id] = "granted"
|
|
|
|
def withdraw(self, session_id: str) -> None:
|
|
self._states[session_id] = "withdrawn"
|
|
|
|
def clear(self, session_id: str) -> None:
|
|
self._states.pop(session_id, None)
|
|
|
|
async def assert_allowed(self, *, principal: Principal, session_id: str) -> None:
|
|
del principal
|
|
state = self._states.get(session_id)
|
|
if state == "withdrawn":
|
|
raise multimodal_alliance_store.MultimodalConsentWithdrawnError(
|
|
"multimodal voice consent was withdrawn"
|
|
)
|
|
if state != "granted":
|
|
raise multimodal_alliance_store.MultimodalConsentRequiredError(
|
|
"active multimodal voice consent is required"
|
|
)
|
|
|
|
|
|
class SyntheticOverrides:
|
|
def __init__(self, gate: ConsentGate, stats: SoakStats) -> None:
|
|
self.gate = gate
|
|
self.stats = stats
|
|
self.voice_available = True
|
|
self.principal = Principal(
|
|
user_id="77000000-0000-4000-8000-000000000201",
|
|
role=Role.LEARNER,
|
|
cohort_ids=["g7-loopback-soak"],
|
|
email="g7-loopback-soak@invalid.local",
|
|
display_name="G7 Loopback Soak",
|
|
consent_at=1.0,
|
|
profile_completed_at=1.0,
|
|
)
|
|
|
|
async def principal_from_websocket(self, websocket: Any) -> Principal | None:
|
|
if websocket.query_params.get("auth") == "denied":
|
|
return None
|
|
return self.principal
|
|
|
|
async def bind_session(
|
|
self,
|
|
websocket: Any,
|
|
principal: Principal,
|
|
) -> tuple[str, VoicePreset, None, dict[str, object]]:
|
|
del principal
|
|
session_id = websocket.query_params.get("session_id") or "g7-soak-session"
|
|
return (
|
|
session_id,
|
|
VOICE_PRESET,
|
|
None,
|
|
{"degraded": False, "persona_catalog_source": "synthetic_override"},
|
|
)
|
|
|
|
async def transcribe(
|
|
self,
|
|
audio: bytes,
|
|
*,
|
|
filename: str,
|
|
content_type: str,
|
|
) -> SimpleNamespace:
|
|
if not audio.startswith(b"RIFF") or audio[8:12] != b"WAVE":
|
|
raise SoakFailure("PCM audio did not traverse production WAV normalization")
|
|
if filename != "audio.wav" or content_type != "audio/wav":
|
|
raise SoakFailure("normalized audio metadata drifted from WAV")
|
|
self.stats.synthetic_stt_calls += 1
|
|
self.stats.max_normalized_audio_bytes_seen = max(
|
|
self.stats.max_normalized_audio_bytes_seen,
|
|
len(audio),
|
|
)
|
|
return SimpleNamespace(
|
|
text="합성 음성 회기 응답",
|
|
duration=0.032,
|
|
provider_events=[
|
|
{
|
|
"event_type": "speech_final",
|
|
"category": "speech_activity",
|
|
"confidence": 1.0,
|
|
}
|
|
],
|
|
)
|
|
|
|
async def run_turn_and_speak(
|
|
self,
|
|
websocket: Any,
|
|
context: voice_routes.VoiceSessionContext,
|
|
turn: voice_routes.VoiceTurnInput,
|
|
) -> None:
|
|
if not turn.learner_text:
|
|
raise SoakFailure("synthetic turn reached reply seam without transcript")
|
|
await voice_routes._safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "reply",
|
|
"text": "결정적 합성 내담자 응답",
|
|
"speaker": "client",
|
|
"stage": "초기",
|
|
"turn_seq": self.stats.successful_utterances + 1,
|
|
"synthetic_override": True,
|
|
},
|
|
)
|
|
await voice_routes._safe_send_json(
|
|
websocket,
|
|
{
|
|
"type": "state",
|
|
"state": "speaking",
|
|
"voice": context.voice_preset.openai_voice,
|
|
"tts_provider": "synthetic",
|
|
},
|
|
)
|
|
payload = b"g7-synthetic-tts-frame"
|
|
await voice_routes._safe_send_bytes(websocket, payload)
|
|
self.stats.synthetic_tts_frames += 1
|
|
await voice_routes._safe_send_json(
|
|
websocket,
|
|
{"type": "tts_end", "chunks": 1, "synthetic_override": True},
|
|
)
|
|
await voice_routes._safe_send_json(
|
|
websocket,
|
|
{"type": "state", "state": "idle"},
|
|
)
|
|
self.stats.successful_utterances += 1
|
|
|
|
|
|
def synthetic_pcm(chunk_bytes: int = DEFAULT_CHUNK_BYTES) -> bytes:
|
|
"""Return deterministic 16 kHz mono signed-16 PCM, never microphone input."""
|
|
|
|
if chunk_bytes <= 0 or chunk_bytes % 2:
|
|
raise ValueError("chunk_bytes must be a positive even number")
|
|
samples = chunk_bytes // 2
|
|
payload = bytearray()
|
|
for index in range(samples):
|
|
value = int(1_200 * math.sin(2 * math.pi * 220 * index / 16_000))
|
|
payload.extend(struct.pack("<h", value))
|
|
return bytes(payload)
|
|
|
|
|
|
def require(condition: bool, message: str) -> None:
|
|
if not condition:
|
|
raise SoakFailure(message)
|
|
|
|
|
|
def process_memory_bytes() -> tuple[int, int]:
|
|
"""Return current RSS and process peak working set without extra packages."""
|
|
|
|
if os.name == "nt":
|
|
class ProcessMemoryCounters(ctypes.Structure):
|
|
_fields_ = [
|
|
("cb", ctypes.c_ulong),
|
|
("PageFaultCount", ctypes.c_ulong),
|
|
("PeakWorkingSetSize", ctypes.c_size_t),
|
|
("WorkingSetSize", ctypes.c_size_t),
|
|
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
|
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
|
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
|
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
|
("PagefileUsage", ctypes.c_size_t),
|
|
("PeakPagefileUsage", ctypes.c_size_t),
|
|
]
|
|
|
|
counters = ProcessMemoryCounters()
|
|
counters.cb = ctypes.sizeof(counters)
|
|
get_current_process = ctypes.windll.kernel32.GetCurrentProcess
|
|
get_current_process.restype = ctypes.c_void_p
|
|
get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo
|
|
get_process_memory_info.argtypes = [
|
|
ctypes.c_void_p,
|
|
ctypes.POINTER(ProcessMemoryCounters),
|
|
ctypes.c_ulong,
|
|
]
|
|
get_process_memory_info.restype = ctypes.c_int
|
|
handle = get_current_process()
|
|
ok = get_process_memory_info(
|
|
handle,
|
|
ctypes.byref(counters),
|
|
counters.cb,
|
|
)
|
|
if not ok:
|
|
raise SoakFailure("GetProcessMemoryInfo failed")
|
|
return int(counters.WorkingSetSize), int(counters.PeakWorkingSetSize)
|
|
|
|
import resource
|
|
|
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
peak = int(usage.ru_maxrss)
|
|
if sys.platform != "darwin":
|
|
peak *= 1024
|
|
return peak, peak
|
|
|
|
|
|
def exercise_production_cap_memory(stats: SoakStats) -> None:
|
|
"""Allocate exactly one production utterance cap and reject one extra byte."""
|
|
|
|
gc.collect()
|
|
rss_before, peak_before = process_memory_bytes()
|
|
buffer = bytearray()
|
|
chunk = bytes(64 * 1024)
|
|
while len(buffer) < voice_routes._MAX_AUDIO_BYTES:
|
|
remaining = voice_routes._MAX_AUDIO_BYTES - len(buffer)
|
|
require(
|
|
voice_routes._append_audio_chunk_with_cap(buffer, chunk[:remaining]),
|
|
"production-cap probe rejected an in-bound chunk",
|
|
)
|
|
stats.production_cap_buffer_high_water_bytes = len(buffer)
|
|
stats.production_cap_overflow_rejected = not (
|
|
voice_routes._append_audio_chunk_with_cap(buffer, b"x")
|
|
)
|
|
rss_at_cap, peak_at_cap = process_memory_bytes()
|
|
buffer.clear()
|
|
del buffer
|
|
gc.collect()
|
|
rss_after, peak_after = process_memory_bytes()
|
|
stats.rss_before_production_cap_probe_bytes = rss_before
|
|
stats.rss_at_production_cap_bytes = rss_at_cap
|
|
stats.rss_after_production_cap_clear_bytes = rss_after
|
|
stats.peak_working_set_delta_bytes = max(
|
|
0, max(peak_at_cap, peak_after) - peak_before
|
|
)
|
|
|
|
|
|
def frame_json(raw: str | bytes) -> dict[str, Any] | None:
|
|
if isinstance(raw, bytes):
|
|
return None
|
|
try:
|
|
value = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
raise SoakFailure("server emitted non-JSON text frame") from exc
|
|
if not isinstance(value, dict):
|
|
raise SoakFailure("server emitted non-object JSON frame")
|
|
return value
|
|
|
|
|
|
async def receive_until(
|
|
websocket: ClientConnection,
|
|
predicate,
|
|
*,
|
|
timeout: float = FRAME_TIMEOUT_SECONDS,
|
|
) -> list[str | bytes]:
|
|
deadline = time.monotonic() + timeout
|
|
frames: list[str | bytes] = []
|
|
while time.monotonic() < deadline:
|
|
remaining = max(0.01, deadline - time.monotonic())
|
|
try:
|
|
raw = await asyncio.wait_for(websocket.recv(), timeout=remaining)
|
|
except TimeoutError as exc:
|
|
raise SoakFailure("timed out waiting for WebSocket contract frame") from exc
|
|
frames.append(raw)
|
|
payload = frame_json(raw)
|
|
if predicate(raw, payload):
|
|
return frames
|
|
raise SoakFailure("WebSocket predicate was not reached before deadline")
|
|
|
|
|
|
async def receive_json_type(
|
|
websocket: ClientConnection,
|
|
expected_type: str,
|
|
) -> tuple[dict[str, Any], list[str | bytes]]:
|
|
frames = await receive_until(
|
|
websocket,
|
|
lambda _raw, payload: payload is not None
|
|
and payload.get("type") == expected_type,
|
|
)
|
|
payload = frame_json(frames[-1])
|
|
assert payload is not None
|
|
return payload, frames
|
|
|
|
|
|
async def receive_close_code(websocket: ClientConnection) -> int:
|
|
try:
|
|
while True:
|
|
await asyncio.wait_for(websocket.recv(), timeout=FRAME_TIMEOUT_SECONDS)
|
|
except ConnectionClosed as exc:
|
|
return exc.code
|
|
except TimeoutError as exc:
|
|
raise SoakFailure("server did not close WebSocket before timeout") from exc
|
|
|
|
|
|
def json_frames(frames: list[str | bytes]) -> list[dict[str, Any]]:
|
|
return [payload for raw in frames if (payload := frame_json(raw)) is not None]
|
|
|
|
|
|
async def open_ready(uri: str, stats: SoakStats) -> ClientConnection:
|
|
websocket = await connect(uri, open_timeout=FRAME_TIMEOUT_SECONDS, max_queue=4)
|
|
stats.connections_opened += 1
|
|
ready, _ = await receive_json_type(websocket, "ready")
|
|
require(ready.get("state") == "idle", "ready frame did not start idle")
|
|
require(
|
|
ready.get("persona_catalog_source") == "synthetic_override",
|
|
"route did not expose the explicit synthetic bind override",
|
|
)
|
|
return websocket
|
|
|
|
|
|
async def close_normally(websocket: ClientConnection, stats: SoakStats) -> None:
|
|
await websocket.send(json.dumps({"type": "close"}))
|
|
code = await receive_close_code(websocket)
|
|
require(code == 1000, f"expected normal close 1000, got {code}")
|
|
stats.normal_close_1000 += 1
|
|
stats.connections_completed += 1
|
|
|
|
|
|
async def send_successful_utterance(
|
|
websocket: ClientConnection,
|
|
audio: bytes,
|
|
) -> list[str | bytes]:
|
|
await websocket.send(
|
|
json.dumps(
|
|
{
|
|
"type": "audio_start",
|
|
"format": "pcm",
|
|
"sample_rate": 16_000,
|
|
"channels": 1,
|
|
"sample_width": 2,
|
|
}
|
|
)
|
|
)
|
|
listening, _ = await receive_json_type(websocket, "state")
|
|
require(
|
|
listening.get("state") == "listening", "audio_start did not enter listening"
|
|
)
|
|
await websocket.send(audio)
|
|
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
|
frames = await receive_until(
|
|
websocket,
|
|
lambda _raw, payload: payload is not None
|
|
and payload.get("type") == "state"
|
|
and payload.get("state") == "idle",
|
|
)
|
|
payloads = json_frames(frames)
|
|
types = [item.get("type") for item in payloads]
|
|
require("transcript" in types, "successful utterance omitted transcript frame")
|
|
require("reply" in types, "successful utterance omitted reply frame")
|
|
require("tts_end" in types, "successful utterance omitted tts_end frame")
|
|
require(
|
|
any(isinstance(frame, bytes) for frame in frames), "synthetic TTS bytes missing"
|
|
)
|
|
return frames
|
|
|
|
|
|
async def send_blocked_utterance(
|
|
websocket: ClientConnection,
|
|
audio: bytes,
|
|
*,
|
|
expected_code: str,
|
|
) -> None:
|
|
await websocket.send(
|
|
json.dumps(
|
|
{
|
|
"type": "audio_start",
|
|
"format": "pcm",
|
|
"sample_rate": 16_000,
|
|
"channels": 1,
|
|
"sample_width": 2,
|
|
}
|
|
)
|
|
)
|
|
listening, _ = await receive_json_type(websocket, "state")
|
|
require(
|
|
listening.get("state") == "listening",
|
|
"blocked utterance did not enter listening",
|
|
)
|
|
await websocket.send(audio)
|
|
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
|
frames = await receive_until(
|
|
websocket,
|
|
lambda _raw, payload: payload is not None
|
|
and payload.get("type") == "state"
|
|
and payload.get("state") == "idle",
|
|
)
|
|
payloads = json_frames(frames)
|
|
errors = [item for item in payloads if item.get("type") == "error"]
|
|
require(
|
|
any(item.get("code") == expected_code for item in errors),
|
|
f"expected fail-closed code {expected_code!r}",
|
|
)
|
|
require(
|
|
not any(
|
|
item.get("type") in {"transcript", "reply", "tts_end"} for item in payloads
|
|
),
|
|
"blocked utterance leaked downstream transcript/reply/TTS frames",
|
|
)
|
|
|
|
|
|
async def exercise_auth_and_degraded_closes(
|
|
base_uri: str,
|
|
overrides: SyntheticOverrides,
|
|
stats: SoakStats,
|
|
) -> None:
|
|
unauthorized = await connect(
|
|
f"{base_uri}?session_id=g7-auth-denied&auth=denied",
|
|
open_timeout=FRAME_TIMEOUT_SECONDS,
|
|
)
|
|
stats.connections_opened += 1
|
|
error, _ = await receive_json_type(unauthorized, "error")
|
|
require(error.get("detail") == "not authenticated", "unauthorized detail drifted")
|
|
code = await receive_close_code(unauthorized)
|
|
require(
|
|
code == voice_routes.WS_CLOSE_UNAUTHORIZED, "unauthorized close code drifted"
|
|
)
|
|
stats.unauthorized_close_1008 += 1
|
|
stats.connections_completed += 1
|
|
|
|
overrides.voice_available = False
|
|
degraded = await connect(
|
|
f"{base_uri}?session_id=g7-provider-degraded",
|
|
open_timeout=FRAME_TIMEOUT_SECONDS,
|
|
)
|
|
stats.connections_opened += 1
|
|
payload, _ = await receive_json_type(degraded, "degraded")
|
|
require(
|
|
payload.get("reason") == "voice STT/TTS is not configured",
|
|
"degraded reason drifted",
|
|
)
|
|
code = await receive_close_code(degraded)
|
|
require(code == voice_routes.WS_CLOSE_DEGRADED, "degraded close code drifted")
|
|
stats.degraded_close_1011 += 1
|
|
stats.connections_completed += 1
|
|
overrides.voice_available = True
|
|
|
|
|
|
async def exercise_consent_reconnect(
|
|
base_uri: str,
|
|
gate: ConsentGate,
|
|
stats: SoakStats,
|
|
audio: bytes,
|
|
) -> None:
|
|
session_id = "g7-consent-reconnect"
|
|
gate.grant(session_id)
|
|
first = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
|
await send_successful_utterance(first, audio)
|
|
await close_normally(first, stats)
|
|
|
|
second = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
|
stats.reconnect_successes += 1
|
|
await send_successful_utterance(second, audio)
|
|
gate.withdraw(session_id)
|
|
stt_before = stats.synthetic_stt_calls
|
|
await send_blocked_utterance(
|
|
second,
|
|
audio,
|
|
expected_code="multimodal_consent_withdrawn",
|
|
)
|
|
require(
|
|
stats.synthetic_stt_calls == stt_before,
|
|
"withdrawal did not stop before synthetic STT seam",
|
|
)
|
|
stats.withdrawal_blocks += 1
|
|
await close_normally(second, stats)
|
|
|
|
third = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
|
stats.reconnect_successes += 1
|
|
await send_blocked_utterance(
|
|
third,
|
|
audio,
|
|
expected_code="multimodal_consent_withdrawn",
|
|
)
|
|
require(
|
|
stats.synthetic_stt_calls == stt_before,
|
|
"withdrawn reconnect reached synthetic STT seam",
|
|
)
|
|
stats.withdrawal_blocks += 1
|
|
await close_normally(third, stats)
|
|
|
|
missing_session = "g7-consent-missing"
|
|
gate.clear(missing_session)
|
|
missing = await open_ready(f"{base_uri}?session_id={missing_session}", stats)
|
|
await send_blocked_utterance(
|
|
missing,
|
|
audio,
|
|
expected_code="multimodal_consent_required",
|
|
)
|
|
stats.consent_required_blocks += 1
|
|
await close_normally(missing, stats)
|
|
|
|
|
|
async def exercise_bounded_buffer(
|
|
base_uri: str,
|
|
gate: ConsentGate,
|
|
stats: SoakStats,
|
|
*,
|
|
audio_cap_bytes: int,
|
|
chunk: bytes,
|
|
) -> None:
|
|
session_id = "g7-bounded-buffer"
|
|
gate.grant(session_id)
|
|
websocket = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
|
await websocket.send(
|
|
json.dumps(
|
|
{
|
|
"type": "audio_start",
|
|
"format": "pcm",
|
|
"sample_rate": 16_000,
|
|
"channels": 1,
|
|
"sample_width": 2,
|
|
}
|
|
)
|
|
)
|
|
listening, _ = await receive_json_type(websocket, "state")
|
|
require(listening.get("state") == "listening", "overflow path did not listen")
|
|
frames_to_overflow = audio_cap_bytes // len(chunk) + 1
|
|
stt_before = stats.synthetic_stt_calls
|
|
for _ in range(frames_to_overflow):
|
|
await websocket.send(chunk)
|
|
stats.burst_frames_sent += frames_to_overflow
|
|
frames = await receive_until(
|
|
websocket,
|
|
lambda _raw, payload: payload is not None
|
|
and payload.get("type") == "error"
|
|
and "audio too large" in str(payload.get("detail")),
|
|
)
|
|
require(
|
|
any(
|
|
"audio too large" in str(item.get("detail")) for item in json_frames(frames)
|
|
),
|
|
"audio cap did not reject burst",
|
|
)
|
|
stats.overflow_rejections += 1
|
|
await websocket.send(json.dumps({"type": "audio_end", "format": "pcm"}))
|
|
await receive_until(
|
|
websocket,
|
|
lambda _raw, payload: payload is not None
|
|
and payload.get("type") == "state"
|
|
and payload.get("state") == "idle",
|
|
)
|
|
require(
|
|
stats.synthetic_stt_calls == stt_before,
|
|
"overflowed audio reached synthetic STT seam",
|
|
)
|
|
await send_successful_utterance(websocket, chunk[: min(512, len(chunk))])
|
|
stats.overflow_recoveries += 1
|
|
await close_normally(websocket, stats)
|
|
|
|
|
|
async def run_timed_soak(
|
|
base_uri: str,
|
|
gate: ConsentGate,
|
|
stats: SoakStats,
|
|
*,
|
|
duration_seconds: float,
|
|
cycle_interval_ms: int,
|
|
audio: bytes,
|
|
) -> None:
|
|
session_id = "g7-timed-reconnect"
|
|
gate.grant(session_id)
|
|
started = time.monotonic()
|
|
deadline = started + duration_seconds
|
|
cycles = 0
|
|
while time.monotonic() < deadline or cycles == 0:
|
|
websocket = await open_ready(f"{base_uri}?session_id={session_id}", stats)
|
|
if cycles > 0:
|
|
stats.reconnect_successes += 1
|
|
await websocket.send(json.dumps({"type": "ping"}))
|
|
await receive_json_type(websocket, "pong")
|
|
await send_successful_utterance(websocket, audio)
|
|
await close_normally(websocket, stats)
|
|
cycles += 1
|
|
remaining = deadline - time.monotonic()
|
|
if remaining > 0 and cycle_interval_ms > 0:
|
|
await asyncio.sleep(min(remaining, cycle_interval_ms / 1000))
|
|
stats.timed_soak_elapsed_seconds = round(time.monotonic() - started, 3)
|
|
|
|
|
|
async def wait_for_server(server: uvicorn.Server, task: asyncio.Task[Any]) -> None:
|
|
deadline = time.monotonic() + 5.0
|
|
while not server.started:
|
|
if task.done():
|
|
await task
|
|
raise SoakFailure("loopback Uvicorn server exited before startup")
|
|
if time.monotonic() >= deadline:
|
|
raise SoakFailure("loopback Uvicorn server startup timed out")
|
|
await asyncio.sleep(0.01)
|
|
|
|
|
|
async def run(args: argparse.Namespace, stats: SoakStats) -> None:
|
|
gate = ConsentGate()
|
|
overrides = SyntheticOverrides(gate, stats)
|
|
app = FastAPI()
|
|
app.include_router(voice_routes.router)
|
|
|
|
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
listener.bind(("127.0.0.1", 0))
|
|
listener.listen(128)
|
|
host, port = listener.getsockname()
|
|
config = uvicorn.Config(
|
|
app,
|
|
host=host,
|
|
port=port,
|
|
log_level="error",
|
|
access_log=False,
|
|
lifespan="off",
|
|
ws="websockets",
|
|
ws_max_queue=args.ws_max_queue,
|
|
)
|
|
server = uvicorn.Server(config)
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes,
|
|
"_principal_from_websocket",
|
|
new=overrides.principal_from_websocket,
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(voice_routes, "_bind_session", new=overrides.bind_session)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes.voice_service,
|
|
"is_available",
|
|
new=lambda: overrides.voice_available,
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes.voice_service,
|
|
"tts_provider_for_voice",
|
|
new=lambda _voice: "synthetic",
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes.voice_service,
|
|
"transcribe",
|
|
new=overrides.transcribe,
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes,
|
|
"_run_turn_and_speak",
|
|
new=overrides.run_turn_and_speak,
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(
|
|
voice_routes.multimodal_alliance_store,
|
|
"assert_voice_processing_allowed",
|
|
new=gate.assert_allowed,
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch.object(voice_routes, "_MAX_AUDIO_BYTES", args.test_audio_cap_bytes)
|
|
)
|
|
|
|
server_task = asyncio.create_task(server.serve(sockets=[listener]))
|
|
try:
|
|
await wait_for_server(server, server_task)
|
|
base_uri = f"ws://127.0.0.1:{port}/voice/ws"
|
|
audio = synthetic_pcm(args.chunk_bytes)
|
|
await exercise_auth_and_degraded_closes(base_uri, overrides, stats)
|
|
await exercise_consent_reconnect(base_uri, gate, stats, audio)
|
|
await exercise_bounded_buffer(
|
|
base_uri,
|
|
gate,
|
|
stats,
|
|
audio_cap_bytes=args.test_audio_cap_bytes,
|
|
chunk=audio,
|
|
)
|
|
await run_timed_soak(
|
|
base_uri,
|
|
gate,
|
|
stats,
|
|
duration_seconds=args.duration_seconds,
|
|
cycle_interval_ms=args.cycle_interval_ms,
|
|
audio=audio,
|
|
)
|
|
finally:
|
|
server.should_exit = True
|
|
try:
|
|
await asyncio.wait_for(server_task, timeout=5.0)
|
|
finally:
|
|
listener.close()
|
|
|
|
|
|
def parser() -> argparse.ArgumentParser:
|
|
result = argparse.ArgumentParser(
|
|
description=(
|
|
"Loopback synthetic soak for /voice/ws. Use --duration-seconds 3000 "
|
|
"for the optional 50-minute run."
|
|
)
|
|
)
|
|
result.add_argument(
|
|
"--duration-seconds",
|
|
type=float,
|
|
default=DEFAULT_DURATION_SECONDS,
|
|
help="wall-clock timed reconnect soak duration; 3000 means 50 minutes",
|
|
)
|
|
result.add_argument(
|
|
"--cycle-interval-ms",
|
|
type=int,
|
|
default=DEFAULT_CYCLE_INTERVAL_MS,
|
|
help="delay between reconnect cycles",
|
|
)
|
|
result.add_argument(
|
|
"--ws-max-queue",
|
|
type=int,
|
|
default=DEFAULT_WS_MAX_QUEUE,
|
|
help="bounded Uvicorn/WebSockets inbound frame queue for this loopback server",
|
|
)
|
|
result.add_argument(
|
|
"--test-audio-cap-bytes",
|
|
type=int,
|
|
default=DEFAULT_TEST_AUDIO_CAP_BYTES,
|
|
help="reduced per-utterance cap override used to prove overflow quickly",
|
|
)
|
|
result.add_argument(
|
|
"--chunk-bytes",
|
|
type=int,
|
|
default=DEFAULT_CHUNK_BYTES,
|
|
help="even-sized deterministic PCM frame payload",
|
|
)
|
|
return result
|
|
|
|
|
|
def validate_args(args: argparse.Namespace, cli: argparse.ArgumentParser) -> None:
|
|
if args.duration_seconds <= 0:
|
|
cli.error("--duration-seconds must be > 0")
|
|
if args.cycle_interval_ms < 0:
|
|
cli.error("--cycle-interval-ms must be >= 0")
|
|
if args.ws_max_queue < 1:
|
|
cli.error("--ws-max-queue must be >= 1")
|
|
if args.chunk_bytes < 2 or args.chunk_bytes % 2:
|
|
cli.error("--chunk-bytes must be a positive even number")
|
|
if args.test_audio_cap_bytes < args.chunk_bytes:
|
|
cli.error("--test-audio-cap-bytes must be >= --chunk-bytes")
|
|
|
|
|
|
def main() -> int:
|
|
cli = parser()
|
|
args = cli.parse_args()
|
|
validate_args(args, cli)
|
|
stats = SoakStats(
|
|
requested_duration_seconds=args.duration_seconds,
|
|
ws_max_queue=args.ws_max_queue,
|
|
test_audio_cap_bytes=args.test_audio_cap_bytes,
|
|
production_audio_cap_bytes=voice_routes._MAX_AUDIO_BYTES,
|
|
streaming_event_queue_max_items=(
|
|
voice_routes._STREAMING_EVENT_QUEUE_MAX_ITEMS
|
|
),
|
|
)
|
|
try:
|
|
exercise_production_cap_memory(stats)
|
|
asyncio.run(run(args, stats))
|
|
require(stats.paid_provider_calls == 0, "paid provider call counter changed")
|
|
require(
|
|
stats.connections_completed == stats.connections_opened, "connection leak"
|
|
)
|
|
require(stats.withdrawal_blocks >= 2, "withdrawal reconnect was not proven")
|
|
require(stats.consent_required_blocks >= 1, "missing consent was not proven")
|
|
require(stats.overflow_rejections == 1, "bounded buffer rejection missing")
|
|
require(stats.overflow_recoveries == 1, "post-overflow recovery missing")
|
|
require(
|
|
stats.production_cap_buffer_high_water_bytes
|
|
== stats.production_audio_cap_bytes,
|
|
"production-cap allocation did not reach the configured boundary",
|
|
)
|
|
require(
|
|
stats.production_cap_overflow_rejected,
|
|
"production-cap overflow was not rejected before buffer growth",
|
|
)
|
|
require(
|
|
stats.timed_soak_elapsed_seconds >= args.duration_seconds,
|
|
"timed soak ended early",
|
|
)
|
|
stats.status = "passed"
|
|
print(json.dumps(asdict(stats), ensure_ascii=False, indent=2, sort_keys=True))
|
|
return 0
|
|
except Exception as exc:
|
|
stats.status = "failed"
|
|
stats.unexpected_errors.append(f"{type(exc).__name__}: {exc}")
|
|
print(json.dumps(asdict(stats), ensure_ascii=False, indent=2, sort_keys=True))
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|