G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
434
apps/api/app/services/voice_runtime.py
Normal file
434
apps/api/app/services/voice_runtime.py
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
"""Metadata-only runtime telemetry for the voice cascade.
|
||||
|
||||
The tracker is intentionally process-local. It never stores session IDs,
|
||||
transcripts, provider payloads, or audio bytes. A multi-worker deployment must
|
||||
sample every worker separately and aggregate outside the application.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import platform
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
VOICE_AUDIO_BUFFER_MAX_BYTES = 10 * 1024 * 1024
|
||||
VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS = 32
|
||||
VOICE_UVICORN_WS_MAX_QUEUE = 4
|
||||
|
||||
|
||||
class VoiceRuntimeLimits(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
max_utterance_audio_bytes: int = Field(ge=1)
|
||||
streaming_event_queue_max_items: int = Field(ge=1)
|
||||
uvicorn_ws_max_queue: int = Field(ge=1)
|
||||
|
||||
|
||||
class VoiceRuntimeProcess(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
worker_instance_id: str = Field(min_length=16, max_length=64)
|
||||
pid: int = Field(ge=1)
|
||||
platform: str
|
||||
started_at_utc: str
|
||||
uptime_seconds: float = Field(ge=0.0)
|
||||
rss_bytes: int = Field(ge=0)
|
||||
peak_rss_bytes: int = Field(ge=0)
|
||||
cpu_user_seconds: float = Field(ge=0.0)
|
||||
cpu_system_seconds: float = Field(ge=0.0)
|
||||
threads: int = Field(ge=1)
|
||||
open_file_descriptors: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class VoiceRuntimeCounters(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
active_websockets: int = Field(ge=0)
|
||||
websocket_high_water: int = Field(ge=0)
|
||||
websockets_opened_total: int = Field(ge=0)
|
||||
active_streaming_provider_sessions: int = Field(ge=0)
|
||||
streaming_provider_session_high_water: int = Field(ge=0)
|
||||
streaming_provider_sessions_opened_total: int = Field(ge=0)
|
||||
route_audio_buffer_bytes: int = Field(ge=0)
|
||||
route_audio_buffer_high_water_bytes: int = Field(ge=0)
|
||||
audio_bytes_received_total: int = Field(ge=0)
|
||||
audio_overflow_rejections_total: int = Field(ge=0)
|
||||
streaming_event_queue_items: int = Field(ge=0)
|
||||
streaming_event_queue_high_water_items: int = Field(ge=0)
|
||||
streaming_event_queue_saturation_total: int = Field(ge=0)
|
||||
streaming_event_queue_wait_seconds_total: float = Field(ge=0.0)
|
||||
provider_finalize_total: int = Field(ge=0)
|
||||
provider_abort_total: int = Field(ge=0)
|
||||
provider_fallback_total: int = Field(ge=0)
|
||||
websocket_error_total: int = Field(ge=0)
|
||||
|
||||
|
||||
class VoiceRuntimeSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
schema_version: str = "vignette.voice-runtime.v1"
|
||||
scope: str = "single_api_worker"
|
||||
privacy_boundary: str = "metadata_only_no_audio_transcript_or_session_ids"
|
||||
reset_supported: bool = False
|
||||
limits: VoiceRuntimeLimits
|
||||
process: VoiceRuntimeProcess
|
||||
counters: VoiceRuntimeCounters
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ConnectionState:
|
||||
audio_buffer_bytes: int = 0
|
||||
streaming_queue_items: int = 0
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _linux_process_memory_bytes() -> tuple[int, int]:
|
||||
status_path = Path("/proc/self/status")
|
||||
if not status_path.is_file():
|
||||
return 0, 0
|
||||
values: dict[str, int] = {}
|
||||
try:
|
||||
for line in status_path.read_text(encoding="ascii").splitlines():
|
||||
if line.startswith(("VmRSS:", "VmHWM:")):
|
||||
key, raw = line.split(":", 1)
|
||||
values[key] = int(raw.strip().split()[0]) * 1024
|
||||
except (OSError, UnicodeError, ValueError, IndexError):
|
||||
return 0, 0
|
||||
current = max(0, values.get("VmRSS", 0))
|
||||
peak = max(current, values.get("VmHWM", 0))
|
||||
return current, peak
|
||||
|
||||
|
||||
def _windows_process_memory_bytes() -> tuple[int, int]:
|
||||
if os.name != "nt":
|
||||
return 0, 0
|
||||
try:
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
class ProcessMemoryCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("cb", wintypes.DWORD),
|
||||
("PageFaultCount", wintypes.DWORD),
|
||||
("PeakWorkingSetSize", ctypes.c_size_t),
|
||||
("WorkingSetSize", ctypes.c_size_t),
|
||||
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
||||
("PagefileUsage", ctypes.c_size_t),
|
||||
("PeakPagefileUsage", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
psapi = ctypes.WinDLL("psapi", use_last_error=True)
|
||||
kernel32.GetCurrentProcess.restype = wintypes.HANDLE
|
||||
psapi.GetProcessMemoryInfo.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
ctypes.POINTER(ProcessMemoryCounters),
|
||||
wintypes.DWORD,
|
||||
]
|
||||
psapi.GetProcessMemoryInfo.restype = wintypes.BOOL
|
||||
|
||||
counters = ProcessMemoryCounters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
handle = kernel32.GetCurrentProcess()
|
||||
ok = psapi.GetProcessMemoryInfo(
|
||||
handle,
|
||||
ctypes.byref(counters),
|
||||
counters.cb,
|
||||
)
|
||||
if not ok:
|
||||
return 0, 0
|
||||
current = max(0, int(counters.WorkingSetSize))
|
||||
peak = max(current, int(counters.PeakWorkingSetSize))
|
||||
return current, peak
|
||||
except (AttributeError, OSError, TypeError, ValueError):
|
||||
return 0, 0
|
||||
|
||||
|
||||
def process_memory_bytes() -> tuple[int, int]:
|
||||
"""Return current and peak RSS when the host exposes process counters."""
|
||||
|
||||
current, peak = _linux_process_memory_bytes()
|
||||
if current or peak:
|
||||
return current, peak
|
||||
current, peak = _windows_process_memory_bytes()
|
||||
if current or peak:
|
||||
return current, peak
|
||||
try:
|
||||
import resource
|
||||
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
raw_peak = max(0, int(usage.ru_maxrss))
|
||||
scaled_peak = raw_peak if platform.system() == "Darwin" else raw_peak * 1024
|
||||
return 0, scaled_peak
|
||||
except (ImportError, OSError, ValueError):
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _open_file_descriptor_count() -> int | None:
|
||||
fd_root = Path("/proc/self/fd")
|
||||
if not fd_root.is_dir():
|
||||
return None
|
||||
try:
|
||||
return sum(1 for _ in fd_root.iterdir())
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
class VoiceRuntimeMetrics:
|
||||
"""Thread-safe, metadata-only high-water tracker for one API worker."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._started_monotonic = time.monotonic()
|
||||
self._started_at_utc = _utc_now()
|
||||
self._worker_instance_id = secrets.token_hex(12)
|
||||
self._next_connection_id = 0
|
||||
self._connections: dict[int, _ConnectionState] = {}
|
||||
self._websocket_high_water = 0
|
||||
self._websockets_opened_total = 0
|
||||
self._active_streaming_provider_sessions = 0
|
||||
self._streaming_provider_session_high_water = 0
|
||||
self._streaming_provider_sessions_opened_total = 0
|
||||
self._route_audio_buffer_bytes = 0
|
||||
self._route_audio_buffer_high_water_bytes = 0
|
||||
self._audio_bytes_received_total = 0
|
||||
self._audio_overflow_rejections_total = 0
|
||||
self._streaming_event_queue_items = 0
|
||||
self._streaming_event_queue_high_water_items = 0
|
||||
self._streaming_event_queue_saturation_total = 0
|
||||
self._streaming_event_queue_wait_seconds_total = 0.0
|
||||
self._provider_finalize_total = 0
|
||||
self._provider_abort_total = 0
|
||||
self._provider_fallback_total = 0
|
||||
self._websocket_error_total = 0
|
||||
|
||||
def websocket_opened(self) -> int:
|
||||
with self._lock:
|
||||
self._next_connection_id += 1
|
||||
connection_id = self._next_connection_id
|
||||
self._connections[connection_id] = _ConnectionState()
|
||||
self._websockets_opened_total += 1
|
||||
self._websocket_high_water = max(
|
||||
self._websocket_high_water,
|
||||
len(self._connections),
|
||||
)
|
||||
return connection_id
|
||||
|
||||
def websocket_closed(self, connection_id: int) -> None:
|
||||
with self._lock:
|
||||
connection = self._connections.pop(connection_id, None)
|
||||
if connection is not None:
|
||||
self._route_audio_buffer_bytes = max(
|
||||
0,
|
||||
self._route_audio_buffer_bytes - connection.audio_buffer_bytes,
|
||||
)
|
||||
self._streaming_event_queue_items = max(
|
||||
0,
|
||||
self._streaming_event_queue_items
|
||||
- connection.streaming_queue_items,
|
||||
)
|
||||
|
||||
def audio_chunk_received(
|
||||
self,
|
||||
connection_id: int,
|
||||
*,
|
||||
current_buffer_bytes: int,
|
||||
chunk_bytes: int,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
connection = self._connections.get(connection_id)
|
||||
if connection is None:
|
||||
return
|
||||
next_buffer_bytes = max(0, current_buffer_bytes)
|
||||
self._route_audio_buffer_bytes = max(
|
||||
0,
|
||||
self._route_audio_buffer_bytes
|
||||
- connection.audio_buffer_bytes
|
||||
+ next_buffer_bytes,
|
||||
)
|
||||
connection.audio_buffer_bytes = next_buffer_bytes
|
||||
self._audio_bytes_received_total += max(0, chunk_bytes)
|
||||
self._route_audio_buffer_high_water_bytes = max(
|
||||
self._route_audio_buffer_high_water_bytes,
|
||||
self._route_audio_buffer_bytes,
|
||||
)
|
||||
|
||||
def audio_buffer_cleared(self, connection_id: int) -> None:
|
||||
with self._lock:
|
||||
connection = self._connections.get(connection_id)
|
||||
if connection is not None:
|
||||
self._route_audio_buffer_bytes = max(
|
||||
0,
|
||||
self._route_audio_buffer_bytes - connection.audio_buffer_bytes,
|
||||
)
|
||||
connection.audio_buffer_bytes = 0
|
||||
|
||||
def audio_overflow_rejected(self, connection_id: int) -> None:
|
||||
with self._lock:
|
||||
self._audio_overflow_rejections_total += 1
|
||||
connection = self._connections.get(connection_id)
|
||||
if connection is not None:
|
||||
self._route_audio_buffer_bytes = max(
|
||||
0,
|
||||
self._route_audio_buffer_bytes - connection.audio_buffer_bytes,
|
||||
)
|
||||
connection.audio_buffer_bytes = 0
|
||||
|
||||
def streaming_queue_observed(
|
||||
self,
|
||||
connection_id: int,
|
||||
*,
|
||||
queue_items: int,
|
||||
saturated: bool = False,
|
||||
wait_seconds: float = 0.0,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
connection = self._connections.get(connection_id)
|
||||
if connection is None:
|
||||
return
|
||||
next_queue_items = max(0, queue_items)
|
||||
self._streaming_event_queue_items = max(
|
||||
0,
|
||||
self._streaming_event_queue_items
|
||||
- connection.streaming_queue_items
|
||||
+ next_queue_items,
|
||||
)
|
||||
connection.streaming_queue_items = next_queue_items
|
||||
self._streaming_event_queue_high_water_items = max(
|
||||
self._streaming_event_queue_high_water_items,
|
||||
self._streaming_event_queue_items,
|
||||
)
|
||||
if saturated:
|
||||
self._streaming_event_queue_saturation_total += 1
|
||||
self._streaming_event_queue_wait_seconds_total += max(
|
||||
0.0,
|
||||
wait_seconds,
|
||||
)
|
||||
|
||||
def streaming_provider_opened(self) -> None:
|
||||
with self._lock:
|
||||
self._active_streaming_provider_sessions += 1
|
||||
self._streaming_provider_sessions_opened_total += 1
|
||||
self._streaming_provider_session_high_water = max(
|
||||
self._streaming_provider_session_high_water,
|
||||
self._active_streaming_provider_sessions,
|
||||
)
|
||||
|
||||
def streaming_provider_closed(self, *, outcome: str) -> None:
|
||||
with self._lock:
|
||||
self._active_streaming_provider_sessions = max(
|
||||
0,
|
||||
self._active_streaming_provider_sessions - 1,
|
||||
)
|
||||
if outcome == "finalized":
|
||||
self._provider_finalize_total += 1
|
||||
else:
|
||||
self._provider_abort_total += 1
|
||||
|
||||
def provider_fallback(self) -> None:
|
||||
with self._lock:
|
||||
self._provider_fallback_total += 1
|
||||
|
||||
def websocket_error(self) -> None:
|
||||
with self._lock:
|
||||
self._websocket_error_total += 1
|
||||
|
||||
def snapshot(self) -> VoiceRuntimeSnapshot:
|
||||
with self._lock:
|
||||
counters = VoiceRuntimeCounters(
|
||||
active_websockets=len(self._connections),
|
||||
websocket_high_water=self._websocket_high_water,
|
||||
websockets_opened_total=self._websockets_opened_total,
|
||||
active_streaming_provider_sessions=(
|
||||
self._active_streaming_provider_sessions
|
||||
),
|
||||
streaming_provider_session_high_water=(
|
||||
self._streaming_provider_session_high_water
|
||||
),
|
||||
streaming_provider_sessions_opened_total=(
|
||||
self._streaming_provider_sessions_opened_total
|
||||
),
|
||||
route_audio_buffer_bytes=self._route_audio_buffer_bytes,
|
||||
route_audio_buffer_high_water_bytes=(
|
||||
self._route_audio_buffer_high_water_bytes
|
||||
),
|
||||
audio_bytes_received_total=self._audio_bytes_received_total,
|
||||
audio_overflow_rejections_total=(
|
||||
self._audio_overflow_rejections_total
|
||||
),
|
||||
streaming_event_queue_items=self._streaming_event_queue_items,
|
||||
streaming_event_queue_high_water_items=(
|
||||
self._streaming_event_queue_high_water_items
|
||||
),
|
||||
streaming_event_queue_saturation_total=(
|
||||
self._streaming_event_queue_saturation_total
|
||||
),
|
||||
streaming_event_queue_wait_seconds_total=round(
|
||||
self._streaming_event_queue_wait_seconds_total,
|
||||
6,
|
||||
),
|
||||
provider_finalize_total=self._provider_finalize_total,
|
||||
provider_abort_total=self._provider_abort_total,
|
||||
provider_fallback_total=self._provider_fallback_total,
|
||||
websocket_error_total=self._websocket_error_total,
|
||||
)
|
||||
|
||||
rss_bytes, peak_rss_bytes = process_memory_bytes()
|
||||
process_times = os.times()
|
||||
return VoiceRuntimeSnapshot(
|
||||
limits=VoiceRuntimeLimits(
|
||||
max_utterance_audio_bytes=VOICE_AUDIO_BUFFER_MAX_BYTES,
|
||||
streaming_event_queue_max_items=(
|
||||
VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS
|
||||
),
|
||||
uvicorn_ws_max_queue=VOICE_UVICORN_WS_MAX_QUEUE,
|
||||
),
|
||||
process=VoiceRuntimeProcess(
|
||||
worker_instance_id=self._worker_instance_id,
|
||||
pid=os.getpid(),
|
||||
platform=platform.system().lower() or "unknown",
|
||||
started_at_utc=self._started_at_utc,
|
||||
uptime_seconds=round(
|
||||
max(0.0, time.monotonic() - self._started_monotonic),
|
||||
3,
|
||||
),
|
||||
rss_bytes=rss_bytes,
|
||||
peak_rss_bytes=max(rss_bytes, peak_rss_bytes),
|
||||
cpu_user_seconds=max(0.0, float(process_times.user)),
|
||||
cpu_system_seconds=max(0.0, float(process_times.system)),
|
||||
threads=max(1, threading.active_count()),
|
||||
open_file_descriptors=_open_file_descriptor_count(),
|
||||
),
|
||||
counters=counters,
|
||||
)
|
||||
|
||||
|
||||
voice_runtime_metrics = VoiceRuntimeMetrics()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"VOICE_AUDIO_BUFFER_MAX_BYTES",
|
||||
"VOICE_STREAMING_EVENT_QUEUE_MAX_ITEMS",
|
||||
"VOICE_UVICORN_WS_MAX_QUEUE",
|
||||
"VoiceRuntimeCounters",
|
||||
"VoiceRuntimeLimits",
|
||||
"VoiceRuntimeMetrics",
|
||||
"VoiceRuntimeProcess",
|
||||
"VoiceRuntimeSnapshot",
|
||||
"process_memory_bytes",
|
||||
"voice_runtime_metrics",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue