G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
374
scripts/test_g7_external_proof.py
Normal file
374
scripts/test_g7_external_proof.py
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
"""Fail-closed tests for the complete G7 external proof checker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CHECKER_PATH = Path(__file__).with_name("check-g7-external-proof.py")
|
||||
|
||||
|
||||
def load_checker():
|
||||
spec = importlib.util.spec_from_file_location("g7_external_proof", CHECKER_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("G7 external proof checker could not be loaded")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _sha(number: int) -> str:
|
||||
return f"{number:064x}"
|
||||
|
||||
|
||||
def human_pack() -> dict[str, object]:
|
||||
participants = [
|
||||
{
|
||||
"participant_key": "calibration-000",
|
||||
"split": "calibration",
|
||||
"consent_receipt_sha256": _sha(1),
|
||||
}
|
||||
]
|
||||
participants.extend(
|
||||
{
|
||||
"participant_key": f"held-{index:03d}",
|
||||
"split": "held_out",
|
||||
"consent_receipt_sha256": _sha(index + 2),
|
||||
}
|
||||
for index in range(30)
|
||||
)
|
||||
observations = []
|
||||
axes = ("goal", "task", "bond")
|
||||
for session_index in range(50):
|
||||
participant = f"held-{session_index % 30:03d}"
|
||||
for axis_index, axis in enumerate(axes):
|
||||
target = 0.2 + axis_index * 0.2 + (session_index % 5) * 0.01
|
||||
observation_number = session_index * 3 + axis_index + 1
|
||||
observations.append(
|
||||
{
|
||||
"observation_id": f"g7-human-observation-{observation_number:03d}",
|
||||
"participant_key": participant,
|
||||
"session_key": f"session-{session_index:03d}",
|
||||
"axis": axis,
|
||||
"text_only_status": "observed",
|
||||
"text_only_score": target + 0.10,
|
||||
"voice_enabled_status": "observed",
|
||||
"voice_enabled_score": target,
|
||||
"labels": [
|
||||
{"labeler_key": "labeler-001", "score": target},
|
||||
{"labeler_key": "labeler-002", "score": target},
|
||||
],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"provenance": {
|
||||
"protocol_sha256": _sha(100),
|
||||
"consent_protocol_sha256": _sha(101),
|
||||
"dataset_manifest_sha256": _sha(102),
|
||||
"split_manifest_sha256": _sha(103),
|
||||
"labeling_protocol_sha256": _sha(104),
|
||||
"analysis_plan_sha256": _sha(105),
|
||||
"registered_at": "2026-08-01T00:00:00Z",
|
||||
"held_out_labels_opened_at": "2026-08-02T00:00:00Z",
|
||||
},
|
||||
"text_only_model": {
|
||||
"role": "text_only_baseline",
|
||||
"provider": "provider",
|
||||
"model_id": "baseline",
|
||||
"model_version": "v1",
|
||||
"artifact_sha256": _sha(106),
|
||||
"configuration_sha256": _sha(107),
|
||||
},
|
||||
"voice_enabled_model": {
|
||||
"role": "voice_enabled_candidate",
|
||||
"provider": "provider",
|
||||
"model_id": "voice",
|
||||
"model_version": "v1",
|
||||
"artifact_sha256": _sha(108),
|
||||
"configuration_sha256": _sha(109),
|
||||
},
|
||||
"power_plan": {
|
||||
"required_held_out_participants": 30,
|
||||
"required_held_out_sessions": 50,
|
||||
"required_paired_axis_observations": 150,
|
||||
"alpha": 0.05,
|
||||
"target_power": 0.8,
|
||||
"minimally_detectable_gain": 0.01,
|
||||
},
|
||||
"participants": participants,
|
||||
"labeler_attestations": [
|
||||
{"labeler_key": "labeler-001", "attestation_sha256": _sha(110)},
|
||||
{"labeler_key": "labeler-002", "attestation_sha256": _sha(111)},
|
||||
],
|
||||
"reliability": {
|
||||
"labeler_keys": ["labeler-001", "labeler-002"],
|
||||
"reported_icc": 1.0,
|
||||
"report_sha256": _sha(112),
|
||||
},
|
||||
"observations": observations,
|
||||
}
|
||||
|
||||
|
||||
def public_soak() -> dict[str, object]:
|
||||
metrics = [
|
||||
{
|
||||
"turn_number": index + 1,
|
||||
"interim_transcript_frames": 1,
|
||||
"speech_final_transcript_frames": 1,
|
||||
"first_interim_latency_ms": 100.0,
|
||||
"speech_final_latency_ms": 300.0,
|
||||
}
|
||||
for index in range(10)
|
||||
]
|
||||
return {
|
||||
"schema_version": "vignette.g7-public-voice-soak.v4",
|
||||
"mode": "public_soak",
|
||||
"status": "passed",
|
||||
"started_at_utc": "2026-08-07T00:00:00Z",
|
||||
"ended_at_utc": "2026-08-07T01:00:00Z",
|
||||
"target_host": "api.example.test",
|
||||
"requested_duration_seconds": 3000.0,
|
||||
"elapsed_seconds": 3000.1,
|
||||
"microphone_device_enumerated": True,
|
||||
"physical_capture_confirmed": True,
|
||||
"physical_microphone_used": True,
|
||||
"raw_audio_retained": False,
|
||||
"public_wss_used": True,
|
||||
"authenticated_public_wss_ready": True,
|
||||
"real_database_session_binding_required": True,
|
||||
"provider_overrides_used": False,
|
||||
"ready_provider_metadata_validated": True,
|
||||
"cookie_present": True,
|
||||
"cookie_value_logged": False,
|
||||
"session_id_present": True,
|
||||
"cloudflare_ray_present": True,
|
||||
"unauthenticated_handshake_accepted": True,
|
||||
"tls_version": "TLSv1.3",
|
||||
"unauthenticated_close_code": 1008,
|
||||
"close_code": 1000,
|
||||
"expected_stt_provider": "local_whisper",
|
||||
"ready_stt_provider": "local_whisper",
|
||||
"expected_stt_model": "large-v3",
|
||||
"ready_stt_model": "large-v3",
|
||||
"expected_tts_provider": "higgs",
|
||||
"ready_tts_provider": "higgs",
|
||||
"expected_tts_model": "higgs-audio-v3-tts-4b",
|
||||
"ready_tts_model": "higgs-audio-v3-tts-4b",
|
||||
"turns_attempted": 10,
|
||||
"turns_succeeded": 10,
|
||||
"turn_transcript_metrics": metrics,
|
||||
"interim_transcript_frames": 10,
|
||||
"speech_final_transcript_frames": 10,
|
||||
"reply_frames": 10,
|
||||
"tts_end_frames": 10,
|
||||
"tts_binary_bytes": 1000,
|
||||
"captured_pcm_bytes": 1000,
|
||||
"blockers": [],
|
||||
"failure_type": None,
|
||||
}
|
||||
|
||||
|
||||
def runtime() -> dict[str, object]:
|
||||
snapshot = {
|
||||
"schema_version": "vignette.voice-runtime.v1",
|
||||
"scope": "single_api_worker",
|
||||
"privacy_boundary": "metadata_only_no_audio_transcript_or_session_ids",
|
||||
"limits": {"uvicorn_ws_max_queue": 4},
|
||||
"process": {},
|
||||
"counters": {
|
||||
"provider_fallback_total": 0,
|
||||
"websocket_error_total": 0,
|
||||
"audio_overflow_rejections_total": 0,
|
||||
},
|
||||
}
|
||||
return {
|
||||
"schema_version": "vignette.g7-runtime-sampling.v1",
|
||||
"status": "passed",
|
||||
"started_at_utc": "2026-08-07T00:00:00Z",
|
||||
"ended_at_utc": "2026-08-07T01:00:00Z",
|
||||
"target_host": "api.example.test",
|
||||
"worker_started_at_utc": "2026-08-07T00:00:00Z",
|
||||
"privacy_boundary": "metadata_only_no_audio_transcript_session_or_cookie_values",
|
||||
"cookie_present": True,
|
||||
"cookie_value_logged": False,
|
||||
"requested_samples": 31,
|
||||
"samples_completed": 31,
|
||||
"interval_seconds": 100.0,
|
||||
"samples": [
|
||||
{"sequence": index + 1, "snapshot": copy.deepcopy(snapshot)}
|
||||
for index in range(31)
|
||||
],
|
||||
"high_water": {
|
||||
"process_peak_rss_bytes": 1,
|
||||
"process_threads_max": 1,
|
||||
"websocket_high_water": 1,
|
||||
"streaming_provider_session_high_water": 1,
|
||||
"route_audio_buffer_high_water_bytes": 1,
|
||||
"streaming_event_queue_high_water_items": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def topology() -> dict[str, object]:
|
||||
container_summary = {
|
||||
"cgroup_memory_peak_bytes_max": 1,
|
||||
"cgroup_cpu_usage_usec_max": 1,
|
||||
"cgroup_pids_current_max": 1,
|
||||
"proc_vm_hwm_bytes_max": 1,
|
||||
"proc_threads_max": 1,
|
||||
"proc_fd_count_max": 1,
|
||||
}
|
||||
return {
|
||||
"schema_version": "vignette.g7-topology-evidence.v1",
|
||||
"status": "passed",
|
||||
"started_at_utc": "2026-08-07T00:00:00Z",
|
||||
"ended_at_utc": "2026-08-07T01:00:00Z",
|
||||
"scope": {
|
||||
"metadata_only": True,
|
||||
"raw_command_output_retained": False,
|
||||
"socket_endpoints_retained": False,
|
||||
"request_payloads_retained": False,
|
||||
"audio_retained": False,
|
||||
"transcripts_retained": False,
|
||||
"cloudflare_edge": {
|
||||
"internal_queue_measured": False,
|
||||
"evidence_boundary": "separate_external_artifact_required",
|
||||
},
|
||||
},
|
||||
"requested": {
|
||||
"public_host": "api.example.test",
|
||||
"samples": 31,
|
||||
"interval_seconds": 100.0,
|
||||
},
|
||||
"samples_completed": 31,
|
||||
"targets": {
|
||||
role: {
|
||||
"container_id": character * 64,
|
||||
"image_digest": "sha256:" + character * 64,
|
||||
"started_at": "2026-08-07T00:00:00Z",
|
||||
}
|
||||
for role, character in (("api", "a"), ("caddy", "c"))
|
||||
},
|
||||
"summary": {
|
||||
"containers": {
|
||||
"api": copy.deepcopy(container_summary),
|
||||
"caddy": copy.deepcopy(container_summary),
|
||||
},
|
||||
"host_tcp": {"connections_max": 1},
|
||||
},
|
||||
"failure_type": None,
|
||||
}
|
||||
|
||||
|
||||
class G7ExternalProofTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.checker = load_checker()
|
||||
|
||||
def test_complete_external_proof_passes(self) -> None:
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
runtime_payload = runtime()
|
||||
topology_payload = topology()
|
||||
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.checker.validate_runtime(runtime_payload, errors)
|
||||
self.checker.validate_topology(topology_payload, errors)
|
||||
gain = self.checker.validate_human_gain(human_pack(), errors)
|
||||
self.checker.validate_binding(voice, runtime_payload, topology_payload, errors)
|
||||
|
||||
self.assertEqual([], errors)
|
||||
self.assertTrue(gain["passed"])
|
||||
self.assertGreaterEqual(gain["held_out_participants"], 30)
|
||||
|
||||
def test_decided_local_stack_is_accepted(self) -> None:
|
||||
"""2026-08-08 소유자 결정: 노트북 faster-whisper STT + Higgs TTS."""
|
||||
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
self.assertEqual(voice["expected_stt_provider"], "local_whisper")
|
||||
self.assertEqual(voice["expected_tts_provider"], "higgs")
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.assertEqual([], errors)
|
||||
|
||||
def test_deepgram_remains_an_operated_alternative(self) -> None:
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
voice["expected_stt_provider"] = "deepgram"
|
||||
voice["ready_stt_provider"] = "deepgram"
|
||||
voice["expected_stt_model"] = "nova-3"
|
||||
voice["ready_stt_model"] = "nova-3"
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.assertEqual([], errors)
|
||||
|
||||
def test_batch_openai_stt_cannot_satisfy_the_streaming_gate(self) -> None:
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
voice["expected_stt_provider"] = "openai"
|
||||
voice["ready_stt_provider"] = "openai"
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.assertIn("voice_soak:stt_provider_not_operated", errors)
|
||||
|
||||
def test_unoperated_providers_fail_closed(self) -> None:
|
||||
for field, value, code in (
|
||||
("expected_stt_provider", "some-other-vendor", "stt_provider_not_operated"),
|
||||
("expected_tts_provider", "some-other-vendor", "tts_provider_not_operated"),
|
||||
):
|
||||
with self.subTest(field=field):
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
voice[field] = value
|
||||
voice[field.replace("expected", "ready")] = value
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.assertIn(f"voice_soak:{code}", errors)
|
||||
|
||||
def test_declared_provider_must_match_the_runtime_provider(self) -> None:
|
||||
"""허용 목록을 넓혀도 선언/실제 불일치는 계속 막아야 한다."""
|
||||
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
voice["expected_stt_provider"] = "local_whisper"
|
||||
voice["ready_stt_provider"] = "deepgram"
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.assertIn("voice_soak:ready_stt_provider_mismatch", errors)
|
||||
|
||||
def test_preflight_synthetic_and_nonoverlapping_evidence_fail(self) -> None:
|
||||
errors: list[str] = []
|
||||
voice = public_soak()
|
||||
voice["mode"] = "preflight"
|
||||
voice["physical_microphone_used"] = False
|
||||
runtime_payload = runtime()
|
||||
runtime_payload["started_at_utc"] = "2026-08-08T00:00:00Z"
|
||||
runtime_payload["ended_at_utc"] = "2026-08-08T01:00:00Z"
|
||||
|
||||
self.checker.validate_public_soak(voice, errors)
|
||||
self.checker.validate_binding(voice, runtime_payload, topology(), errors)
|
||||
|
||||
self.assertIn("voice_soak:mode", errors)
|
||||
self.assertIn("voice_soak:physical_microphone_used", errors)
|
||||
self.assertIn("binding:no_concurrent_overlap", errors)
|
||||
|
||||
def test_runtime_fallback_and_short_topology_fail(self) -> None:
|
||||
errors: list[str] = []
|
||||
runtime_payload = runtime()
|
||||
runtime_payload["samples"][-1]["snapshot"]["counters"][
|
||||
"provider_fallback_total"
|
||||
] = 1
|
||||
topology_payload = topology()
|
||||
topology_payload["requested"]["samples"] = 2
|
||||
topology_payload["samples_completed"] = 2
|
||||
|
||||
self.checker.validate_runtime(runtime_payload, errors)
|
||||
self.checker.validate_topology(topology_payload, errors)
|
||||
|
||||
self.assertIn("runtime:provider_fallback_total", errors)
|
||||
self.assertIn("topology:coverage", errors)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue