"""Fail-closed tests for the complete G7 external proof checker.""" from __future__ import annotations import copy import importlib.util import sys import unittest from pathlib import Path CHECKER_PATH = Path(__file__).with_name("check-g7-external-proof.py") def load_checker(): spec = importlib.util.spec_from_file_location("g7_external_proof", CHECKER_PATH) if spec is None or spec.loader is None: raise RuntimeError("G7 external proof checker could not be loaded") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _sha(number: int) -> str: return f"{number:064x}" def human_pack() -> dict[str, object]: participants = [ { "participant_key": "calibration-000", "split": "calibration", "consent_receipt_sha256": _sha(1), } ] participants.extend( { "participant_key": f"held-{index:03d}", "split": "held_out", "consent_receipt_sha256": _sha(index + 2), } for index in range(30) ) observations = [] axes = ("goal", "task", "bond") for session_index in range(50): participant = f"held-{session_index % 30:03d}" for axis_index, axis in enumerate(axes): target = 0.2 + axis_index * 0.2 + (session_index % 5) * 0.01 observation_number = session_index * 3 + axis_index + 1 observations.append( { "observation_id": f"g7-human-observation-{observation_number:03d}", "participant_key": participant, "session_key": f"session-{session_index:03d}", "axis": axis, "text_only_status": "observed", "text_only_score": target + 0.10, "voice_enabled_status": "observed", "voice_enabled_score": target, "labels": [ {"labeler_key": "labeler-001", "score": target}, {"labeler_key": "labeler-002", "score": target}, ], } ) return { "provenance": { "protocol_sha256": _sha(100), "consent_protocol_sha256": _sha(101), "dataset_manifest_sha256": _sha(102), "split_manifest_sha256": _sha(103), "labeling_protocol_sha256": _sha(104), "analysis_plan_sha256": _sha(105), "registered_at": "2026-08-01T00:00:00Z", "held_out_labels_opened_at": "2026-08-02T00:00:00Z", }, "text_only_model": { "role": "text_only_baseline", "provider": "provider", "model_id": "baseline", "model_version": "v1", "artifact_sha256": _sha(106), "configuration_sha256": _sha(107), }, "voice_enabled_model": { "role": "voice_enabled_candidate", "provider": "provider", "model_id": "voice", "model_version": "v1", "artifact_sha256": _sha(108), "configuration_sha256": _sha(109), }, "power_plan": { "required_held_out_participants": 30, "required_held_out_sessions": 50, "required_paired_axis_observations": 150, "alpha": 0.05, "target_power": 0.8, "minimally_detectable_gain": 0.01, }, "participants": participants, "labeler_attestations": [ {"labeler_key": "labeler-001", "attestation_sha256": _sha(110)}, {"labeler_key": "labeler-002", "attestation_sha256": _sha(111)}, ], "reliability": { "labeler_keys": ["labeler-001", "labeler-002"], "reported_icc": 1.0, "report_sha256": _sha(112), }, "observations": observations, } def public_soak() -> dict[str, object]: metrics = [ { "turn_number": index + 1, "interim_transcript_frames": 1, "speech_final_transcript_frames": 1, "first_interim_latency_ms": 100.0, "speech_final_latency_ms": 300.0, } for index in range(10) ] return { "schema_version": "vignette.g7-public-voice-soak.v4", "mode": "public_soak", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "target_host": "api.example.test", "requested_duration_seconds": 3000.0, "elapsed_seconds": 3000.1, "microphone_device_enumerated": True, "physical_capture_confirmed": True, "physical_microphone_used": True, "raw_audio_retained": False, "public_wss_used": True, "authenticated_public_wss_ready": True, "real_database_session_binding_required": True, "provider_overrides_used": False, "ready_provider_metadata_validated": True, "cookie_present": True, "cookie_value_logged": False, "session_id_present": True, "cloudflare_ray_present": True, "unauthenticated_handshake_accepted": True, "tls_version": "TLSv1.3", "unauthenticated_close_code": 1008, "close_code": 1000, "expected_stt_provider": "local_whisper", "ready_stt_provider": "local_whisper", "expected_stt_model": "large-v3", "ready_stt_model": "large-v3", "expected_tts_provider": "melotts", "ready_tts_provider": "melotts", "expected_tts_model": "melotts-korean", "ready_tts_model": "melotts-korean", "turns_attempted": 10, "turns_succeeded": 10, "turn_transcript_metrics": metrics, "interim_transcript_frames": 10, "speech_final_transcript_frames": 10, "reply_frames": 10, "tts_end_frames": 10, "tts_binary_bytes": 1000, "captured_pcm_bytes": 1000, "blockers": [], "failure_type": None, } def runtime() -> dict[str, object]: snapshot = { "schema_version": "vignette.voice-runtime.v1", "scope": "single_api_worker", "privacy_boundary": "metadata_only_no_audio_transcript_or_session_ids", "limits": {"uvicorn_ws_max_queue": 4}, "process": {}, "counters": { "provider_fallback_total": 0, "websocket_error_total": 0, "audio_overflow_rejections_total": 0, }, } return { "schema_version": "vignette.g7-runtime-sampling.v1", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "target_host": "api.example.test", "worker_started_at_utc": "2026-08-07T00:00:00Z", "privacy_boundary": "metadata_only_no_audio_transcript_session_or_cookie_values", "cookie_present": True, "cookie_value_logged": False, "requested_samples": 31, "samples_completed": 31, "interval_seconds": 100.0, "samples": [ {"sequence": index + 1, "snapshot": copy.deepcopy(snapshot)} for index in range(31) ], "high_water": { "process_peak_rss_bytes": 1, "process_threads_max": 1, "websocket_high_water": 1, "streaming_provider_session_high_water": 1, "route_audio_buffer_high_water_bytes": 1, "streaming_event_queue_high_water_items": 1, }, } def topology() -> dict[str, object]: container_summary = { "cgroup_memory_peak_bytes_max": 1, "cgroup_cpu_usage_usec_max": 1, "cgroup_pids_current_max": 1, "proc_vm_hwm_bytes_max": 1, "proc_threads_max": 1, "proc_fd_count_max": 1, } return { "schema_version": "vignette.g7-topology-evidence.v1", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "scope": { "metadata_only": True, "raw_command_output_retained": False, "socket_endpoints_retained": False, "request_payloads_retained": False, "audio_retained": False, "transcripts_retained": False, "cloudflare_edge": { "internal_queue_measured": False, "evidence_boundary": "separate_external_artifact_required", }, }, "requested": { "public_host": "api.example.test", "samples": 31, "interval_seconds": 100.0, }, "samples_completed": 31, "targets": { role: { "container_id": character * 64, "image_digest": "sha256:" + character * 64, "started_at": "2026-08-07T00:00:00Z", } for role, character in (("api", "a"), ("caddy", "c")) }, "summary": { "containers": { "api": copy.deepcopy(container_summary), "caddy": copy.deepcopy(container_summary), }, "host_tcp": {"connections_max": 1}, }, "failure_type": None, } class G7ExternalProofTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.checker = load_checker() def test_complete_external_proof_passes(self) -> None: errors: list[str] = [] voice = public_soak() runtime_payload = runtime() topology_payload = topology() self.checker.validate_public_soak(voice, errors) self.checker.validate_runtime(runtime_payload, errors) self.checker.validate_topology(topology_payload, errors) gain = self.checker.validate_human_gain(human_pack(), errors) self.checker.validate_binding(voice, runtime_payload, topology_payload, errors) self.assertEqual([], errors) self.assertTrue(gain["passed"]) self.assertGreaterEqual(gain["held_out_participants"], 30) def test_decided_local_stack_is_accepted(self) -> None: """2026-08-08 소유자 결정: 노트북 faster-whisper STT + MeloTTS TTS (둘 다 MIT).""" errors: list[str] = [] voice = public_soak() self.assertEqual(voice["expected_stt_provider"], "local_whisper") self.assertEqual(voice["expected_tts_provider"], "melotts") self.checker.validate_public_soak(voice, errors) self.assertEqual([], errors) def test_deepgram_remains_an_operated_alternative(self) -> None: errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "deepgram" voice["ready_stt_provider"] = "deepgram" voice["expected_stt_model"] = "nova-3" voice["ready_stt_model"] = "nova-3" self.checker.validate_public_soak(voice, errors) self.assertEqual([], errors) def test_batch_openai_stt_cannot_satisfy_the_streaming_gate(self) -> None: errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "openai" voice["ready_stt_provider"] = "openai" self.checker.validate_public_soak(voice, errors) self.assertIn("voice_soak:stt_provider_not_operated", errors) def test_unoperated_providers_fail_closed(self) -> None: for field, value, code in ( ("expected_stt_provider", "some-other-vendor", "stt_provider_not_operated"), ("expected_tts_provider", "some-other-vendor", "tts_provider_not_operated"), ): with self.subTest(field=field): errors: list[str] = [] voice = public_soak() voice[field] = value voice[field.replace("expected", "ready")] = value self.checker.validate_public_soak(voice, errors) self.assertIn(f"voice_soak:{code}", errors) def test_declared_provider_must_match_the_runtime_provider(self) -> None: """허용 목록을 넓혀도 선언/실제 불일치는 계속 막아야 한다.""" errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "local_whisper" voice["ready_stt_provider"] = "deepgram" self.checker.validate_public_soak(voice, errors) self.assertIn("voice_soak:ready_stt_provider_mismatch", errors) def test_preflight_synthetic_and_nonoverlapping_evidence_fail(self) -> None: errors: list[str] = [] voice = public_soak() voice["mode"] = "preflight" voice["physical_microphone_used"] = False runtime_payload = runtime() runtime_payload["started_at_utc"] = "2026-08-08T00:00:00Z" runtime_payload["ended_at_utc"] = "2026-08-08T01:00:00Z" self.checker.validate_public_soak(voice, errors) self.checker.validate_binding(voice, runtime_payload, topology(), errors) self.assertIn("voice_soak:mode", errors) self.assertIn("voice_soak:physical_microphone_used", errors) self.assertIn("binding:no_concurrent_overlap", errors) def test_runtime_fallback_and_short_topology_fail(self) -> None: errors: list[str] = [] runtime_payload = runtime() runtime_payload["samples"][-1]["snapshot"]["counters"][ "provider_fallback_total" ] = 1 topology_payload = topology() topology_payload["requested"]["samples"] = 2 topology_payload["samples_completed"] = 2 self.checker.validate_runtime(runtime_payload, errors) self.checker.validate_topology(topology_payload, errors) self.assertIn("runtime:provider_fallback_total", errors) self.assertIn("topology:coverage", errors) if __name__ == "__main__": unittest.main()