"""Fail-closed tests for the complete G7 external proof checker.""" from __future__ import annotations import copy import importlib.util import sys import unittest from pathlib import Path CHECKER_PATH = Path(__file__).with_name("check-g7-external-proof.py") def load_checker(): spec = importlib.util.spec_from_file_location("g7_external_proof", CHECKER_PATH) if spec is None or spec.loader is None: raise RuntimeError("G7 external proof checker could not be loaded") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _sha(number: int) -> str: return f"{number:064x}" def human_pack() -> dict[str, object]: participants = [ { "participant_key": "calibration-000", "split": "calibration", "consent_receipt_sha256": _sha(1), } ] participants.extend( { "participant_key": f"held-{index:03d}", "split": "held_out", "consent_receipt_sha256": _sha(index + 2), } for index in range(30) ) observations = [] axes = ("goal", "task", "bond") categories = ("low", "medium", "high") for session_index in range(50): participant = f"held-{session_index % 30:03d}" for axis_index, axis in enumerate(axes): target = 0.2 + axis_index * 0.2 + (session_index % 5) * 0.01 observation_number = session_index * 3 + axis_index + 1 observations.append( { "observation_id": f"g7-human-observation-{observation_number:03d}", "participant_key": participant, "session_key": f"session-{session_index:03d}", "axis": axis, "text_only_status": "observed", "text_only_score": target + 0.10, "voice_enabled_status": "observed", "voice_enabled_score": target, "labels": [ { "labeler_key": "labeler-001", "score": target, "category": categories[axis_index], }, { "labeler_key": "labeler-002", "score": target, "category": categories[axis_index], }, ], } ) return { "provenance": { "protocol_sha256": _sha(100), "consent_protocol_sha256": _sha(101), "dataset_manifest_sha256": _sha(102), "split_manifest_sha256": _sha(103), "labeling_protocol_sha256": _sha(104), "analysis_plan_sha256": _sha(105), "registered_at": "2026-08-01T00:00:00Z", "held_out_labels_opened_at": "2026-08-02T00:00:00Z", }, "text_only_model": { "role": "text_only_baseline", "provider": "provider", "model_id": "baseline", "model_version": "v1", "artifact_sha256": _sha(106), "configuration_sha256": _sha(107), }, "voice_enabled_model": { "role": "voice_enabled_candidate", "provider": "provider", "model_id": "voice", "model_version": "v1", "artifact_sha256": _sha(108), "configuration_sha256": _sha(109), }, "power_plan": { "required_held_out_participants": 30, "required_held_out_sessions": 50, "required_paired_axis_observations": 150, "alpha": 0.05, "target_power": 0.8, "minimally_detectable_gain": 0.01, }, "participants": participants, "labeler_attestations": [ {"labeler_key": "labeler-001", "attestation_sha256": _sha(110)}, {"labeler_key": "labeler-002", "attestation_sha256": _sha(111)}, ], "reliability": { "labeler_keys": ["labeler-001", "labeler-002"], "reported_icc": 1.0, "reported_categorical_kappa": 1.0, "report_sha256": _sha(112), }, "observations": observations, } def public_soak() -> dict[str, object]: metrics = [ { "turn_number": index + 1, "interim_transcript_frames": 1, "speech_final_transcript_frames": 1, "first_interim_latency_ms": 100.0, "speech_final_latency_ms": 300.0, } for index in range(10) ] return { "schema_version": "vignette.g7-public-voice-soak.v4", "mode": "public_soak", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "target_host": "api.example.test", "requested_duration_seconds": 3000.0, "elapsed_seconds": 3000.1, "microphone_device_enumerated": True, "physical_capture_confirmed": True, "physical_microphone_used": True, "raw_audio_retained": False, "public_wss_used": True, "authenticated_public_wss_ready": True, "real_database_session_binding_required": True, "provider_overrides_used": False, "ready_provider_metadata_validated": True, "cookie_present": True, "cookie_value_logged": False, "session_id_present": True, "cloudflare_ray_present": True, "unauthenticated_handshake_accepted": True, "tls_version": "TLSv1.3", "unauthenticated_close_code": 1008, "close_code": 1000, "expected_stt_provider": "local_whisper", "ready_stt_provider": "local_whisper", "expected_stt_model": "large-v3", "ready_stt_model": "large-v3", "expected_tts_provider": "melotts", "ready_tts_provider": "melotts", "expected_tts_model": "melotts-korean", "ready_tts_model": "melotts-korean", "turns_attempted": 10, "turns_succeeded": 10, "turn_transcript_metrics": metrics, "interim_transcript_frames": 10, "speech_final_transcript_frames": 10, "reply_frames": 10, "tts_end_frames": 10, "tts_binary_bytes": 1000, "captured_pcm_bytes": 1000, "blockers": [], "failure_type": None, } def runtime() -> dict[str, object]: snapshot = { "schema_version": "vignette.voice-runtime.v1", "scope": "single_api_worker", "privacy_boundary": "metadata_only_no_audio_transcript_or_session_ids", "limits": {"uvicorn_ws_max_queue": 4}, "process": {}, "counters": { "provider_fallback_total": 0, "websocket_error_total": 0, "audio_overflow_rejections_total": 0, }, } return { "schema_version": "vignette.g7-runtime-sampling.v1", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "target_host": "api.example.test", "worker_started_at_utc": "2026-08-07T00:00:00Z", "privacy_boundary": "metadata_only_no_audio_transcript_session_or_cookie_values", "cookie_present": True, "cookie_value_logged": False, "requested_samples": 31, "samples_completed": 31, "interval_seconds": 100.0, "samples": [ {"sequence": index + 1, "snapshot": copy.deepcopy(snapshot)} for index in range(31) ], "high_water": { "process_peak_rss_bytes": 1, "process_threads_max": 1, "websocket_high_water": 1, "streaming_provider_session_high_water": 1, "route_audio_buffer_high_water_bytes": 1, "streaming_event_queue_high_water_items": 1, }, } def topology() -> dict[str, object]: container_summary = { "cgroup_memory_peak_bytes_max": 1, "cgroup_cpu_usage_usec_max": 1, "cgroup_pids_current_max": 1, "proc_vm_hwm_bytes_max": 1, "proc_threads_max": 1, "proc_fd_count_max": 1, } return { "schema_version": "vignette.g7-topology-evidence.v1", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "scope": { "metadata_only": True, "raw_command_output_retained": False, "socket_endpoints_retained": False, "request_payloads_retained": False, "audio_retained": False, "transcripts_retained": False, "cloudflare_edge": { "internal_queue_measured": False, "evidence_boundary": "separate_external_artifact_required", }, }, "requested": { "public_host": "api.example.test", "samples": 31, "interval_seconds": 100.0, }, "samples_completed": 31, "targets": { role: { "container_id": character * 64, "image_digest": "sha256:" + character * 64, "started_at": "2026-08-07T00:00:00Z", } for role, character in (("api", "a"), ("caddy", "c")) }, "summary": { "containers": { "api": copy.deepcopy(container_summary), "caddy": copy.deepcopy(container_summary), }, "host_tcp": {"connections_max": 1}, }, "failure_type": None, } def windows_topology() -> dict[str, object]: process_metrics = { "api": { "rss_bytes": 200_000, "peak_rss_bytes": 240_000, "cpu_time_seconds": 2.0, "cpu_percent": 3.0, "handles": 40, "threads": 8, }, "cloudflared": { "rss_bytes": 100_000, "peak_rss_bytes": 120_000, "cpu_time_seconds": 1.0, "cpu_percent": 1.5, "handles": 20, "threads": 4, }, } process_tcp = { "api": {"connections": 2, "established": 1, "listeners": 1}, "cloudflared": {"connections": 4, "established": 4, "listeners": 0}, } samples = [ { "sequence": index + 1, "observed_at_utc": "2026-08-07T00:30:00Z", "processes": copy.deepcopy(process_metrics), "process_tcp": copy.deepcopy(process_tcp), "host_tcp": {"connections": 20}, "api_listener": { "port": 8001, "owned_listener_count": 1, "conflicting_listener_count": 0, }, } for index in range(31) ] return { "schema_version": "vignette.g7-topology-evidence.v1", "topology_mode": "windows_host", "status": "passed", "started_at_utc": "2026-08-07T00:00:00Z", "ended_at_utc": "2026-08-07T01:00:00Z", "scope": { "metadata_only": True, "raw_command_output_retained": False, "socket_endpoints_retained": False, "request_payloads_retained": False, "audio_retained": False, "transcripts_retained": False, "topology_boundary": "windows_host_api_and_cloudflared_processes", "configured_listener_port_retained": True, "cloudflare_edge": { "internal_queue_measured": False, "evidence_boundary": "separate_external_artifact_required", }, }, "requested": { "public_host": "api.example.test", "repo_root": r"D:\workspace\vignette", "git_sha": "5" * 40, "source_pin": { "git_tree_sha": "8" * 40, "script_sha256": { "runner": "9" * 64, "collector": "a" * 64, "checker": "b" * 64, }, "runtime_dependencies": {"psutil": "6.1.1"}, }, "samples": 31, "interval_seconds": 100.0, "api_listen_port": 8001, "roles": { "api": { "pid": 301, "expected_executable_name": "python.exe", "expected_executable_sha256": "3" * 64, "expected_cwd": r"D:\workspace\vignette\apps\api", }, "cloudflared": { "pid": 302, "expected_executable_name": "cloudflared.exe", "expected_executable_sha256": "4" * 64, "expected_cwd": r"D:\workspace\vignette", }, }, }, "source_provenance": { "detached_head": True, "tracked_clean": True, "git_sha": "5" * 40, "git_tree_sha": "8" * 40, "script_sha256": { "runner": "9" * 64, "collector": "a" * 64, "checker": "b" * 64, }, "runtime_dependencies": {"psutil": "6.1.1"}, }, "samples_completed": 31, "targets": { "api": { "role": "api", "pid": 301, "started_at": "2026-08-06T00:00:00Z", "executable_name": "python.exe", "executable_sha256": "3" * 64, "command_line_sha256": "6" * 64, "git_sha": "5" * 40, "cwd": r"D:\workspace\vignette\apps\api", }, "cloudflared": { "role": "cloudflared", "pid": 302, "started_at": "2026-08-06T00:00:00Z", "executable_name": "cloudflared.exe", "executable_sha256": "4" * 64, "command_line_sha256": "7" * 64, "git_sha": "5" * 40, "cwd": r"D:\workspace\vignette", }, }, "samples": samples, "summary": { "processes": { role: { **{f"{field}_max": value for field, value in metrics.items()}, **{ f"tcp_{field}_max": value for field, value in process_tcp[role].items() }, } for role, metrics in process_metrics.items() }, "api_listener": { "port": 8001, "owned_listener_count_min": 1, "conflicting_listener_count_max": 0, }, "host_tcp": {"connections_max": 20}, }, "failure_type": None, } class G7ExternalProofTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.checker = load_checker() def test_complete_external_proof_passes(self) -> None: errors: list[str] = [] voice = public_soak() runtime_payload = runtime() topology_payload = topology() self.checker.validate_public_soak(voice, errors) self.checker.validate_runtime(runtime_payload, errors) self.checker.validate_topology(topology_payload, errors) gain = self.checker.validate_human_gain(human_pack(), errors) self.checker.validate_binding(voice, runtime_payload, topology_payload, errors) self.assertEqual([], errors) self.assertTrue(gain["passed"]) self.assertGreaterEqual(gain["held_out_participants"], 30) self.assertGreaterEqual(gain["recomputed_categorical_kappa"], 0.70) def test_human_pack_cannot_bypass_the_required_categorical_kappa(self) -> None: errors: list[str] = [] payload = human_pack() reliability = payload["reliability"] assert isinstance(reliability, dict) reliability.pop("reported_categorical_kappa") observations = payload["observations"] assert isinstance(observations, list) for observation in observations: labels = observation["labels"] for label in labels: label.pop("category") gain = self.checker.validate_human_gain(payload, errors) self.assertEqual({}, gain) self.assertIn("human_gain:invalid:ValidationError", errors) def test_windows_host_topology_passes_without_weakening_compose(self) -> None: errors: list[str] = [] payload = windows_topology() compose_payload = topology() compose_payload["topology_mode"] = "linux_compose" self.checker.validate_topology(payload, errors) self.checker.validate_topology(compose_payload, errors) self.checker.validate_binding(public_soak(), runtime(), payload, errors) self.assertEqual([], errors) def test_windows_host_identity_listener_and_summary_fail_closed(self) -> None: cases = ( ( lambda payload: payload["targets"]["api"].__setitem__("pid", 999), "topology:api_pid_pin", ), ( lambda payload: payload["samples"][0]["api_listener"].__setitem__( "owned_listener_count", 0 ), "topology:listener_owner_sample", ), ( lambda payload: payload["summary"]["processes"]["api"].__setitem__( "rss_bytes_max", 1 ), "topology:api_rss_bytes_max", ), ) for mutate, failure in cases: with self.subTest(failure=failure): payload = windows_topology() mutate(payload) errors: list[str] = [] self.checker.validate_topology(payload, errors) self.assertIn(failure, errors) def test_windows_source_and_toolchain_provenance_fail_closed(self) -> None: cases = ( ( lambda payload: payload["source_provenance"].__setitem__( "detached_head", False ), "topology:windows_detached_head", ), ( lambda payload: payload["source_provenance"].__setitem__( "tracked_clean", False ), "topology:windows_tracked_clean", ), ( lambda payload: payload["source_provenance"].__setitem__( "git_tree_sha", "c" * 40 ), "topology:windows_source_git_tree_sha", ), ( lambda payload: payload["source_provenance"]["script_sha256"].__setitem__( "runner", "c" * 64 ), "topology:windows_source_script_sha256", ), ( lambda payload: payload["source_provenance"][ "runtime_dependencies" ].__setitem__("psutil", "6.1.0"), "topology:windows_source_psutil_version", ), ) for mutate, failure in cases: with self.subTest(failure=failure): payload = windows_topology() mutate(payload) errors: list[str] = [] self.checker.validate_topology(payload, errors) self.assertIn(failure, errors) unpinned_version = windows_topology() unpinned_version["requested"]["source_pin"]["runtime_dependencies"][ "psutil" ] = "6.1.0" unpinned_version["source_provenance"]["runtime_dependencies"][ "psutil" ] = "6.1.0" errors = [] self.checker.validate_topology(unpinned_version, errors) self.assertIn("topology:windows_psutil_version_pin", errors) def test_decided_local_stack_is_accepted(self) -> None: """2026-08-08 소유자 결정: 노트북 faster-whisper STT + MeloTTS TTS (둘 다 MIT).""" errors: list[str] = [] voice = public_soak() self.assertEqual(voice["expected_stt_provider"], "local_whisper") self.assertEqual(voice["expected_tts_provider"], "melotts") self.checker.validate_public_soak(voice, errors) self.assertEqual([], errors) def test_deepgram_remains_an_operated_alternative(self) -> None: errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "deepgram" voice["ready_stt_provider"] = "deepgram" voice["expected_stt_model"] = "nova-3" voice["ready_stt_model"] = "nova-3" self.checker.validate_public_soak(voice, errors) self.assertEqual([], errors) def test_batch_openai_stt_cannot_satisfy_the_streaming_gate(self) -> None: errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "openai" voice["ready_stt_provider"] = "openai" self.checker.validate_public_soak(voice, errors) self.assertIn("voice_soak:stt_provider_not_operated", errors) def test_unoperated_providers_fail_closed(self) -> None: for field, value, code in ( ("expected_stt_provider", "some-other-vendor", "stt_provider_not_operated"), ("expected_tts_provider", "some-other-vendor", "tts_provider_not_operated"), ): with self.subTest(field=field): errors: list[str] = [] voice = public_soak() voice[field] = value voice[field.replace("expected", "ready")] = value self.checker.validate_public_soak(voice, errors) self.assertIn(f"voice_soak:{code}", errors) def test_declared_provider_must_match_the_runtime_provider(self) -> None: """허용 목록을 넓혀도 선언/실제 불일치는 계속 막아야 한다.""" errors: list[str] = [] voice = public_soak() voice["expected_stt_provider"] = "local_whisper" voice["ready_stt_provider"] = "deepgram" self.checker.validate_public_soak(voice, errors) self.assertIn("voice_soak:ready_stt_provider_mismatch", errors) def test_preflight_synthetic_and_nonoverlapping_evidence_fail(self) -> None: errors: list[str] = [] voice = public_soak() voice["mode"] = "preflight" voice["physical_microphone_used"] = False runtime_payload = runtime() runtime_payload["started_at_utc"] = "2026-08-08T00:00:00Z" runtime_payload["ended_at_utc"] = "2026-08-08T01:00:00Z" self.checker.validate_public_soak(voice, errors) self.checker.validate_binding(voice, runtime_payload, topology(), errors) self.assertIn("voice_soak:mode", errors) self.assertIn("voice_soak:physical_microphone_used", errors) self.assertIn("binding:no_concurrent_overlap", errors) def test_concurrent_window_requires_full_3000_second_intersection(self) -> None: exact_errors: list[str] = [] exact_voice = public_soak() exact_voice["ended_at_utc"] = "2026-08-07T00:50:00Z" exact_runtime = runtime() exact_runtime["ended_at_utc"] = "2026-08-07T00:50:00Z" exact_topology = topology() exact_topology["ended_at_utc"] = "2026-08-07T00:50:00Z" self.checker.validate_binding( exact_voice, exact_runtime, exact_topology, exact_errors, ) self.assertEqual([], exact_errors) short_errors: list[str] = [] short_voice = public_soak() short_voice["ended_at_utc"] = "2026-08-07T00:50:00Z" short_runtime = runtime() short_runtime["started_at_utc"] = "2026-08-07T00:00:01Z" short_runtime["ended_at_utc"] = "2026-08-07T00:50:00Z" short_topology = topology() short_topology["ended_at_utc"] = "2026-08-07T00:50:00Z" self.checker.validate_binding( short_voice, short_runtime, short_topology, short_errors, ) self.assertIn( "binding:concurrent_overlap_below_3000_seconds", short_errors, ) def test_runtime_fallback_and_short_topology_fail(self) -> None: errors: list[str] = [] runtime_payload = runtime() runtime_payload["samples"][-1]["snapshot"]["counters"][ "provider_fallback_total" ] = 1 topology_payload = topology() topology_payload["requested"]["samples"] = 2 topology_payload["samples_completed"] = 2 self.checker.validate_runtime(runtime_payload, errors) self.checker.validate_topology(topology_payload, errors) self.assertIn("runtime:provider_fallback_total", errors) self.assertIn("topology:coverage", errors) if __name__ == "__main__": unittest.main()