2624d49984
Higgs Audio v3 는 연구/비상업 라이선스라 config.py 가 environment != dev 에서 차단하고 있었다. 그 가드를 푸는 건 법적 판단이라 코드로 결정할 수 없어서, 상업 사용이 허용된 설치형을 다시 찾아 MeloTTS Korean 으로 바꿨다. 결과적으로 가드를 건드릴 필요 자체가 사라졌다 — Higgs 가드는 그대로 두고 provider 만 melotts 로 두면 운영에서도 동작한다. 검토 결과: - MeloTTS MIT 한국어 지원 -> 채택. CPU 실시간, 사전학습 다화자 - Kokoro-82M Apache2.0 한국어 없음 -> 탈락. 공식 VOICES.md 언어 목록에 부재 - Piper GPL -> 탈락 - XTTS-v2 / Fish Speech 비상업 -> 탈락. Higgs 와 같은 문제 사전학습 다화자 모델이라 실존 인물 reference 를 쓰지 않는다. Higgs 경로가 P1 프리셋 한정이던 이유가 없으므로 모든 페르소나 프리셋에 적용된다. 구현: - scripts/melotts-server.py loopback HTTP 사이드카(/health, POST /tts -> WAV) - voice_tts_provider=melotts 경로와 VIGNETTE_MELOTTS_TTS_* 설정 - scripts/start-melotts.ps1 런처(설치 순서 안내 포함) 실측: - CPU 정상 상태 RTF 0.27~0.28(실시간 3.6배). 첫 실행 13.25 는 모델 다운로드 - POST /tts 200, WAV 350,566 bytes, 3.61s, 헤더 provider/model/license - 빈 텍스트 422, 미지 경로 404 로 fail-closed - 왕복 검증: MeloTTS 합성음을 로컬 faster-whisper 가 완전 일치 전사 "그렇게 느끼셨군요. 조금 더 이야기해 주실 수 있을까요?" (word timestamp 8개) 설치 함정 3가지를 decisions/local-voice-stack.md 에 남겼다. librosa 0.9.1 의 pkg_resources(setuptools<81), MeloTTS 가 언어와 무관하게 임포트하는 일본어 unidic 사전, Windows 한국어 g2p 의 eunjeon. G7 게이트의 TTS 허용목록에 melotts 를 추가했다. 선언/실제 불일치 차단과 배치 STT 배제는 그대로다. 검증: API 914 passed, 사이드카 melotts 16/16 + whisper 37/37, SSOT FAIL 0, ruff clean.
374 lines
14 KiB
Python
374 lines
14 KiB
Python
"""Fail-closed tests for the complete G7 external proof checker."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib.util
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
CHECKER_PATH = Path(__file__).with_name("check-g7-external-proof.py")
|
|
|
|
|
|
def load_checker():
|
|
spec = importlib.util.spec_from_file_location("g7_external_proof", CHECKER_PATH)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError("G7 external proof checker could not be loaded")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _sha(number: int) -> str:
|
|
return f"{number:064x}"
|
|
|
|
|
|
def human_pack() -> dict[str, object]:
|
|
participants = [
|
|
{
|
|
"participant_key": "calibration-000",
|
|
"split": "calibration",
|
|
"consent_receipt_sha256": _sha(1),
|
|
}
|
|
]
|
|
participants.extend(
|
|
{
|
|
"participant_key": f"held-{index:03d}",
|
|
"split": "held_out",
|
|
"consent_receipt_sha256": _sha(index + 2),
|
|
}
|
|
for index in range(30)
|
|
)
|
|
observations = []
|
|
axes = ("goal", "task", "bond")
|
|
for session_index in range(50):
|
|
participant = f"held-{session_index % 30:03d}"
|
|
for axis_index, axis in enumerate(axes):
|
|
target = 0.2 + axis_index * 0.2 + (session_index % 5) * 0.01
|
|
observation_number = session_index * 3 + axis_index + 1
|
|
observations.append(
|
|
{
|
|
"observation_id": f"g7-human-observation-{observation_number:03d}",
|
|
"participant_key": participant,
|
|
"session_key": f"session-{session_index:03d}",
|
|
"axis": axis,
|
|
"text_only_status": "observed",
|
|
"text_only_score": target + 0.10,
|
|
"voice_enabled_status": "observed",
|
|
"voice_enabled_score": target,
|
|
"labels": [
|
|
{"labeler_key": "labeler-001", "score": target},
|
|
{"labeler_key": "labeler-002", "score": target},
|
|
],
|
|
}
|
|
)
|
|
return {
|
|
"provenance": {
|
|
"protocol_sha256": _sha(100),
|
|
"consent_protocol_sha256": _sha(101),
|
|
"dataset_manifest_sha256": _sha(102),
|
|
"split_manifest_sha256": _sha(103),
|
|
"labeling_protocol_sha256": _sha(104),
|
|
"analysis_plan_sha256": _sha(105),
|
|
"registered_at": "2026-08-01T00:00:00Z",
|
|
"held_out_labels_opened_at": "2026-08-02T00:00:00Z",
|
|
},
|
|
"text_only_model": {
|
|
"role": "text_only_baseline",
|
|
"provider": "provider",
|
|
"model_id": "baseline",
|
|
"model_version": "v1",
|
|
"artifact_sha256": _sha(106),
|
|
"configuration_sha256": _sha(107),
|
|
},
|
|
"voice_enabled_model": {
|
|
"role": "voice_enabled_candidate",
|
|
"provider": "provider",
|
|
"model_id": "voice",
|
|
"model_version": "v1",
|
|
"artifact_sha256": _sha(108),
|
|
"configuration_sha256": _sha(109),
|
|
},
|
|
"power_plan": {
|
|
"required_held_out_participants": 30,
|
|
"required_held_out_sessions": 50,
|
|
"required_paired_axis_observations": 150,
|
|
"alpha": 0.05,
|
|
"target_power": 0.8,
|
|
"minimally_detectable_gain": 0.01,
|
|
},
|
|
"participants": participants,
|
|
"labeler_attestations": [
|
|
{"labeler_key": "labeler-001", "attestation_sha256": _sha(110)},
|
|
{"labeler_key": "labeler-002", "attestation_sha256": _sha(111)},
|
|
],
|
|
"reliability": {
|
|
"labeler_keys": ["labeler-001", "labeler-002"],
|
|
"reported_icc": 1.0,
|
|
"report_sha256": _sha(112),
|
|
},
|
|
"observations": observations,
|
|
}
|
|
|
|
|
|
def public_soak() -> dict[str, object]:
|
|
metrics = [
|
|
{
|
|
"turn_number": index + 1,
|
|
"interim_transcript_frames": 1,
|
|
"speech_final_transcript_frames": 1,
|
|
"first_interim_latency_ms": 100.0,
|
|
"speech_final_latency_ms": 300.0,
|
|
}
|
|
for index in range(10)
|
|
]
|
|
return {
|
|
"schema_version": "vignette.g7-public-voice-soak.v4",
|
|
"mode": "public_soak",
|
|
"status": "passed",
|
|
"started_at_utc": "2026-08-07T00:00:00Z",
|
|
"ended_at_utc": "2026-08-07T01:00:00Z",
|
|
"target_host": "api.example.test",
|
|
"requested_duration_seconds": 3000.0,
|
|
"elapsed_seconds": 3000.1,
|
|
"microphone_device_enumerated": True,
|
|
"physical_capture_confirmed": True,
|
|
"physical_microphone_used": True,
|
|
"raw_audio_retained": False,
|
|
"public_wss_used": True,
|
|
"authenticated_public_wss_ready": True,
|
|
"real_database_session_binding_required": True,
|
|
"provider_overrides_used": False,
|
|
"ready_provider_metadata_validated": True,
|
|
"cookie_present": True,
|
|
"cookie_value_logged": False,
|
|
"session_id_present": True,
|
|
"cloudflare_ray_present": True,
|
|
"unauthenticated_handshake_accepted": True,
|
|
"tls_version": "TLSv1.3",
|
|
"unauthenticated_close_code": 1008,
|
|
"close_code": 1000,
|
|
"expected_stt_provider": "local_whisper",
|
|
"ready_stt_provider": "local_whisper",
|
|
"expected_stt_model": "large-v3",
|
|
"ready_stt_model": "large-v3",
|
|
"expected_tts_provider": "melotts",
|
|
"ready_tts_provider": "melotts",
|
|
"expected_tts_model": "melotts-korean",
|
|
"ready_tts_model": "melotts-korean",
|
|
"turns_attempted": 10,
|
|
"turns_succeeded": 10,
|
|
"turn_transcript_metrics": metrics,
|
|
"interim_transcript_frames": 10,
|
|
"speech_final_transcript_frames": 10,
|
|
"reply_frames": 10,
|
|
"tts_end_frames": 10,
|
|
"tts_binary_bytes": 1000,
|
|
"captured_pcm_bytes": 1000,
|
|
"blockers": [],
|
|
"failure_type": None,
|
|
}
|
|
|
|
|
|
def runtime() -> dict[str, object]:
|
|
snapshot = {
|
|
"schema_version": "vignette.voice-runtime.v1",
|
|
"scope": "single_api_worker",
|
|
"privacy_boundary": "metadata_only_no_audio_transcript_or_session_ids",
|
|
"limits": {"uvicorn_ws_max_queue": 4},
|
|
"process": {},
|
|
"counters": {
|
|
"provider_fallback_total": 0,
|
|
"websocket_error_total": 0,
|
|
"audio_overflow_rejections_total": 0,
|
|
},
|
|
}
|
|
return {
|
|
"schema_version": "vignette.g7-runtime-sampling.v1",
|
|
"status": "passed",
|
|
"started_at_utc": "2026-08-07T00:00:00Z",
|
|
"ended_at_utc": "2026-08-07T01:00:00Z",
|
|
"target_host": "api.example.test",
|
|
"worker_started_at_utc": "2026-08-07T00:00:00Z",
|
|
"privacy_boundary": "metadata_only_no_audio_transcript_session_or_cookie_values",
|
|
"cookie_present": True,
|
|
"cookie_value_logged": False,
|
|
"requested_samples": 31,
|
|
"samples_completed": 31,
|
|
"interval_seconds": 100.0,
|
|
"samples": [
|
|
{"sequence": index + 1, "snapshot": copy.deepcopy(snapshot)}
|
|
for index in range(31)
|
|
],
|
|
"high_water": {
|
|
"process_peak_rss_bytes": 1,
|
|
"process_threads_max": 1,
|
|
"websocket_high_water": 1,
|
|
"streaming_provider_session_high_water": 1,
|
|
"route_audio_buffer_high_water_bytes": 1,
|
|
"streaming_event_queue_high_water_items": 1,
|
|
},
|
|
}
|
|
|
|
|
|
def topology() -> dict[str, object]:
|
|
container_summary = {
|
|
"cgroup_memory_peak_bytes_max": 1,
|
|
"cgroup_cpu_usage_usec_max": 1,
|
|
"cgroup_pids_current_max": 1,
|
|
"proc_vm_hwm_bytes_max": 1,
|
|
"proc_threads_max": 1,
|
|
"proc_fd_count_max": 1,
|
|
}
|
|
return {
|
|
"schema_version": "vignette.g7-topology-evidence.v1",
|
|
"status": "passed",
|
|
"started_at_utc": "2026-08-07T00:00:00Z",
|
|
"ended_at_utc": "2026-08-07T01:00:00Z",
|
|
"scope": {
|
|
"metadata_only": True,
|
|
"raw_command_output_retained": False,
|
|
"socket_endpoints_retained": False,
|
|
"request_payloads_retained": False,
|
|
"audio_retained": False,
|
|
"transcripts_retained": False,
|
|
"cloudflare_edge": {
|
|
"internal_queue_measured": False,
|
|
"evidence_boundary": "separate_external_artifact_required",
|
|
},
|
|
},
|
|
"requested": {
|
|
"public_host": "api.example.test",
|
|
"samples": 31,
|
|
"interval_seconds": 100.0,
|
|
},
|
|
"samples_completed": 31,
|
|
"targets": {
|
|
role: {
|
|
"container_id": character * 64,
|
|
"image_digest": "sha256:" + character * 64,
|
|
"started_at": "2026-08-07T00:00:00Z",
|
|
}
|
|
for role, character in (("api", "a"), ("caddy", "c"))
|
|
},
|
|
"summary": {
|
|
"containers": {
|
|
"api": copy.deepcopy(container_summary),
|
|
"caddy": copy.deepcopy(container_summary),
|
|
},
|
|
"host_tcp": {"connections_max": 1},
|
|
},
|
|
"failure_type": None,
|
|
}
|
|
|
|
|
|
class G7ExternalProofTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.checker = load_checker()
|
|
|
|
def test_complete_external_proof_passes(self) -> None:
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
runtime_payload = runtime()
|
|
topology_payload = topology()
|
|
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.checker.validate_runtime(runtime_payload, errors)
|
|
self.checker.validate_topology(topology_payload, errors)
|
|
gain = self.checker.validate_human_gain(human_pack(), errors)
|
|
self.checker.validate_binding(voice, runtime_payload, topology_payload, errors)
|
|
|
|
self.assertEqual([], errors)
|
|
self.assertTrue(gain["passed"])
|
|
self.assertGreaterEqual(gain["held_out_participants"], 30)
|
|
|
|
def test_decided_local_stack_is_accepted(self) -> None:
|
|
"""2026-08-08 소유자 결정: 노트북 faster-whisper STT + MeloTTS TTS (둘 다 MIT)."""
|
|
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
self.assertEqual(voice["expected_stt_provider"], "local_whisper")
|
|
self.assertEqual(voice["expected_tts_provider"], "melotts")
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.assertEqual([], errors)
|
|
|
|
def test_deepgram_remains_an_operated_alternative(self) -> None:
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
voice["expected_stt_provider"] = "deepgram"
|
|
voice["ready_stt_provider"] = "deepgram"
|
|
voice["expected_stt_model"] = "nova-3"
|
|
voice["ready_stt_model"] = "nova-3"
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.assertEqual([], errors)
|
|
|
|
def test_batch_openai_stt_cannot_satisfy_the_streaming_gate(self) -> None:
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
voice["expected_stt_provider"] = "openai"
|
|
voice["ready_stt_provider"] = "openai"
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.assertIn("voice_soak:stt_provider_not_operated", errors)
|
|
|
|
def test_unoperated_providers_fail_closed(self) -> None:
|
|
for field, value, code in (
|
|
("expected_stt_provider", "some-other-vendor", "stt_provider_not_operated"),
|
|
("expected_tts_provider", "some-other-vendor", "tts_provider_not_operated"),
|
|
):
|
|
with self.subTest(field=field):
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
voice[field] = value
|
|
voice[field.replace("expected", "ready")] = value
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.assertIn(f"voice_soak:{code}", errors)
|
|
|
|
def test_declared_provider_must_match_the_runtime_provider(self) -> None:
|
|
"""허용 목록을 넓혀도 선언/실제 불일치는 계속 막아야 한다."""
|
|
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
voice["expected_stt_provider"] = "local_whisper"
|
|
voice["ready_stt_provider"] = "deepgram"
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.assertIn("voice_soak:ready_stt_provider_mismatch", errors)
|
|
|
|
def test_preflight_synthetic_and_nonoverlapping_evidence_fail(self) -> None:
|
|
errors: list[str] = []
|
|
voice = public_soak()
|
|
voice["mode"] = "preflight"
|
|
voice["physical_microphone_used"] = False
|
|
runtime_payload = runtime()
|
|
runtime_payload["started_at_utc"] = "2026-08-08T00:00:00Z"
|
|
runtime_payload["ended_at_utc"] = "2026-08-08T01:00:00Z"
|
|
|
|
self.checker.validate_public_soak(voice, errors)
|
|
self.checker.validate_binding(voice, runtime_payload, topology(), errors)
|
|
|
|
self.assertIn("voice_soak:mode", errors)
|
|
self.assertIn("voice_soak:physical_microphone_used", errors)
|
|
self.assertIn("binding:no_concurrent_overlap", errors)
|
|
|
|
def test_runtime_fallback_and_short_topology_fail(self) -> None:
|
|
errors: list[str] = []
|
|
runtime_payload = runtime()
|
|
runtime_payload["samples"][-1]["snapshot"]["counters"][
|
|
"provider_fallback_total"
|
|
] = 1
|
|
topology_payload = topology()
|
|
topology_payload["requested"]["samples"] = 2
|
|
topology_payload["samples_completed"] = 2
|
|
|
|
self.checker.validate_runtime(runtime_payload, errors)
|
|
self.checker.validate_topology(topology_payload, errors)
|
|
|
|
self.assertIn("runtime:provider_fallback_total", errors)
|
|
self.assertIn("topology:coverage", errors)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|