yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

G0~G8 성과·동맹 측정 OS 작업 일괄 고정

Browse source 
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본
폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을
git 이력으로 고정하는 것이 목적이다.

- contracts/routes/services: measurement, outcome_trajectory, rupture_repair,
  deliberate_practice, calibration_transfer, supervision_research,
  multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트
- infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행)
- apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가
- docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG
- scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트

engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan 2026-08-08 01:30:53 +09:00
parent 93dd8f82d7
commit 16e791e044
390 changed files with 243188 additions and 499 deletions
Download patch file Download diff file Expand all files Collapse all files

246
apps/api/app/test_g7_voice_gain_evidence.py Normal file
View file

@ -0,0 +1,246 @@
from __future__ import annotations
import copy
import unittest
from pydantic import ValidationError
from .contracts.g7_external_evidence import G7HumanVoiceGainEvidencePack
from .services.g7_voice_gain_evidence import (
VoiceGainEvidenceThresholds,
evaluate_human_voice_gain,
)
def _sha(character: str) -> str:
return character * 64
def _valid_payload() -> dict[str, object]:
participants = [
{
"participant_key": "calibration-001",
"split": "calibration",
"consent_receipt_sha256": _sha("1"),
},
{
"participant_key": "held-out-001",
"split": "held_out",
"consent_receipt_sha256": _sha("2"),
},
{
"participant_key": "held-out-002",
"split": "held_out",
"consent_receipt_sha256": _sha("3"),
},
]
observations: list[dict[str, object]] = []
targets = {
"goal": (0.20, "low"),
"task": (0.50, "medium"),
"bond": (0.80, "high"),
}
observation_number = 0
for participant_number in (1, 2):
participant = f"held-out-{participant_number:03d}"
session = f"session-{participant_number:03d}"
for axis, (target, category) in targets.items():
observation_number += 1
observations.append(
{
"observation_id": (
f"g7-human-observation-{observation_number:03d}"
),
"participant_key": participant,
"session_key": session,
"axis": axis,
"text_only_status": "observed",
"text_only_score": target + 0.20,
"voice_enabled_status": "observed",
"voice_enabled_score": target + 0.05,
"labels": [
{
"labeler_key": "labeler-001",
"score": target,
"category": category,
},
{
"labeler_key": "labeler-002",
"score": target,
"category": category,
},
],
}
)
return {
"provenance": {
"protocol_sha256": _sha("a"),
"consent_protocol_sha256": _sha("b"),
"dataset_manifest_sha256": _sha("c"),
"split_manifest_sha256": _sha("d"),
"labeling_protocol_sha256": _sha("e"),
"analysis_plan_sha256": _sha("f"),
"registered_at": "2026-08-01T00:00:00Z",
"held_out_labels_opened_at": "2026-08-02T00:00:00Z",
},
"text_only_model": {
"role": "text_only_baseline",
"provider": "example-provider",
"model_id": "alliance-text",
"model_version": "v1",
"artifact_sha256": _sha("4"),
"configuration_sha256": _sha("5"),
},
"voice_enabled_model": {
"role": "voice_enabled_candidate",
"provider": "example-provider",
"model_id": "alliance-voice",
"model_version": "v2",
"artifact_sha256": _sha("6"),
"configuration_sha256": _sha("7"),
},
"power_plan": {
"required_held_out_participants": 2,
"required_held_out_sessions": 2,
"required_paired_axis_observations": 6,
"alpha": 0.05,
"target_power": 0.8,
"minimally_detectable_gain": 0.01,
},
"participants": participants,
"labeler_attestations": [
{
"labeler_key": "labeler-001",
"attestation_sha256": _sha("8"),
},
{
"labeler_key": "labeler-002",
"attestation_sha256": _sha("9"),
},
],
"reliability": {
"labeler_keys": ["labeler-001", "labeler-002"],
"reported_icc": 1.0,
"reported_categorical_kappa": 1.0,
"report_sha256": _sha("0"),
},
"observations": observations,
}
def _test_thresholds() -> VoiceGainEvidenceThresholds:
return VoiceGainEvidenceThresholds.for_test(
min_held_out_participants=2,
min_held_out_sessions=2,
min_paired_axis_observations=6,
bootstrap_samples=500,
)
class G7HumanVoiceGainEvidenceTests(unittest.TestCase):
def test_valid_human_holdout_recomputes_paired_gain_and_cluster_ci(self) -> None:
pack = G7HumanVoiceGainEvidencePack.model_validate(_valid_payload())
result = evaluate_human_voice_gain(pack, thresholds=_test_thresholds())
self.assertTrue(result.passed, result.failure_reasons)
self.assertAlmostEqual(result.text_only_one_minus_mae, 0.8)
self.assertAlmostEqual(result.voice_enabled_one_minus_mae, 0.95)
self.assertAlmostEqual(result.paired_gain, 0.15)
self.assertGreater(result.ci_lower, 0.0)
self.assertEqual(result.recomputed_icc, 1.0)
self.assertEqual(result.recomputed_categorical_kappa, 1.0)
self.assertFalse(result.clinical_claim_allowed)
def test_production_defaults_reject_small_fixture(self) -> None:
pack = G7HumanVoiceGainEvidencePack.model_validate(_valid_payload())
result = evaluate_human_voice_gain(pack)
self.assertFalse(result.passed)
self.assertIn("production_participant_floor", result.failure_reasons)
self.assertIn("production_session_floor", result.failure_reasons)
self.assertIn("production_observation_floor", result.failure_reasons)
self.assertIn("power_plan_participant_floor", result.failure_reasons)
self.assertIn("power_plan_session_floor", result.failure_reasons)
self.assertIn("power_plan_observation_floor", result.failure_reasons)
self.assertEqual(result.bootstrap_samples, 10_000)
def test_missing_or_error_prediction_is_max_error_intention_to_evaluate(self) -> None:
payload = _valid_payload()
observations = payload["observations"]
assert isinstance(observations, list)
observations[0]["voice_enabled_status"] = "error"
observations[0]["voice_enabled_score"] = None
pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
result = evaluate_human_voice_gain(pack, thresholds=_test_thresholds())
self.assertEqual(result.intention_to_evaluate_imputations, 1)
expected_voice_accuracy = 1.0 - ((1.0 + (0.05 * 5)) / 6)
self.assertAlmostEqual(result.voice_enabled_one_minus_mae, expected_voice_accuracy)
self.assertFalse(result.passed)
def test_duplicate_or_calibration_observation_is_rejected(self) -> None:
duplicate = _valid_payload()
duplicate_rows = duplicate["observations"]
assert isinstance(duplicate_rows, list)
duplicate_rows.append(copy.deepcopy(duplicate_rows[0]))
with self.assertRaisesRegex(ValidationError, "observation ids must be unique"):
G7HumanVoiceGainEvidencePack.model_validate(duplicate)
leakage = _valid_payload()
leakage_rows = leakage["observations"]
assert isinstance(leakage_rows, list)
leakage_rows[0]["participant_key"] = "calibration-001"
with self.assertRaisesRegex(ValidationError, "held-out participants"):
G7HumanVoiceGainEvidencePack.model_validate(leakage)
def test_blind_independent_attestation_and_full_axis_coverage_are_required(self) -> None:
unblinded = _valid_payload()
attestations = unblinded["labeler_attestations"]
assert isinstance(attestations, list)
attestations[0]["blinded_to_model_condition"] = False
with self.assertRaises(ValidationError):
G7HumanVoiceGainEvidencePack.model_validate(unblinded)
incomplete = _valid_payload()
rows = incomplete["observations"]
assert isinstance(rows, list)
rows.pop()
with self.assertRaisesRegex(ValidationError, "goal, task, and bond"):
G7HumanVoiceGainEvidencePack.model_validate(incomplete)
def test_raw_material_and_synthetic_pack_cannot_enter_contract(self) -> None:
raw = _valid_payload()
rows = raw["observations"]
assert isinstance(rows, list)
rows[0]["raw_audio"] = "forbidden"
with self.assertRaises(ValidationError):
G7HumanVoiceGainEvidencePack.model_validate(raw)
synthetic = _valid_payload()
synthetic["synthetic_pack"] = True
with self.assertRaises(ValidationError):
G7HumanVoiceGainEvidencePack.model_validate(synthetic)
def test_reported_reliability_is_checked_against_rows(self) -> None:
payload = _valid_payload()
reliability = payload["reliability"]
assert isinstance(reliability, dict)
reliability["reported_icc"] = 0.8
pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
result = evaluate_human_voice_gain(pack, thresholds=_test_thresholds())
self.assertFalse(result.passed)
self.assertIn("reported_icc_matches_rows", result.failure_reasons)
def test_custom_thresholds_require_explicit_test_factory(self) -> None:
pack = G7HumanVoiceGainEvidencePack.model_validate(_valid_payload())
with self.assertRaisesRegex(ValueError, "test-only"):
evaluate_human_voice_gain(pack, thresholds=VoiceGainEvidenceThresholds())
if __name__ == "__main__":
unittest.main()