G0~G8 성과·동맹 측정 OS 작업 일괄 고정

8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본
폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을
git 이력으로 고정하는 것이 목적이다.

- contracts/routes/services: measurement, outcome_trajectory, rupture_repair,
  deliberate_practice, calibration_transfer, supervision_research,
  multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트
- infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행)
- apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가
- docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG
- scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트

engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan 2026-08-08 01:30:53 +09:00
parent 93dd8f82d7
commit 16e791e044
390 changed files with 243188 additions and 499 deletions

View file

@ -0,0 +1,126 @@
"""Alliance calibration/drift report contract tests."""
from __future__ import annotations
import json
import unittest
from pathlib import Path
from pydantic import ValidationError
from .contracts.measurement import BenchmarkCase
from .services.alliance_calibration import (
AllianceCalibrationPrediction,
AllianceCalibrationSnapshot,
compare_alliance_snapshots,
evaluate_alliance_snapshot,
gold_reference_snapshot,
render_alliance_comparison_markdown,
)
BENCHMARK_PATH = (
Path(__file__).resolve().parent
/ "data"
/ "outcome_alliance_benchmark_g0.v1.json"
)
def _cases() -> tuple[BenchmarkCase, ...]:
payload = json.loads(BENCHMARK_PATH.read_text(encoding="utf-8"))
return tuple(BenchmarkCase.model_validate(item) for item in payload["cases"])
def _prediction(
case_id: str,
perspective: str,
dimension: str,
value: float,
evidence: tuple[int, ...],
) -> AllianceCalibrationPrediction:
return AllianceCalibrationPrediction.model_validate(
{
"case_id": case_id,
"perspective": perspective,
"dimension": dimension,
"value": value,
"confidence": 0.8,
"evidence_turn_indices": evidence,
}
)
class AllianceCalibrationTest(unittest.TestCase):
def test_gold_reference_is_explicit_and_complete(self) -> None:
reference = gold_reference_snapshot(_cases())
report = evaluate_alliance_snapshot(reference, _cases())
self.assertEqual(reference.provider, "human_gold")
self.assertEqual(len(reference.predictions), 9)
self.assertEqual(report["direction_accuracy"], 1.0)
self.assertEqual(report["evidence_recall"], 1.0)
def test_failure_predictions_cannot_hide_behind_neutral_scores(self) -> None:
with self.assertRaises(ValidationError):
AllianceCalibrationPrediction(
case_id="oas-g0-001",
perspective="independent_observer",
dimension="goal",
value=0.5,
status="error",
error_code="provider_timeout",
)
def test_snapshot_scores_each_expected_axis_without_total(self) -> None:
snapshot = AllianceCalibrationSnapshot(
run_id="candidate",
provider="fake",
model="fake-v1",
prompt_bundle_version="1.0.0",
predictions=(
_prediction("oas-g0-001", "independent_observer", "goal", 0.2, (0, 1, 2)),
_prediction("oas-g0-002", "independent_observer", "task", 0.2, (0, 1, 2)),
_prediction("oas-g0-003", "independent_observer", "bond", 0.2, (1, 2)),
_prediction("oas-g0-005", "client_agent_report", "bond", 0.2, (1,)),
_prediction("oas-g0-006", "client_agent_report", "task", 0.8, (2, 3)),
_prediction("oas-g0-007", "client_agent_report", "bond", 0.2, (2,)),
_prediction("oas-g0-008", "client_agent_report", "bond", 0.8, (0, 1)),
_prediction("oas-g0-008", "independent_observer", "goal", 0.2, (2, 3)),
_prediction("oas-g0-008", "independent_observer", "task", 0.2, (2, 3)),
),
)
report = evaluate_alliance_snapshot(snapshot, _cases())
self.assertEqual(report["expected_count"], 9)
self.assertEqual(report["ready_count"], 9)
self.assertEqual(report["direction_accuracy"], 1.0)
self.assertEqual(report["evidence_recall"], 1.0)
self.assertNotIn("total", report["dimension_means"])
def test_comparison_reports_version_drift_and_errors(self) -> None:
baseline = AllianceCalibrationSnapshot(
run_id="baseline",
provider="reference",
model="gold-v1",
prompt_bundle_version="1.0.0",
predictions=(
_prediction("oas-g0-001", "independent_observer", "goal", 0.2, (0, 1, 2)),
),
)
candidate = AllianceCalibrationSnapshot(
run_id="candidate",
provider="gateway",
model="candidate-v2",
prompt_bundle_version="1.1.0",
predictions=(
_prediction("oas-g0-001", "independent_observer", "goal", 0.35, (1, 2)),
),
)
report = compare_alliance_snapshots(baseline, candidate, _cases())
self.assertAlmostEqual(report["drift"]["mean_absolute_score_delta"], 0.15)
markdown = render_alliance_comparison_markdown(report)
self.assertIn("candidate-v2", markdown)
self.assertIn("교육용 합성 장면", markdown)
self.assertIn("oas-g0-001", markdown)
if __name__ == "__main__":
unittest.main()