G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
126
apps/api/app/test_alliance_calibration.py
Normal file
126
apps/api/app/test_alliance_calibration.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""Alliance calibration/drift report contract tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from .contracts.measurement import BenchmarkCase
|
||||
from .services.alliance_calibration import (
|
||||
AllianceCalibrationPrediction,
|
||||
AllianceCalibrationSnapshot,
|
||||
compare_alliance_snapshots,
|
||||
evaluate_alliance_snapshot,
|
||||
gold_reference_snapshot,
|
||||
render_alliance_comparison_markdown,
|
||||
)
|
||||
|
||||
|
||||
BENCHMARK_PATH = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "data"
|
||||
/ "outcome_alliance_benchmark_g0.v1.json"
|
||||
)
|
||||
|
||||
|
||||
def _cases() -> tuple[BenchmarkCase, ...]:
|
||||
payload = json.loads(BENCHMARK_PATH.read_text(encoding="utf-8"))
|
||||
return tuple(BenchmarkCase.model_validate(item) for item in payload["cases"])
|
||||
|
||||
|
||||
def _prediction(
|
||||
case_id: str,
|
||||
perspective: str,
|
||||
dimension: str,
|
||||
value: float,
|
||||
evidence: tuple[int, ...],
|
||||
) -> AllianceCalibrationPrediction:
|
||||
return AllianceCalibrationPrediction.model_validate(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"perspective": perspective,
|
||||
"dimension": dimension,
|
||||
"value": value,
|
||||
"confidence": 0.8,
|
||||
"evidence_turn_indices": evidence,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class AllianceCalibrationTest(unittest.TestCase):
|
||||
def test_gold_reference_is_explicit_and_complete(self) -> None:
|
||||
reference = gold_reference_snapshot(_cases())
|
||||
report = evaluate_alliance_snapshot(reference, _cases())
|
||||
self.assertEqual(reference.provider, "human_gold")
|
||||
self.assertEqual(len(reference.predictions), 9)
|
||||
self.assertEqual(report["direction_accuracy"], 1.0)
|
||||
self.assertEqual(report["evidence_recall"], 1.0)
|
||||
|
||||
def test_failure_predictions_cannot_hide_behind_neutral_scores(self) -> None:
|
||||
with self.assertRaises(ValidationError):
|
||||
AllianceCalibrationPrediction(
|
||||
case_id="oas-g0-001",
|
||||
perspective="independent_observer",
|
||||
dimension="goal",
|
||||
value=0.5,
|
||||
status="error",
|
||||
error_code="provider_timeout",
|
||||
)
|
||||
|
||||
def test_snapshot_scores_each_expected_axis_without_total(self) -> None:
|
||||
snapshot = AllianceCalibrationSnapshot(
|
||||
run_id="candidate",
|
||||
provider="fake",
|
||||
model="fake-v1",
|
||||
prompt_bundle_version="1.0.0",
|
||||
predictions=(
|
||||
_prediction("oas-g0-001", "independent_observer", "goal", 0.2, (0, 1, 2)),
|
||||
_prediction("oas-g0-002", "independent_observer", "task", 0.2, (0, 1, 2)),
|
||||
_prediction("oas-g0-003", "independent_observer", "bond", 0.2, (1, 2)),
|
||||
_prediction("oas-g0-005", "client_agent_report", "bond", 0.2, (1,)),
|
||||
_prediction("oas-g0-006", "client_agent_report", "task", 0.8, (2, 3)),
|
||||
_prediction("oas-g0-007", "client_agent_report", "bond", 0.2, (2,)),
|
||||
_prediction("oas-g0-008", "client_agent_report", "bond", 0.8, (0, 1)),
|
||||
_prediction("oas-g0-008", "independent_observer", "goal", 0.2, (2, 3)),
|
||||
_prediction("oas-g0-008", "independent_observer", "task", 0.2, (2, 3)),
|
||||
),
|
||||
)
|
||||
report = evaluate_alliance_snapshot(snapshot, _cases())
|
||||
self.assertEqual(report["expected_count"], 9)
|
||||
self.assertEqual(report["ready_count"], 9)
|
||||
self.assertEqual(report["direction_accuracy"], 1.0)
|
||||
self.assertEqual(report["evidence_recall"], 1.0)
|
||||
self.assertNotIn("total", report["dimension_means"])
|
||||
|
||||
def test_comparison_reports_version_drift_and_errors(self) -> None:
|
||||
baseline = AllianceCalibrationSnapshot(
|
||||
run_id="baseline",
|
||||
provider="reference",
|
||||
model="gold-v1",
|
||||
prompt_bundle_version="1.0.0",
|
||||
predictions=(
|
||||
_prediction("oas-g0-001", "independent_observer", "goal", 0.2, (0, 1, 2)),
|
||||
),
|
||||
)
|
||||
candidate = AllianceCalibrationSnapshot(
|
||||
run_id="candidate",
|
||||
provider="gateway",
|
||||
model="candidate-v2",
|
||||
prompt_bundle_version="1.1.0",
|
||||
predictions=(
|
||||
_prediction("oas-g0-001", "independent_observer", "goal", 0.35, (1, 2)),
|
||||
),
|
||||
)
|
||||
report = compare_alliance_snapshots(baseline, candidate, _cases())
|
||||
self.assertAlmostEqual(report["drift"]["mean_absolute_score_delta"], 0.15)
|
||||
markdown = render_alliance_comparison_markdown(report)
|
||||
self.assertIn("candidate-v2", markdown)
|
||||
self.assertIn("교육용 합성 장면", markdown)
|
||||
self.assertIn("oas-g0-001", markdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue