G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
225
apps/api/app/test_supervision_research.py
Normal file
225
apps/api/app/test_supervision_research.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from .contracts.supervision_research import (
|
||||
EvaluationVersionBatch,
|
||||
LearnerAttentionSignal,
|
||||
Phase3OutcomeEvidenceManifest,
|
||||
TeacherAiDisagreement,
|
||||
)
|
||||
from .services.supervision_research import (
|
||||
build_attention_queue,
|
||||
build_calibration_dataset,
|
||||
build_phase3_outcome_manifest,
|
||||
compare_evaluation_versions,
|
||||
evaluate_supervision_research_benchmark,
|
||||
load_supervision_research_benchmark,
|
||||
)
|
||||
|
||||
|
||||
BENCHMARK_PATH = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "data"
|
||||
/ "supervision_research_benchmark_g6.v1.json"
|
||||
)
|
||||
|
||||
|
||||
def _all_keys(value: object) -> set[str]:
|
||||
if isinstance(value, dict):
|
||||
children = set().union(*(_all_keys(item) for item in value.values()))
|
||||
return set(value) | children
|
||||
if isinstance(value, (list, tuple)):
|
||||
return set().union(*(_all_keys(item) for item in value)) if value else set()
|
||||
return set()
|
||||
|
||||
|
||||
class AttentionQueueTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
||||
|
||||
def test_queue_follows_explicit_signal_priority(self) -> None:
|
||||
queue = build_attention_queue(self.pack.attention_signals)
|
||||
self.assertEqual(
|
||||
tuple(item.learner_ref for item in queue), self.pack.expected_queue_order
|
||||
)
|
||||
self.assertEqual(
|
||||
tuple(item.queue_position for item in queue), tuple(range(1, 7))
|
||||
)
|
||||
self.assertEqual(queue[0].primary_signal, "safety_boundary")
|
||||
self.assertEqual(queue[1].primary_signal, "deterioration")
|
||||
self.assertEqual(queue[2].primary_signal, "unresolved_rupture")
|
||||
|
||||
def test_every_queue_reason_drills_to_ledger_in_three_clicks(self) -> None:
|
||||
queue = build_attention_queue(self.pack.attention_signals)
|
||||
for item in queue:
|
||||
self.assertGreaterEqual(len(item.drilldown_routes), 1)
|
||||
self.assertLessEqual(len(item.drilldown_routes), 3)
|
||||
self.assertTrue(all(reason.evidence for reason in item.reasons))
|
||||
|
||||
def test_resolved_and_insufficient_signals_do_not_enter_queue(self) -> None:
|
||||
base = self.pack.attention_signals[-1]
|
||||
resolved_payload = base.model_dump(mode="json")
|
||||
resolved_payload.update(
|
||||
signal_id="oas-g6-signal-resolved",
|
||||
learner_ref="learner-resolved",
|
||||
state="resolved",
|
||||
counterevidence=["teacher_confirmed_resolution"],
|
||||
)
|
||||
insufficient_payload = base.model_dump(mode="json")
|
||||
insufficient_payload.update(
|
||||
signal_id="oas-g6-signal-insufficient",
|
||||
learner_ref="learner-insufficient",
|
||||
state="insufficient_evidence",
|
||||
uncertainty=1.0,
|
||||
evidence=[],
|
||||
)
|
||||
signals = (
|
||||
*self.pack.attention_signals,
|
||||
LearnerAttentionSignal.model_validate(resolved_payload),
|
||||
LearnerAttentionSignal.model_validate(insufficient_payload),
|
||||
)
|
||||
refs = {item.learner_ref for item in build_attention_queue(signals)}
|
||||
self.assertNotIn("learner-resolved", refs)
|
||||
self.assertNotIn("learner-insufficient", refs)
|
||||
|
||||
def test_duplicate_signal_id_is_rejected(self) -> None:
|
||||
signal = self.pack.attention_signals[0]
|
||||
with self.assertRaisesRegex(ValueError, "duplicate attention signal id"):
|
||||
build_attention_queue((signal, signal))
|
||||
|
||||
def test_classified_signal_without_ledger_evidence_is_rejected(self) -> None:
|
||||
payload = self.pack.attention_signals[0].model_dump(mode="json")
|
||||
payload["evidence"] = []
|
||||
with self.assertRaisesRegex(ValidationError, "requires ledger evidence"):
|
||||
LearnerAttentionSignal.model_validate(payload)
|
||||
|
||||
|
||||
class CalibrationDatasetTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
||||
|
||||
def test_teacher_ai_disagreement_becomes_metadata_only_dataset_row(self) -> None:
|
||||
rows = build_calibration_dataset(self.pack.disagreements)
|
||||
self.assertEqual(len(rows), 1)
|
||||
row = rows[0]
|
||||
self.assertEqual(len(row.row_id), 64)
|
||||
self.assertFalse(row.raw_transcript_included)
|
||||
self.assertEqual(row.ai_label, "resolved")
|
||||
self.assertEqual(row.teacher_label, "partial")
|
||||
self.assertEqual(len(row.evidence_event_ids), 2)
|
||||
|
||||
def test_dataset_row_hash_is_deterministic(self) -> None:
|
||||
first = build_calibration_dataset(self.pack.disagreements)
|
||||
second = build_calibration_dataset(self.pack.disagreements)
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_matching_ai_and_teacher_labels_are_not_disagreement_data(self) -> None:
|
||||
payload = self.pack.disagreements[0].model_dump(mode="json")
|
||||
payload["teacher_label"] = payload["ai_label"]
|
||||
with self.assertRaisesRegex(ValidationError, "different labels"):
|
||||
TeacherAiDisagreement.model_validate(payload)
|
||||
|
||||
def test_duplicate_disagreement_is_rejected(self) -> None:
|
||||
item = self.pack.disagreements[0]
|
||||
with self.assertRaisesRegex(ValueError, "duplicate disagreement id"):
|
||||
build_calibration_dataset((item, item))
|
||||
|
||||
|
||||
class VersionDriftTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
||||
|
||||
def test_model_instrument_version_regression_is_flagged(self) -> None:
|
||||
report = compare_evaluation_versions(
|
||||
self.pack.baseline_batch, self.pack.candidate_batch
|
||||
)
|
||||
self.assertEqual(report.status, "drift_flagged")
|
||||
self.assertEqual(report.baseline_accuracy, 1.0)
|
||||
self.assertAlmostEqual(report.candidate_accuracy or 0.0, 2 / 3)
|
||||
self.assertIn("overall_accuracy_regression", report.alerts)
|
||||
self.assertIn("synthetic_subgroup_regression:synthetic-b", report.alerts)
|
||||
self.assertEqual(set(report.disagreement_case_refs), {"case-b2", "case-b3"})
|
||||
|
||||
def test_underpowered_version_comparison_remains_scoreless(self) -> None:
|
||||
baseline_payload = self.pack.baseline_batch.model_dump(mode="json")
|
||||
candidate_payload = self.pack.candidate_batch.model_dump(mode="json")
|
||||
baseline_payload["observations"] = baseline_payload["observations"][:3]
|
||||
candidate_payload["observations"] = candidate_payload["observations"][:3]
|
||||
report = compare_evaluation_versions(
|
||||
EvaluationVersionBatch.model_validate(baseline_payload),
|
||||
EvaluationVersionBatch.model_validate(candidate_payload),
|
||||
)
|
||||
self.assertEqual(report.status, "insufficient_evidence")
|
||||
self.assertIsNone(report.baseline_accuracy)
|
||||
self.assertIsNone(report.candidate_accuracy)
|
||||
self.assertIsNone(report.accuracy_delta)
|
||||
|
||||
def test_version_batch_rejects_duplicate_case_competency(self) -> None:
|
||||
payload = self.pack.baseline_batch.model_dump(mode="json")
|
||||
payload["observations"].append(payload["observations"][0])
|
||||
with self.assertRaisesRegex(ValidationError, "keys must be unique"):
|
||||
EvaluationVersionBatch.model_validate(payload)
|
||||
|
||||
|
||||
class Phase3ManifestTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
||||
|
||||
def test_manifest_requires_all_four_outcome_domains(self) -> None:
|
||||
manifest = build_phase3_outcome_manifest(self.pack.phase3_artifacts)
|
||||
self.assertEqual(
|
||||
{item.domain for item in manifest.artifacts},
|
||||
{"alliance", "rupture", "transfer", "calibration"},
|
||||
)
|
||||
self.assertTrue(
|
||||
all(not item.clinical_claim_allowed for item in manifest.artifacts)
|
||||
)
|
||||
|
||||
def test_manifest_missing_domain_is_rejected(self) -> None:
|
||||
payload = {
|
||||
"schema_version": "vignette.phase3-outcome-evidence-manifest.v1",
|
||||
"artifacts": [
|
||||
item.model_dump(mode="json") for item in self.pack.phase3_artifacts[:3]
|
||||
],
|
||||
}
|
||||
with self.assertRaises(ValidationError):
|
||||
Phase3OutcomeEvidenceManifest.model_validate(payload)
|
||||
|
||||
|
||||
class SupervisionResearchBenchmarkTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
||||
|
||||
def test_benchmark_closes_queue_dataset_drift_and_manifest(self) -> None:
|
||||
report = evaluate_supervision_research_benchmark(self.pack)
|
||||
self.assertTrue(report["queue_order_correct"])
|
||||
self.assertEqual(report["calibration_dataset_rows"], 1)
|
||||
self.assertEqual(report["raw_transcript_rows"], 0)
|
||||
self.assertEqual(report["drift_status"], "drift_flagged")
|
||||
self.assertTrue(report["drift_status_correct"])
|
||||
self.assertEqual(
|
||||
report["manifest_domains"],
|
||||
["alliance", "calibration", "rupture", "transfer"],
|
||||
)
|
||||
self.assertFalse(report["clinical_claim_allowed"])
|
||||
|
||||
def test_operational_outputs_do_not_contain_learner_total_scores(self) -> None:
|
||||
report = evaluate_supervision_research_benchmark(self.pack)
|
||||
self.assertTrue(
|
||||
{"total", "total_score", "overall_score", "learner_rank"}.isdisjoint(
|
||||
_all_keys(report)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue