from __future__ import annotations import unittest from pathlib import Path from pydantic import ValidationError from .contracts.supervision_research import ( EvaluationVersionBatch, LearnerAttentionSignal, Phase3OutcomeEvidenceManifest, TeacherAiDisagreement, ) from .services.supervision_research import ( build_attention_queue, build_calibration_dataset, build_phase3_outcome_manifest, compare_evaluation_versions, evaluate_supervision_research_benchmark, load_supervision_research_benchmark, ) BENCHMARK_PATH = ( Path(__file__).resolve().parent / "data" / "supervision_research_benchmark_g6.v1.json" ) def _all_keys(value: object) -> set[str]: if isinstance(value, dict): children = set().union(*(_all_keys(item) for item in value.values())) return set(value) | children if isinstance(value, (list, tuple)): return set().union(*(_all_keys(item) for item in value)) if value else set() return set() class AttentionQueueTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH) def test_queue_follows_explicit_signal_priority(self) -> None: queue = build_attention_queue(self.pack.attention_signals) self.assertEqual( tuple(item.learner_ref for item in queue), self.pack.expected_queue_order ) self.assertEqual( tuple(item.queue_position for item in queue), tuple(range(1, 7)) ) self.assertEqual(queue[0].primary_signal, "safety_boundary") self.assertEqual(queue[1].primary_signal, "deterioration") self.assertEqual(queue[2].primary_signal, "unresolved_rupture") def test_every_queue_reason_drills_to_ledger_in_three_clicks(self) -> None: queue = build_attention_queue(self.pack.attention_signals) for item in queue: self.assertGreaterEqual(len(item.drilldown_routes), 1) self.assertLessEqual(len(item.drilldown_routes), 3) self.assertTrue(all(reason.evidence for reason in item.reasons)) def test_resolved_and_insufficient_signals_do_not_enter_queue(self) -> None: base = self.pack.attention_signals[-1] resolved_payload = base.model_dump(mode="json") resolved_payload.update( signal_id="oas-g6-signal-resolved", learner_ref="learner-resolved", state="resolved", counterevidence=["teacher_confirmed_resolution"], ) insufficient_payload = base.model_dump(mode="json") insufficient_payload.update( signal_id="oas-g6-signal-insufficient", learner_ref="learner-insufficient", state="insufficient_evidence", uncertainty=1.0, evidence=[], ) signals = ( *self.pack.attention_signals, LearnerAttentionSignal.model_validate(resolved_payload), LearnerAttentionSignal.model_validate(insufficient_payload), ) refs = {item.learner_ref for item in build_attention_queue(signals)} self.assertNotIn("learner-resolved", refs) self.assertNotIn("learner-insufficient", refs) def test_duplicate_signal_id_is_rejected(self) -> None: signal = self.pack.attention_signals[0] with self.assertRaisesRegex(ValueError, "duplicate attention signal id"): build_attention_queue((signal, signal)) def test_classified_signal_without_ledger_evidence_is_rejected(self) -> None: payload = self.pack.attention_signals[0].model_dump(mode="json") payload["evidence"] = [] with self.assertRaisesRegex(ValidationError, "requires ledger evidence"): LearnerAttentionSignal.model_validate(payload) class CalibrationDatasetTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH) def test_teacher_ai_disagreement_becomes_metadata_only_dataset_row(self) -> None: rows = build_calibration_dataset(self.pack.disagreements) self.assertEqual(len(rows), 1) row = rows[0] self.assertEqual(len(row.row_id), 64) self.assertFalse(row.raw_transcript_included) self.assertEqual(row.ai_label, "resolved") self.assertEqual(row.teacher_label, "partial") self.assertEqual(len(row.evidence_event_ids), 2) def test_dataset_row_hash_is_deterministic(self) -> None: first = build_calibration_dataset(self.pack.disagreements) second = build_calibration_dataset(self.pack.disagreements) self.assertEqual(first, second) def test_matching_ai_and_teacher_labels_are_not_disagreement_data(self) -> None: payload = self.pack.disagreements[0].model_dump(mode="json") payload["teacher_label"] = payload["ai_label"] with self.assertRaisesRegex(ValidationError, "different labels"): TeacherAiDisagreement.model_validate(payload) def test_duplicate_disagreement_is_rejected(self) -> None: item = self.pack.disagreements[0] with self.assertRaisesRegex(ValueError, "duplicate disagreement id"): build_calibration_dataset((item, item)) class VersionDriftTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH) def test_model_instrument_version_regression_is_flagged(self) -> None: report = compare_evaluation_versions( self.pack.baseline_batch, self.pack.candidate_batch ) self.assertEqual(report.status, "drift_flagged") self.assertEqual(report.baseline_accuracy, 1.0) self.assertAlmostEqual(report.candidate_accuracy or 0.0, 2 / 3) self.assertIn("overall_accuracy_regression", report.alerts) self.assertIn("synthetic_subgroup_regression:synthetic-b", report.alerts) self.assertEqual(set(report.disagreement_case_refs), {"case-b2", "case-b3"}) def test_underpowered_version_comparison_remains_scoreless(self) -> None: baseline_payload = self.pack.baseline_batch.model_dump(mode="json") candidate_payload = self.pack.candidate_batch.model_dump(mode="json") baseline_payload["observations"] = baseline_payload["observations"][:3] candidate_payload["observations"] = candidate_payload["observations"][:3] report = compare_evaluation_versions( EvaluationVersionBatch.model_validate(baseline_payload), EvaluationVersionBatch.model_validate(candidate_payload), ) self.assertEqual(report.status, "insufficient_evidence") self.assertIsNone(report.baseline_accuracy) self.assertIsNone(report.candidate_accuracy) self.assertIsNone(report.accuracy_delta) def test_version_batch_rejects_duplicate_case_competency(self) -> None: payload = self.pack.baseline_batch.model_dump(mode="json") payload["observations"].append(payload["observations"][0]) with self.assertRaisesRegex(ValidationError, "keys must be unique"): EvaluationVersionBatch.model_validate(payload) class Phase3ManifestTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH) def test_manifest_requires_all_four_outcome_domains(self) -> None: manifest = build_phase3_outcome_manifest(self.pack.phase3_artifacts) self.assertEqual( {item.domain for item in manifest.artifacts}, {"alliance", "rupture", "transfer", "calibration"}, ) self.assertTrue( all(not item.clinical_claim_allowed for item in manifest.artifacts) ) def test_manifest_missing_domain_is_rejected(self) -> None: payload = { "schema_version": "vignette.phase3-outcome-evidence-manifest.v1", "artifacts": [ item.model_dump(mode="json") for item in self.pack.phase3_artifacts[:3] ], } with self.assertRaises(ValidationError): Phase3OutcomeEvidenceManifest.model_validate(payload) class SupervisionResearchBenchmarkTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH) def test_benchmark_closes_queue_dataset_drift_and_manifest(self) -> None: report = evaluate_supervision_research_benchmark(self.pack) self.assertTrue(report["queue_order_correct"]) self.assertEqual(report["calibration_dataset_rows"], 1) self.assertEqual(report["raw_transcript_rows"], 0) self.assertEqual(report["drift_status"], "drift_flagged") self.assertTrue(report["drift_status_correct"]) self.assertEqual( report["manifest_domains"], ["alliance", "calibration", "rupture", "transfer"], ) self.assertFalse(report["clinical_claim_allowed"]) def test_operational_outputs_do_not_contain_learner_total_scores(self) -> None: report = evaluate_supervision_research_benchmark(self.pack) self.assertTrue( {"total", "total_score", "overall_score", "learner_rank"}.isdisjoint( _all_keys(report) ) ) if __name__ == "__main__": unittest.main()