282 lines
11 KiB
Python
282 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from pydantic import ValidationError
|
|
|
|
from .contracts.supervision_research import (
|
|
AttentionQueueItem,
|
|
EvaluationVersionBatch,
|
|
LedgerEvidencePointer,
|
|
LearnerAttentionSignal,
|
|
Phase3OutcomeEvidenceManifest,
|
|
TeacherAiDisagreement,
|
|
)
|
|
from .services.supervision_research import (
|
|
build_attention_queue,
|
|
build_calibration_dataset,
|
|
build_phase3_outcome_manifest,
|
|
compare_evaluation_versions,
|
|
evaluate_supervision_research_benchmark,
|
|
load_supervision_research_benchmark,
|
|
)
|
|
|
|
|
|
BENCHMARK_PATH = (
|
|
Path(__file__).resolve().parent
|
|
/ "data"
|
|
/ "supervision_research_benchmark_g6.v1.json"
|
|
)
|
|
|
|
|
|
def _all_keys(value: object) -> set[str]:
|
|
if isinstance(value, dict):
|
|
children = set().union(*(_all_keys(item) for item in value.values()))
|
|
return set(value) | children
|
|
if isinstance(value, (list, tuple)):
|
|
return set().union(*(_all_keys(item) for item in value)) if value else set()
|
|
return set()
|
|
|
|
|
|
class AttentionQueueTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
|
|
def test_queue_follows_explicit_signal_priority(self) -> None:
|
|
queue = build_attention_queue(self.pack.attention_signals)
|
|
self.assertEqual(
|
|
tuple(item.learner_ref for item in queue), self.pack.expected_queue_order
|
|
)
|
|
self.assertEqual(
|
|
tuple(item.queue_position for item in queue), tuple(range(1, 7))
|
|
)
|
|
self.assertEqual(queue[0].primary_signal, "safety_boundary")
|
|
self.assertEqual(queue[1].primary_signal, "deterioration")
|
|
self.assertEqual(queue[2].primary_signal, "unresolved_rupture")
|
|
|
|
def test_every_queue_reason_drills_to_ledger_in_three_clicks(self) -> None:
|
|
queue = build_attention_queue(self.pack.attention_signals)
|
|
for item in queue:
|
|
self.assertGreaterEqual(len(item.drilldown_routes), 1)
|
|
self.assertLessEqual(len(item.drilldown_routes), 3)
|
|
self.assertTrue(all(reason.evidence for reason in item.reasons))
|
|
|
|
def test_resolved_and_insufficient_signals_do_not_enter_queue(self) -> None:
|
|
base = self.pack.attention_signals[-1]
|
|
resolved_payload = base.model_dump(mode="json")
|
|
resolved_payload.update(
|
|
signal_id="oas-g6-signal-resolved",
|
|
learner_ref="learner-resolved",
|
|
state="resolved",
|
|
counterevidence=["teacher_confirmed_resolution"],
|
|
)
|
|
insufficient_payload = base.model_dump(mode="json")
|
|
insufficient_payload.update(
|
|
signal_id="oas-g6-signal-insufficient",
|
|
learner_ref="learner-insufficient",
|
|
state="insufficient_evidence",
|
|
uncertainty=1.0,
|
|
evidence=[],
|
|
)
|
|
signals = (
|
|
*self.pack.attention_signals,
|
|
LearnerAttentionSignal.model_validate(resolved_payload),
|
|
LearnerAttentionSignal.model_validate(insufficient_payload),
|
|
)
|
|
refs = {item.learner_ref for item in build_attention_queue(signals)}
|
|
self.assertNotIn("learner-resolved", refs)
|
|
self.assertNotIn("learner-insufficient", refs)
|
|
|
|
def test_duplicate_signal_id_is_rejected(self) -> None:
|
|
signal = self.pack.attention_signals[0]
|
|
with self.assertRaisesRegex(ValueError, "duplicate attention signal id"):
|
|
build_attention_queue((signal, signal))
|
|
|
|
def test_classified_signal_without_ledger_evidence_is_rejected(self) -> None:
|
|
payload = self.pack.attention_signals[0].model_dump(mode="json")
|
|
payload["evidence"] = []
|
|
with self.assertRaisesRegex(ValidationError, "requires ledger evidence"):
|
|
LearnerAttentionSignal.model_validate(payload)
|
|
|
|
|
|
class SupervisionRouteContractTests(unittest.TestCase):
|
|
VALID_ROUTES = (
|
|
"/teach",
|
|
"/teach/session/uuid/review",
|
|
"/teacher/dashboard?cohort=abc",
|
|
)
|
|
INVALID_ROUTES = (
|
|
"//evil.test/path",
|
|
"https://evil.test/path",
|
|
"/\\evil.test/path",
|
|
"/teach\nnext",
|
|
)
|
|
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
cls.queue_item_payload = build_attention_queue(cls.pack.attention_signals)[
|
|
0
|
|
].model_dump(mode="json")
|
|
|
|
def test_ledger_evidence_routes_allow_app_relative_paths(self) -> None:
|
|
for route in self.VALID_ROUTES:
|
|
with self.subTest(route=route):
|
|
pointer = LedgerEvidencePointer(
|
|
ledger="measurement_event",
|
|
event_id="event-1",
|
|
route_hint=route,
|
|
)
|
|
self.assertEqual(pointer.route_hint, route)
|
|
|
|
def test_ledger_evidence_routes_reject_external_and_unsupported_paths(self) -> None:
|
|
for route in self.INVALID_ROUTES:
|
|
with self.subTest(route=route):
|
|
with self.assertRaises(ValidationError):
|
|
LedgerEvidencePointer(
|
|
ledger="measurement_event",
|
|
event_id="event-1",
|
|
route_hint=route,
|
|
)
|
|
|
|
def test_attention_queue_routes_allow_app_relative_paths(self) -> None:
|
|
payload = dict(self.queue_item_payload)
|
|
payload["drilldown_routes"] = list(self.VALID_ROUTES)
|
|
item = AttentionQueueItem.model_validate(payload)
|
|
self.assertEqual(item.drilldown_routes, self.VALID_ROUTES)
|
|
|
|
def test_attention_queue_routes_reject_external_and_unsupported_paths(self) -> None:
|
|
for route in self.INVALID_ROUTES:
|
|
with self.subTest(route=route):
|
|
payload = dict(self.queue_item_payload)
|
|
payload["drilldown_routes"] = [route]
|
|
with self.assertRaises(ValidationError):
|
|
AttentionQueueItem.model_validate(payload)
|
|
|
|
|
|
class CalibrationDatasetTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
|
|
def test_teacher_ai_disagreement_becomes_metadata_only_dataset_row(self) -> None:
|
|
rows = build_calibration_dataset(self.pack.disagreements)
|
|
self.assertEqual(len(rows), 1)
|
|
row = rows[0]
|
|
self.assertEqual(len(row.row_id), 64)
|
|
self.assertFalse(row.raw_transcript_included)
|
|
self.assertEqual(row.ai_label, "resolved")
|
|
self.assertEqual(row.teacher_label, "partial")
|
|
self.assertEqual(len(row.evidence_event_ids), 2)
|
|
|
|
def test_dataset_row_hash_is_deterministic(self) -> None:
|
|
first = build_calibration_dataset(self.pack.disagreements)
|
|
second = build_calibration_dataset(self.pack.disagreements)
|
|
self.assertEqual(first, second)
|
|
|
|
def test_matching_ai_and_teacher_labels_are_not_disagreement_data(self) -> None:
|
|
payload = self.pack.disagreements[0].model_dump(mode="json")
|
|
payload["teacher_label"] = payload["ai_label"]
|
|
with self.assertRaisesRegex(ValidationError, "different labels"):
|
|
TeacherAiDisagreement.model_validate(payload)
|
|
|
|
def test_duplicate_disagreement_is_rejected(self) -> None:
|
|
item = self.pack.disagreements[0]
|
|
with self.assertRaisesRegex(ValueError, "duplicate disagreement id"):
|
|
build_calibration_dataset((item, item))
|
|
|
|
|
|
class VersionDriftTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
|
|
def test_model_instrument_version_regression_is_flagged(self) -> None:
|
|
report = compare_evaluation_versions(
|
|
self.pack.baseline_batch, self.pack.candidate_batch
|
|
)
|
|
self.assertEqual(report.status, "drift_flagged")
|
|
self.assertEqual(report.baseline_accuracy, 1.0)
|
|
self.assertAlmostEqual(report.candidate_accuracy or 0.0, 2 / 3)
|
|
self.assertIn("overall_accuracy_regression", report.alerts)
|
|
self.assertIn("synthetic_subgroup_regression:synthetic-b", report.alerts)
|
|
self.assertEqual(set(report.disagreement_case_refs), {"case-b2", "case-b3"})
|
|
|
|
def test_underpowered_version_comparison_remains_scoreless(self) -> None:
|
|
baseline_payload = self.pack.baseline_batch.model_dump(mode="json")
|
|
candidate_payload = self.pack.candidate_batch.model_dump(mode="json")
|
|
baseline_payload["observations"] = baseline_payload["observations"][:3]
|
|
candidate_payload["observations"] = candidate_payload["observations"][:3]
|
|
report = compare_evaluation_versions(
|
|
EvaluationVersionBatch.model_validate(baseline_payload),
|
|
EvaluationVersionBatch.model_validate(candidate_payload),
|
|
)
|
|
self.assertEqual(report.status, "insufficient_evidence")
|
|
self.assertIsNone(report.baseline_accuracy)
|
|
self.assertIsNone(report.candidate_accuracy)
|
|
self.assertIsNone(report.accuracy_delta)
|
|
|
|
def test_version_batch_rejects_duplicate_case_competency(self) -> None:
|
|
payload = self.pack.baseline_batch.model_dump(mode="json")
|
|
payload["observations"].append(payload["observations"][0])
|
|
with self.assertRaisesRegex(ValidationError, "keys must be unique"):
|
|
EvaluationVersionBatch.model_validate(payload)
|
|
|
|
|
|
class Phase3ManifestTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
|
|
def test_manifest_requires_all_four_outcome_domains(self) -> None:
|
|
manifest = build_phase3_outcome_manifest(self.pack.phase3_artifacts)
|
|
self.assertEqual(
|
|
{item.domain for item in manifest.artifacts},
|
|
{"alliance", "rupture", "transfer", "calibration"},
|
|
)
|
|
self.assertTrue(
|
|
all(not item.clinical_claim_allowed for item in manifest.artifacts)
|
|
)
|
|
|
|
def test_manifest_missing_domain_is_rejected(self) -> None:
|
|
payload = {
|
|
"schema_version": "vignette.phase3-outcome-evidence-manifest.v1",
|
|
"artifacts": [
|
|
item.model_dump(mode="json") for item in self.pack.phase3_artifacts[:3]
|
|
],
|
|
}
|
|
with self.assertRaises(ValidationError):
|
|
Phase3OutcomeEvidenceManifest.model_validate(payload)
|
|
|
|
|
|
class SupervisionResearchBenchmarkTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.pack = load_supervision_research_benchmark(BENCHMARK_PATH)
|
|
|
|
def test_benchmark_closes_queue_dataset_drift_and_manifest(self) -> None:
|
|
report = evaluate_supervision_research_benchmark(self.pack)
|
|
self.assertTrue(report["queue_order_correct"])
|
|
self.assertEqual(report["calibration_dataset_rows"], 1)
|
|
self.assertEqual(report["raw_transcript_rows"], 0)
|
|
self.assertEqual(report["drift_status"], "drift_flagged")
|
|
self.assertTrue(report["drift_status_correct"])
|
|
self.assertEqual(
|
|
report["manifest_domains"],
|
|
["alliance", "calibration", "rupture", "transfer"],
|
|
)
|
|
self.assertFalse(report["clinical_claim_allowed"])
|
|
|
|
def test_operational_outputs_do_not_contain_learner_total_scores(self) -> None:
|
|
report = evaluate_supervision_research_benchmark(self.pack)
|
|
self.assertTrue(
|
|
{"total", "total_score", "overall_score", "learner_rank"}.isdisjoint(
|
|
_all_keys(report)
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|