vignette/apps/api/app/test_supervision_research_version_evaluator.py

from __future__ import annotations

import copy
import unittest
from unittest.mock import AsyncMock, patch
from uuid import UUID, uuid5

from .services import supervision_research_version_evaluator as evaluator


ROW_NAMESPACE = UUID("68f117f9-4b63-4f2d-a3b6-57f558497f45")
LEARNER_ID = UUID("00000000-0000-0000-0000-0000000000a1")
SESSION_ID = UUID("10000000-0000-0000-0000-0000000000a1")


def _complete_rows() -> list[dict[str, object]]:
    benchmark = evaluator.load_validated_repository_benchmark()
    rows: list[dict[str, object]] = []
    for batch in (
        benchmark.pack.baseline_batch,
        benchmark.pack.candidate_batch,
    ):
        for observation in batch.observations:
            rows.append(
                {
                    "measurement_id": uuid5(
                        ROW_NAMESPACE, observation.evidence_event_id
                    ),
                    "session_id": SESSION_ID,
                    "learner_id": LEARNER_ID,
                    "cohort_id": "g6-repo-benchmark-v1",
                    "metadata": evaluator.benchmark_anchor_metadata(
                        benchmark, batch, observation
                    ),
                }
            )
    return rows


class RepositoryBenchmarkValidationTests(unittest.TestCase):
    def test_repo_pack_preserves_gold_and_version_provenance(self) -> None:
        benchmark = evaluator.load_validated_repository_benchmark()
        report = evaluator.build_repository_comparison_input(
            _complete_rows(), benchmark=benchmark
        )
        assert report is not None
        self.assertEqual(report.cohort_id, "g6-repo-benchmark-v1")
        self.assertEqual(report.baseline.model, "evaluator-v1")
        self.assertEqual(report.baseline.prompt_version, "1.0.0")
        self.assertEqual(report.baseline.instrument_version, "1.0.0")
        self.assertEqual(report.candidate.model, "evaluator-v2")
        self.assertEqual(report.candidate.prompt_version, "2.0.0")
        self.assertEqual(report.candidate.instrument_version, "1.1.0")
        baseline_gold = {
            (item.case_ref, item.competency_id): item.gold_label
            for item in report.baseline.observations
        }
        candidate_gold = {
            (item.case_ref, item.competency_id): item.gold_label
            for item in report.candidate.observations
        }
        self.assertEqual(baseline_gold, candidate_gold)
        self.assertNotIn("transcript", str(report).lower())

    def test_incomplete_runtime_evidence_skips_without_inventing_candidate(
        self,
    ) -> None:
        rows = _complete_rows()[:-1]
        self.assertIsNone(evaluator.build_repository_comparison_input(rows))

    def test_runtime_anchor_cannot_change_repo_gold_or_model_provenance(self) -> None:
        rows = copy.deepcopy(_complete_rows())
        metadata = rows[0]["metadata"]
        assert isinstance(metadata, dict)
        metadata["gold_label"] = "invented-gold"
        with self.assertRaisesRegex(ValueError, "differs from repository gold"):
            evaluator.build_repository_comparison_input(rows)

    def test_runtime_anchor_forbids_transcript_fields(self) -> None:
        rows = copy.deepcopy(_complete_rows())
        metadata = rows[0]["metadata"]
        assert isinstance(metadata, dict)
        metadata["raw_transcript"] = "forbidden"
        with self.assertRaisesRegex(ValueError, "forbidden source text"):
            evaluator.build_repository_comparison_input(rows)


class RepositoryBenchmarkProducerTests(unittest.IsolatedAsyncioTestCase):
    async def test_first_append_and_replay_use_stable_ids(self) -> None:
        conn = AsyncMock()
        append = AsyncMock(
            side_effect=[
                {
                    "submission_id": UUID(int=1),
                    "drift_report_id": UUID(int=2),
                    "status": "drift_flagged",
                    "matched_count": 6,
                    "idempotent_replay": False,
                    "clinical_claim_allowed": False,
                },
                {
                    "submission_id": UUID(int=1),
                    "drift_report_id": UUID(int=2),
                    "status": "drift_flagged",
                    "matched_count": 6,
                    "idempotent_replay": True,
                    "clinical_claim_allowed": False,
                },
            ]
        )
        with (
            patch.object(
                evaluator,
                "_load_repository_benchmark_anchors",
                AsyncMock(return_value=_complete_rows()),
            ),
            patch.object(
                evaluator.supervision_research_store,
                "append_evaluation_comparison",
                append,
            ),
        ):
            first = await evaluator.produce_repository_version_comparison(conn)
            replay = await evaluator.produce_repository_version_comparison(conn)

        self.assertFalse(first["idempotent_replay"])
        self.assertTrue(replay["idempotent_replay"])
        self.assertEqual(first["status"], "drift_flagged")
        self.assertFalse(first["raw_transcript_included"])
        self.assertFalse(first["clinical_claim_allowed"])
        self.assertEqual(append.await_count, 2)
        first_call, second_call = append.await_args_list
        for key in (
            "submission_id",
            "drift_report_id",
            "baseline_submission_id",
            "baseline_batch_record_id",
            "candidate_submission_id",
            "candidate_batch_record_id",
        ):
            self.assertEqual(first_call.kwargs[key], second_call.kwargs[key])
        self.assertEqual(first_call.kwargs["baseline"].model, "evaluator-v1")
        self.assertEqual(first_call.kwargs["candidate"].model, "evaluator-v2")

    async def test_missing_approved_anchors_is_safe_skip(self) -> None:
        conn = AsyncMock()
        append = AsyncMock()
        with (
            patch.object(
                evaluator,
                "_load_repository_benchmark_anchors",
                AsyncMock(return_value=[]),
            ),
            patch.object(
                evaluator.supervision_research_store,
                "append_evaluation_comparison",
                append,
            ),
        ):
            result = await evaluator.produce_repository_version_comparison(conn)
        self.assertEqual(result["status"], "skipped")
        self.assertEqual(
            result["reason"], "repo_approved_synthetic_evidence_incomplete"
        )
        append.assert_not_awaited()


if __name__ == "__main__":
    unittest.main()