G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
169
apps/api/app/test_supervision_research_version_evaluator.py
Normal file
169
apps/api/app/test_supervision_research_version_evaluator.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
from .services import supervision_research_version_evaluator as evaluator
|
||||
|
||||
|
||||
ROW_NAMESPACE = UUID("68f117f9-4b63-4f2d-a3b6-57f558497f45")
|
||||
LEARNER_ID = UUID("00000000-0000-0000-0000-0000000000a1")
|
||||
SESSION_ID = UUID("10000000-0000-0000-0000-0000000000a1")
|
||||
|
||||
|
||||
def _complete_rows() -> list[dict[str, object]]:
|
||||
benchmark = evaluator.load_validated_repository_benchmark()
|
||||
rows: list[dict[str, object]] = []
|
||||
for batch in (
|
||||
benchmark.pack.baseline_batch,
|
||||
benchmark.pack.candidate_batch,
|
||||
):
|
||||
for observation in batch.observations:
|
||||
rows.append(
|
||||
{
|
||||
"measurement_id": uuid5(
|
||||
ROW_NAMESPACE, observation.evidence_event_id
|
||||
),
|
||||
"session_id": SESSION_ID,
|
||||
"learner_id": LEARNER_ID,
|
||||
"cohort_id": "g6-repo-benchmark-v1",
|
||||
"metadata": evaluator.benchmark_anchor_metadata(
|
||||
benchmark, batch, observation
|
||||
),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
class RepositoryBenchmarkValidationTests(unittest.TestCase):
|
||||
def test_repo_pack_preserves_gold_and_version_provenance(self) -> None:
|
||||
benchmark = evaluator.load_validated_repository_benchmark()
|
||||
report = evaluator.build_repository_comparison_input(
|
||||
_complete_rows(), benchmark=benchmark
|
||||
)
|
||||
assert report is not None
|
||||
self.assertEqual(report.cohort_id, "g6-repo-benchmark-v1")
|
||||
self.assertEqual(report.baseline.model, "evaluator-v1")
|
||||
self.assertEqual(report.baseline.prompt_version, "1.0.0")
|
||||
self.assertEqual(report.baseline.instrument_version, "1.0.0")
|
||||
self.assertEqual(report.candidate.model, "evaluator-v2")
|
||||
self.assertEqual(report.candidate.prompt_version, "2.0.0")
|
||||
self.assertEqual(report.candidate.instrument_version, "1.1.0")
|
||||
baseline_gold = {
|
||||
(item.case_ref, item.competency_id): item.gold_label
|
||||
for item in report.baseline.observations
|
||||
}
|
||||
candidate_gold = {
|
||||
(item.case_ref, item.competency_id): item.gold_label
|
||||
for item in report.candidate.observations
|
||||
}
|
||||
self.assertEqual(baseline_gold, candidate_gold)
|
||||
self.assertNotIn("transcript", str(report).lower())
|
||||
|
||||
def test_incomplete_runtime_evidence_skips_without_inventing_candidate(
|
||||
self,
|
||||
) -> None:
|
||||
rows = _complete_rows()[:-1]
|
||||
self.assertIsNone(evaluator.build_repository_comparison_input(rows))
|
||||
|
||||
def test_runtime_anchor_cannot_change_repo_gold_or_model_provenance(self) -> None:
|
||||
rows = copy.deepcopy(_complete_rows())
|
||||
metadata = rows[0]["metadata"]
|
||||
assert isinstance(metadata, dict)
|
||||
metadata["gold_label"] = "invented-gold"
|
||||
with self.assertRaisesRegex(ValueError, "differs from repository gold"):
|
||||
evaluator.build_repository_comparison_input(rows)
|
||||
|
||||
def test_runtime_anchor_forbids_transcript_fields(self) -> None:
|
||||
rows = copy.deepcopy(_complete_rows())
|
||||
metadata = rows[0]["metadata"]
|
||||
assert isinstance(metadata, dict)
|
||||
metadata["raw_transcript"] = "forbidden"
|
||||
with self.assertRaisesRegex(ValueError, "forbidden source text"):
|
||||
evaluator.build_repository_comparison_input(rows)
|
||||
|
||||
|
||||
class RepositoryBenchmarkProducerTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_first_append_and_replay_use_stable_ids(self) -> None:
|
||||
conn = AsyncMock()
|
||||
append = AsyncMock(
|
||||
side_effect=[
|
||||
{
|
||||
"submission_id": UUID(int=1),
|
||||
"drift_report_id": UUID(int=2),
|
||||
"status": "drift_flagged",
|
||||
"matched_count": 6,
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
},
|
||||
{
|
||||
"submission_id": UUID(int=1),
|
||||
"drift_report_id": UUID(int=2),
|
||||
"status": "drift_flagged",
|
||||
"matched_count": 6,
|
||||
"idempotent_replay": True,
|
||||
"clinical_claim_allowed": False,
|
||||
},
|
||||
]
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
evaluator,
|
||||
"_load_repository_benchmark_anchors",
|
||||
AsyncMock(return_value=_complete_rows()),
|
||||
),
|
||||
patch.object(
|
||||
evaluator.supervision_research_store,
|
||||
"append_evaluation_comparison",
|
||||
append,
|
||||
),
|
||||
):
|
||||
first = await evaluator.produce_repository_version_comparison(conn)
|
||||
replay = await evaluator.produce_repository_version_comparison(conn)
|
||||
|
||||
self.assertFalse(first["idempotent_replay"])
|
||||
self.assertTrue(replay["idempotent_replay"])
|
||||
self.assertEqual(first["status"], "drift_flagged")
|
||||
self.assertFalse(first["raw_transcript_included"])
|
||||
self.assertFalse(first["clinical_claim_allowed"])
|
||||
self.assertEqual(append.await_count, 2)
|
||||
first_call, second_call = append.await_args_list
|
||||
for key in (
|
||||
"submission_id",
|
||||
"drift_report_id",
|
||||
"baseline_submission_id",
|
||||
"baseline_batch_record_id",
|
||||
"candidate_submission_id",
|
||||
"candidate_batch_record_id",
|
||||
):
|
||||
self.assertEqual(first_call.kwargs[key], second_call.kwargs[key])
|
||||
self.assertEqual(first_call.kwargs["baseline"].model, "evaluator-v1")
|
||||
self.assertEqual(first_call.kwargs["candidate"].model, "evaluator-v2")
|
||||
|
||||
async def test_missing_approved_anchors_is_safe_skip(self) -> None:
|
||||
conn = AsyncMock()
|
||||
append = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
evaluator,
|
||||
"_load_repository_benchmark_anchors",
|
||||
AsyncMock(return_value=[]),
|
||||
),
|
||||
patch.object(
|
||||
evaluator.supervision_research_store,
|
||||
"append_evaluation_comparison",
|
||||
append,
|
||||
),
|
||||
):
|
||||
result = await evaluator.produce_repository_version_comparison(conn)
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(
|
||||
result["reason"], "repo_approved_synthetic_evidence_incomplete"
|
||||
)
|
||||
append.assert_not_awaited()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue