from __future__ import annotations from collections.abc import Mapping from contextlib import ExitStack from unittest import IsolatedAsyncioTestCase, TestCase from unittest.mock import AsyncMock, patch from uuid import UUID, uuid5 from .contracts.supervision_research import LedgerEvidencePointer from .services import supervision_research_store from .services.supervision_research_producer import ( _PRODUCER_NAMESPACE, _canonical_hash, _load_attention_rows, _signal_id, derive_attention_signals, derive_manifest_input, produce_all_active_cohorts_once, produce_supervision_cycle, ) from .services.supervision_research import build_attention_queue LEARNER_A = UUID("00000000-0000-0000-0000-0000000000a1") LEARNER_B = UUID("00000000-0000-0000-0000-0000000000b2") SESSION_A = UUID("10000000-0000-0000-0000-0000000000a1") class SupervisionResearchProducerDerivationTest(TestCase): def test_escalated_safety_event_becomes_metadata_only_priority_signal( self, ) -> None: safety_rows = [ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "turn_id": None, "trigger_type": "real_crisis", "ko_risk_level": 5, "escalated": True, "observed_sequence": 7, # 호출자가 실수로 전체 row를 넘겨도 민감 원천 payload를 # G6 신호에 복제하면 안 된다. "detail": {"matched": ["resident-010-1234-5678"]}, "raw_transcript": "resident-010-1234-5678", }, { "learner_id": LEARNER_B, "safety_event_id": 90, "session_id": UUID("10000000-0000-0000-0000-0000000000b2"), "turn_id": UUID("90000000-0000-0000-0000-000000000002"), "trigger_type": "boundary_violation", "ko_risk_level": 3, "escalated": True, "observed_sequence": 2, }, ] first = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=safety_rows, ) second = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=list(reversed(safety_rows)), ) self.assertEqual( [item.signal.model_dump(mode="json") for item in first], [item.signal.model_dump(mode="json") for item in second], ) self.assertEqual( {item.signal.signal_type for item in first}, {"safety_boundary"} ) self.assertTrue(all(item.signal.severity == "high" for item in first)) self.assertTrue(all(item.signal.uncertainty == 1.0 for item in first)) self.assertTrue( all(item.competency_id == "competency.safety_boundary" for item in first) ) pointers = [item.signal.evidence[0] for item in first] self.assertTrue(all(pointer.ledger == "safety_event" for pointer in pointers)) self.assertEqual({pointer.event_id for pointer in pointers}, {"90", "91"}) serialized = str([item.signal.model_dump(mode="json") for item in first]) self.assertNotIn("resident-010-1234-5678", serialized) self.assertNotIn("real_crisis", serialized) self.assertNotIn("boundary_violation", serialized) self.assertNotIn("transcript", serialized.lower()) def test_incomplete_or_non_escalated_safety_rows_fail_closed(self) -> None: derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=[ { "learner_id": LEARNER_A, "safety_event_id": 1, "session_id": SESSION_A, "trigger_type": "real_crisis", "escalated": False, "observed_sequence": 1, }, { "learner_id": LEARNER_A, "safety_event_id": 2, "session_id": None, "trigger_type": "real_crisis", "escalated": True, "observed_sequence": 2, }, { "learner_id": LEARNER_A, "safety_event_id": 3, "session_id": SESSION_A, "trigger_type": " ", "escalated": True, "observed_sequence": 3, }, ], ) self.assertEqual(derived, ()) def test_derived_safety_event_outranks_other_active_signals(self) -> None: distinct_learner = UUID("b0000000-0000-0000-0000-0000000000b2") derived = derive_attention_signals( trajectory_rows=[ { "learner_id": distinct_learner, "revision_id": UUID("20000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "deteriorating", "axes": [{"uncertainty": 0.1}], } ] }, } ], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=[ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "trigger_type": "real_crisis", "escalated": True, "observed_sequence": 9, } ], ) queue = build_attention_queue(item.signal for item in derived) self.assertEqual(queue[0].primary_signal, "safety_boundary") self.assertEqual(queue[1].primary_signal, "deterioration") def test_derives_only_observed_active_states_without_transcript(self) -> None: derived = derive_attention_signals( trajectory_rows=[ { "learner_id": LEARNER_A, "revision_id": UUID("20000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 3, "assessment": { "sessions": [ { "session_no": 3, "status": "deteriorating", "axes": [{"uncertainty": 0.2}, {"uncertainty": 0.4}], } ] }, }, { "learner_id": LEARNER_B, "revision_id": UUID("20000000-0000-0000-0000-000000000002"), "anchor_session_id": SESSION_A, "revision_no": 2, "assessment": { "sessions": [{"session_no": 2, "status": "on_track"}] }, }, ], rupture_rows=[ { "learner_id": LEARNER_A, "observation_id": UUID("30000000-0000-0000-0000-000000000001"), "session_id": SESSION_A, "sequence_no": 4, "to_state": "partial", "uncertainty": 0.3, }, { "learner_id": LEARNER_B, "observation_id": UUID("30000000-0000-0000-0000-000000000002"), "session_id": SESSION_A, "sequence_no": 5, "to_state": "resolved", "uncertainty": 0.1, }, ], calibration_rows=[], transfer_rows=[], practice_rows=[], ) self.assertEqual( {item.signal.signal_type for item in derived}, {"deterioration", "unresolved_rupture"}, ) self.assertTrue(all(item.signal.evidence for item in derived)) serialized = str([item.signal.model_dump(mode="json") for item in derived]) self.assertNotIn("transcript", serialized.lower()) def test_produced_pointer_uses_canonical_session_review_route(self) -> None: """세션이 있는 생산 신호는 교수자 세션 리뷰 경로를 가리킨다.""" derived = derive_attention_signals( trajectory_rows=[ { "learner_id": LEARNER_A, "revision_id": UUID("20000000-0000-0000-0000-000000000003"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "off_track", "axes": [{"uncertainty": 0.2}], } ] }, } ], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], ) self.assertEqual(len(derived), 1) self.assertEqual( derived[0].signal.evidence[0].route_hint, f"/teach/session/{SESSION_A}/review", ) def test_produced_pointer_without_session_falls_back_to_teach(self) -> None: """세션 없는 생산 신호는 교수자 첫 화면으로 안전하게 돌아간다.""" derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[ { "learner_id": LEARNER_A, "competency_id": "competency.empathy", "transfer_assessment_id": UUID( "50000000-0000-0000-0000-000000000003" ), "session_id": None, "assessment_payload": { "eligible": True, "transfer_verified": False, "error_interval": {"lower": 0.1, "upper": 0.3}, }, "observed_sequence": 1, } ], practice_rows=[], ) self.assertEqual(len(derived), 1) self.assertEqual(derived[0].signal.evidence[0].route_hint, "/teach") def test_requires_repeated_overconfidence_and_retry_for_persistent_signals( self, ) -> None: calibration_rows = [ { "learner_id": LEARNER_A, "competency_id": "competency.empathy", "assessment_snapshot_id": UUID( f"40000000-0000-0000-0000-00000000000{index}" ), "session_id": SESSION_A, "snapshot_no": index, "assessment_payload": { "bias": "overconfident", "error_interval": {"lower": 0.1, "upper": 0.4}, }, } for index in (1, 2) ] practice_rows = [ { "learner_id": LEARNER_A, "competency_id": "competency.reflection", "attempt_record_id": UUID( f"50000000-0000-0000-0000-00000000000{index}" ), "session_id": SESSION_A, "sequence_no": index, "outcome": "needs_retry", "uncertainty": 0.25, "counterevidence": ["client_response_withdrawn"], } for index in (1, 2) ] derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=calibration_rows, transfer_rows=[], practice_rows=practice_rows, ) self.assertEqual( {item.signal.signal_type for item in derived}, {"persistent_overconfidence", "growth_stagnation"}, ) def test_manifest_requires_all_domains_and_hashes_event_ids(self) -> None: rows = { domain: [ { "event_id": UUID(f"60000000-0000-0000-0000-{index:012d}"), "learner_id": LEARNER_A, "session_id": SESSION_A, } ] for index, domain in enumerate( ("alliance", "rupture", "transfer", "calibration"), start=1 ) } manifest = derive_manifest_input(rows) assert manifest is not None self.assertEqual({item.domain for item in manifest.artifacts}, set(rows)) self.assertTrue(all(item.record_count == 1 for item in manifest.artifacts)) self.assertEqual(len(manifest.source_fingerprint), 64) self.assertIsNone(derive_manifest_input({"alliance": rows["alliance"]})) class SupervisionResearchProducerCycleTest(IsolatedAsyncioTestCase): async def test_loader_reads_safety_metadata_without_sensitive_detail(self) -> None: conn = AsyncMock() conn.fetch.side_effect = [[], [], [], [], [], []] rows = await _load_attention_rows(conn, "cohort-a") self.assertIn("safety_rows", rows) safety_queries = [ str(call.args[0]) for call in conn.fetch.call_args_list if "app.safety_events" in str(call.args[0]) ] self.assertEqual(len(safety_queries), 1) query = safety_queries[0].lower() self.assertIn("se.id", query) self.assertIn("se.session_id", query) self.assertIn("se.turn_id", query) self.assertIn("se.trigger_type", query) self.assertIn("se.ko_risk_level", query) self.assertIn("se.escalated = true", query) self.assertNotIn("se.detail", query) self.assertNotIn("transcript", query) self.assertNotIn("matched", query) async def test_safety_attention_does_not_become_a_curriculum_gap(self) -> None: rows = { "trajectory_rows": [], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], "safety_rows": [ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "turn_id": None, "trigger_type": "real_crisis", "ko_risk_level": 5, "escalated": True, "observed_sequence": 1, } ], } append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) append_gap = AsyncMock() conn = AsyncMock() with ( patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=rows), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock( return_value={ domain: [] for domain in ("alliance", "rupture", "transfer", "calibration") } ), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), patch( "app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap", append_gap, ), ): result = await produce_supervision_cycle(conn, cohort_id="cohort-a") self.assertEqual(result["derived_signal_count"], 1) signal = append_attention.await_args.kwargs["signals"][0] self.assertEqual(signal.signal_type, "safety_boundary") append_gap.assert_not_awaited() async def test_all_cohort_cycle_isolates_one_cohort_failure(self) -> None: class AcquireContext: def __init__(self, conn: AsyncMock) -> None: self.conn = conn async def __aenter__(self) -> AsyncMock: return self.conn async def __aexit__(self, *_: object) -> None: return None list_conn = AsyncMock() list_conn.fetch.return_value = [{"cohort": "a"}, {"cohort": "b"}] cohort_a_conn = AsyncMock() cohort_b_conn = AsyncMock() comparison_conn = AsyncMock() cycle = AsyncMock(side_effect=[RuntimeError("broken cohort"), {"ok": True}]) comparison = AsyncMock(side_effect=RuntimeError("broken benchmark anchor")) with ( patch( "app.services.supervision_research_producer.db.acquire", side_effect=[ AcquireContext(list_conn), AcquireContext(cohort_a_conn), AcquireContext(cohort_b_conn), AcquireContext(comparison_conn), ], ), patch( "app.services.supervision_research_producer.produce_supervision_cycle", cycle, ), patch( "app.services.supervision_research_producer." "supervision_research_version_evaluator." "produce_repository_version_comparison", comparison, ), ): result = await produce_all_active_cohorts_once() self.assertEqual(result["cohort_count"], 2) self.assertEqual(result["completed"], 1) self.assertEqual(result["failed"], 1) self.assertEqual(result["version_comparison_failed"], 1) self.assertEqual(result["version_comparison"]["status"], "error") self.assertEqual(cycle.await_count, 2) comparison.assert_awaited_once_with(comparison_conn) async def test_cycle_derives_then_calls_append_only_stores(self) -> None: rows = { "trajectory_rows": [ { "learner_id": LEARNER_A, "revision_id": UUID("70000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "off_track", "axes": [{"uncertainty": 0.5}], } ] }, } ], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], } append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) append_gap = AsyncMock( return_value={"submission_id": UUID(int=3), "gap_snapshot_id": UUID(int=4)} ) conn = AsyncMock() with ( patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=rows), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock( return_value={ domain: [] for domain in ("alliance", "rupture", "transfer", "calibration") } ), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), patch( "app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap", append_gap, ), ): result = await produce_supervision_cycle(conn, cohort_id="cohort-a") self.assertEqual(result["derived_signal_count"], 1) self.assertFalse(result["raw_transcript_included"]) self.assertFalse(result["clinical_claim_allowed"]) append_attention.assert_awaited_once() append_gap.assert_awaited_once() @staticmethod def _off_track_trajectory_rows() -> dict[str, list[Mapping[str, object]]]: return { "trajectory_rows": [ { "learner_id": LEARNER_A, "revision_id": UUID("70000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "off_track", "axes": [{"uncertainty": 0.5}], } ] }, } ], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], } async def _run_gap_cycle(self, *, route_hint: str | None) -> dict[str, object]: append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) append_gap = AsyncMock( return_value={"submission_id": UUID(int=3), "gap_snapshot_id": UUID(int=4)} ) conn = AsyncMock() patches = [ patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=self._off_track_trajectory_rows()), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock(return_value={}), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), patch( "app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap", append_gap, ), ] if route_hint is not None: patches.append( patch( "app.services.supervision_research_producer._route", return_value=route_hint, ) ) with ExitStack() as stack: for one_patch in patches: stack.enter_context(one_patch) await produce_supervision_cycle(conn, cohort_id="cohort-a") return append_gap.await_args.kwargs async def test_gap_submission_id_changes_when_evidence_route_changes( self, ) -> None: """신호 번호는 같고 근거 포인터의 route_hint만 달라지면 공백 제출 번호도 달라야 한다.""" old = await self._run_gap_cycle(route_hint="/teacher/sessions/old-route") new = await self._run_gap_cycle( route_hint="/teach/session/new-route/review" ) self.assertNotEqual(old["submission_id"], new["submission_id"]) self.assertNotEqual(old["gap_snapshot_id"], new["gap_snapshot_id"]) async def test_gap_submission_is_idempotent_for_identical_input(self) -> None: first = await self._run_gap_cycle(route_hint=None) second = await self._run_gap_cycle(route_hint=None) self.assertEqual(first["submission_id"], second["submission_id"]) self.assertEqual(first["gap_snapshot_id"], second["gap_snapshot_id"]) @staticmethod def _manifest_rows() -> dict[str, list[Mapping[str, object]]]: return { domain: [ { "event_id": UUID(f"60000000-0000-0000-0000-{index:012d}"), "learner_id": LEARNER_A, "session_id": SESSION_A, } ] for index, domain in enumerate( ("alliance", "rupture", "transfer", "calibration"), start=1 ) } async def _run_manifest_cycle(self, *, route_hint: str | None) -> dict[str, object]: append_manifest = AsyncMock( return_value={"submission_id": UUID(int=5), "manifest_id": UUID(int=6)} ) conn = AsyncMock() patches = [ patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock( return_value={ "trajectory_rows": [], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], } ), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock(return_value=self._manifest_rows()), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_phase3_manifest", append_manifest, ), ] if route_hint is not None: patches.append( patch( "app.services.supervision_research_producer._route", return_value=route_hint, ) ) with ExitStack() as stack: for one_patch in patches: stack.enter_context(one_patch) await produce_supervision_cycle(conn, cohort_id="cohort-a") return append_manifest.await_args.kwargs async def test_manifest_submission_id_changes_when_evidence_route_changes( self, ) -> None: """근거 묶음도 신호(이벤트) 번호는 같고 route_hint만 달라지면 제출 번호가 달라야 한다.""" old = await self._run_manifest_cycle(route_hint="/teacher/sessions/old-route") new = await self._run_manifest_cycle( route_hint="/teach/session/new-route/review" ) self.assertNotEqual(old["submission_id"], new["submission_id"]) self.assertNotEqual(old["manifest_id"], new["manifest_id"]) async def test_manifest_submission_is_idempotent_for_identical_input( self, ) -> None: first = await self._run_manifest_cycle(route_hint=None) second = await self._run_manifest_cycle(route_hint=None) self.assertEqual(first["submission_id"], second["submission_id"]) self.assertEqual(first["manifest_id"], second["manifest_id"]) async def test_new_cycle_appends_new_submission_when_old_route_already_stored( self, ) -> None: """운영에 저장된 옛 링크 행이 있어도 새 주기는 충돌 없이 새 번호로 append한다. 2026-09-09 커밋(1306c524)으로 `_route()` 근거 링크 형식이 바뀌기 전에는 교육과정 공백 제출 번호 지문이 신호 번호만 덮어서, 내용(route)만 바뀐 재실행이 같은 제출 번호에 다른 내용을 넣어 SupervisionResearchConflictError로 충돌했다. 수정 후에는 포인터 내용까지 지문에 들어가므로 새 번호로 충돌 없이 append된다. """ revision_id = UUID("70000000-0000-0000-0000-000000000001") old_pointer = LedgerEvidencePointer( ledger="outcome_trajectory_revision", event_id=str(revision_id), session_id=str(SESSION_A), route_hint=f"/teacher/sessions/{SESSION_A}", ) old_signal_id = _signal_id("trajectory", revision_id, "off_track") old_gap_fingerprint = _canonical_hash( ["competency.outcome_monitoring", "coverage", [old_signal_id]] ) old_submission_id = uuid5( _PRODUCER_NAMESPACE, f"gap-submission:cohort-a:{old_gap_fingerprint}" ) old_gap_snapshot_id = uuid5( _PRODUCER_NAMESPACE, f"gap-snapshot:cohort-a:{old_gap_fingerprint}" ) conn = _FakeCurriculumGapConn() await supervision_research_store.append_curriculum_gap( conn, submission_id=old_submission_id, gap_snapshot_id=old_gap_snapshot_id, cohort_id="cohort-a", competency_id="competency.outcome_monitoring", gap_kind="coverage", status="monitoring", uncertainty=0.5, affected_learner_count=1, evidence=[(LEARNER_A, old_pointer)], ) append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) with ( patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=self._off_track_trajectory_rows()), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock(return_value={}), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), ): result = await produce_supervision_cycle(conn, cohort_id="cohort-a") self.assertEqual(len(result["curriculum_gaps"]), 1) gap_result = result["curriculum_gaps"][0] self.assertFalse(gap_result["idempotent_replay"]) self.assertNotEqual(gap_result["submission_id"], old_submission_id) self.assertNotEqual(gap_result["gap_snapshot_id"], old_gap_snapshot_id) class _FakeCurriculumGapConn: """교육과정 공백 제출 번호 충돌 검사만 재현하는 최소 모의 커넥션.""" def __init__(self) -> None: self.gap_rows: dict[UUID, tuple[UUID, str]] = {} self.pointer_rows: dict[UUID, str] = {} async def execute(self, query: str, *args: object) -> None: if "pg_advisory_xact_lock" in query: return if "INSERT INTO app.supervision_evidence_pointer" in query: pointer_id = args[0] content_hash = args[8] self.pointer_rows.setdefault(pointer_id, content_hash) return if "INSERT INTO app.supervision_curriculum_gap_snapshot" in query: gap_snapshot_id, submission_id, content_hash = args[0], args[1], args[2] self.gap_rows[submission_id] = (gap_snapshot_id, content_hash) return raise AssertionError(f"unexpected execute: {query}") async def fetchrow(self, query: str, *args: object) -> Mapping[str, object] | None: if "app.supervision_curriculum_gap_snapshot" in query: submission_id = args[0] if submission_id in self.gap_rows: gap_snapshot_id, content_hash = self.gap_rows[submission_id] return {"gap_snapshot_id": gap_snapshot_id, "content_hash": content_hash} return None if "app.supervision_evidence_pointer" in query: pointer_id = args[0] if pointer_id in self.pointer_rows: return {"content_hash": self.pointer_rows[pointer_id]} return None raise AssertionError(f"unexpected fetchrow: {query}")