from __future__ import annotations from unittest import IsolatedAsyncioTestCase, TestCase from unittest.mock import AsyncMock, patch from uuid import UUID from .services.supervision_research_producer import ( _load_attention_rows, derive_attention_signals, derive_manifest_input, produce_all_active_cohorts_once, produce_supervision_cycle, ) from .services.supervision_research import build_attention_queue LEARNER_A = UUID("00000000-0000-0000-0000-0000000000a1") LEARNER_B = UUID("00000000-0000-0000-0000-0000000000b2") SESSION_A = UUID("10000000-0000-0000-0000-0000000000a1") class SupervisionResearchProducerDerivationTest(TestCase): def test_escalated_safety_event_becomes_metadata_only_priority_signal( self, ) -> None: safety_rows = [ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "turn_id": None, "trigger_type": "real_crisis", "ko_risk_level": 5, "escalated": True, "observed_sequence": 7, # 호출자가 실수로 전체 row를 넘겨도 민감 원천 payload를 # G6 신호에 복제하면 안 된다. "detail": {"matched": ["resident-010-1234-5678"]}, "raw_transcript": "resident-010-1234-5678", }, { "learner_id": LEARNER_B, "safety_event_id": 90, "session_id": UUID("10000000-0000-0000-0000-0000000000b2"), "turn_id": UUID("90000000-0000-0000-0000-000000000002"), "trigger_type": "boundary_violation", "ko_risk_level": 3, "escalated": True, "observed_sequence": 2, }, ] first = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=safety_rows, ) second = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=list(reversed(safety_rows)), ) self.assertEqual( [item.signal.model_dump(mode="json") for item in first], [item.signal.model_dump(mode="json") for item in second], ) self.assertEqual( {item.signal.signal_type for item in first}, {"safety_boundary"} ) self.assertTrue(all(item.signal.severity == "high" for item in first)) self.assertTrue(all(item.signal.uncertainty == 1.0 for item in first)) self.assertTrue( all(item.competency_id == "competency.safety_boundary" for item in first) ) pointers = [item.signal.evidence[0] for item in first] self.assertTrue(all(pointer.ledger == "safety_event" for pointer in pointers)) self.assertEqual({pointer.event_id for pointer in pointers}, {"90", "91"}) serialized = str([item.signal.model_dump(mode="json") for item in first]) self.assertNotIn("resident-010-1234-5678", serialized) self.assertNotIn("real_crisis", serialized) self.assertNotIn("boundary_violation", serialized) self.assertNotIn("transcript", serialized.lower()) def test_incomplete_or_non_escalated_safety_rows_fail_closed(self) -> None: derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=[ { "learner_id": LEARNER_A, "safety_event_id": 1, "session_id": SESSION_A, "trigger_type": "real_crisis", "escalated": False, "observed_sequence": 1, }, { "learner_id": LEARNER_A, "safety_event_id": 2, "session_id": None, "trigger_type": "real_crisis", "escalated": True, "observed_sequence": 2, }, { "learner_id": LEARNER_A, "safety_event_id": 3, "session_id": SESSION_A, "trigger_type": " ", "escalated": True, "observed_sequence": 3, }, ], ) self.assertEqual(derived, ()) def test_derived_safety_event_outranks_other_active_signals(self) -> None: distinct_learner = UUID("b0000000-0000-0000-0000-0000000000b2") derived = derive_attention_signals( trajectory_rows=[ { "learner_id": distinct_learner, "revision_id": UUID("20000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "deteriorating", "axes": [{"uncertainty": 0.1}], } ] }, } ], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], safety_rows=[ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "trigger_type": "real_crisis", "escalated": True, "observed_sequence": 9, } ], ) queue = build_attention_queue(item.signal for item in derived) self.assertEqual(queue[0].primary_signal, "safety_boundary") self.assertEqual(queue[1].primary_signal, "deterioration") def test_derives_only_observed_active_states_without_transcript(self) -> None: derived = derive_attention_signals( trajectory_rows=[ { "learner_id": LEARNER_A, "revision_id": UUID("20000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 3, "assessment": { "sessions": [ { "session_no": 3, "status": "deteriorating", "axes": [{"uncertainty": 0.2}, {"uncertainty": 0.4}], } ] }, }, { "learner_id": LEARNER_B, "revision_id": UUID("20000000-0000-0000-0000-000000000002"), "anchor_session_id": SESSION_A, "revision_no": 2, "assessment": { "sessions": [{"session_no": 2, "status": "on_track"}] }, }, ], rupture_rows=[ { "learner_id": LEARNER_A, "observation_id": UUID("30000000-0000-0000-0000-000000000001"), "session_id": SESSION_A, "sequence_no": 4, "to_state": "partial", "uncertainty": 0.3, }, { "learner_id": LEARNER_B, "observation_id": UUID("30000000-0000-0000-0000-000000000002"), "session_id": SESSION_A, "sequence_no": 5, "to_state": "resolved", "uncertainty": 0.1, }, ], calibration_rows=[], transfer_rows=[], practice_rows=[], ) self.assertEqual( {item.signal.signal_type for item in derived}, {"deterioration", "unresolved_rupture"}, ) self.assertTrue(all(item.signal.evidence for item in derived)) serialized = str([item.signal.model_dump(mode="json") for item in derived]) self.assertNotIn("transcript", serialized.lower()) def test_produced_pointer_uses_canonical_session_review_route(self) -> None: """세션이 있는 생산 신호는 교수자 세션 리뷰 경로를 가리킨다.""" derived = derive_attention_signals( trajectory_rows=[ { "learner_id": LEARNER_A, "revision_id": UUID("20000000-0000-0000-0000-000000000003"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "off_track", "axes": [{"uncertainty": 0.2}], } ] }, } ], rupture_rows=[], calibration_rows=[], transfer_rows=[], practice_rows=[], ) self.assertEqual(len(derived), 1) self.assertEqual( derived[0].signal.evidence[0].route_hint, f"/teach/session/{SESSION_A}/review", ) def test_produced_pointer_without_session_falls_back_to_teach(self) -> None: """세션 없는 생산 신호는 교수자 첫 화면으로 안전하게 돌아간다.""" derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=[], transfer_rows=[ { "learner_id": LEARNER_A, "competency_id": "competency.empathy", "transfer_assessment_id": UUID( "50000000-0000-0000-0000-000000000003" ), "session_id": None, "assessment_payload": { "eligible": True, "transfer_verified": False, "error_interval": {"lower": 0.1, "upper": 0.3}, }, "observed_sequence": 1, } ], practice_rows=[], ) self.assertEqual(len(derived), 1) self.assertEqual(derived[0].signal.evidence[0].route_hint, "/teach") def test_requires_repeated_overconfidence_and_retry_for_persistent_signals( self, ) -> None: calibration_rows = [ { "learner_id": LEARNER_A, "competency_id": "competency.empathy", "assessment_snapshot_id": UUID( f"40000000-0000-0000-0000-00000000000{index}" ), "session_id": SESSION_A, "snapshot_no": index, "assessment_payload": { "bias": "overconfident", "error_interval": {"lower": 0.1, "upper": 0.4}, }, } for index in (1, 2) ] practice_rows = [ { "learner_id": LEARNER_A, "competency_id": "competency.reflection", "attempt_record_id": UUID( f"50000000-0000-0000-0000-00000000000{index}" ), "session_id": SESSION_A, "sequence_no": index, "outcome": "needs_retry", "uncertainty": 0.25, "counterevidence": ["client_response_withdrawn"], } for index in (1, 2) ] derived = derive_attention_signals( trajectory_rows=[], rupture_rows=[], calibration_rows=calibration_rows, transfer_rows=[], practice_rows=practice_rows, ) self.assertEqual( {item.signal.signal_type for item in derived}, {"persistent_overconfidence", "growth_stagnation"}, ) def test_manifest_requires_all_domains_and_hashes_event_ids(self) -> None: rows = { domain: [ { "event_id": UUID(f"60000000-0000-0000-0000-{index:012d}"), "learner_id": LEARNER_A, "session_id": SESSION_A, } ] for index, domain in enumerate( ("alliance", "rupture", "transfer", "calibration"), start=1 ) } manifest = derive_manifest_input(rows) assert manifest is not None self.assertEqual({item.domain for item in manifest.artifacts}, set(rows)) self.assertTrue(all(item.record_count == 1 for item in manifest.artifacts)) self.assertEqual(len(manifest.source_fingerprint), 64) self.assertIsNone(derive_manifest_input({"alliance": rows["alliance"]})) class SupervisionResearchProducerCycleTest(IsolatedAsyncioTestCase): async def test_loader_reads_safety_metadata_without_sensitive_detail(self) -> None: conn = AsyncMock() conn.fetch.side_effect = [[], [], [], [], [], []] rows = await _load_attention_rows(conn, "cohort-a") self.assertIn("safety_rows", rows) safety_queries = [ str(call.args[0]) for call in conn.fetch.call_args_list if "app.safety_events" in str(call.args[0]) ] self.assertEqual(len(safety_queries), 1) query = safety_queries[0].lower() self.assertIn("se.id", query) self.assertIn("se.session_id", query) self.assertIn("se.turn_id", query) self.assertIn("se.trigger_type", query) self.assertIn("se.ko_risk_level", query) self.assertIn("se.escalated = true", query) self.assertNotIn("se.detail", query) self.assertNotIn("transcript", query) self.assertNotIn("matched", query) async def test_safety_attention_does_not_become_a_curriculum_gap(self) -> None: rows = { "trajectory_rows": [], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], "safety_rows": [ { "learner_id": LEARNER_A, "safety_event_id": 91, "session_id": SESSION_A, "turn_id": None, "trigger_type": "real_crisis", "ko_risk_level": 5, "escalated": True, "observed_sequence": 1, } ], } append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) append_gap = AsyncMock() conn = AsyncMock() with ( patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=rows), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock( return_value={ domain: [] for domain in ("alliance", "rupture", "transfer", "calibration") } ), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), patch( "app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap", append_gap, ), ): result = await produce_supervision_cycle(conn, cohort_id="cohort-a") self.assertEqual(result["derived_signal_count"], 1) signal = append_attention.await_args.kwargs["signals"][0] self.assertEqual(signal.signal_type, "safety_boundary") append_gap.assert_not_awaited() async def test_all_cohort_cycle_isolates_one_cohort_failure(self) -> None: class AcquireContext: def __init__(self, conn: AsyncMock) -> None: self.conn = conn async def __aenter__(self) -> AsyncMock: return self.conn async def __aexit__(self, *_: object) -> None: return None list_conn = AsyncMock() list_conn.fetch.return_value = [{"cohort": "a"}, {"cohort": "b"}] cohort_a_conn = AsyncMock() cohort_b_conn = AsyncMock() comparison_conn = AsyncMock() cycle = AsyncMock(side_effect=[RuntimeError("broken cohort"), {"ok": True}]) comparison = AsyncMock(side_effect=RuntimeError("broken benchmark anchor")) with ( patch( "app.services.supervision_research_producer.db.acquire", side_effect=[ AcquireContext(list_conn), AcquireContext(cohort_a_conn), AcquireContext(cohort_b_conn), AcquireContext(comparison_conn), ], ), patch( "app.services.supervision_research_producer.produce_supervision_cycle", cycle, ), patch( "app.services.supervision_research_producer." "supervision_research_version_evaluator." "produce_repository_version_comparison", comparison, ), ): result = await produce_all_active_cohorts_once() self.assertEqual(result["cohort_count"], 2) self.assertEqual(result["completed"], 1) self.assertEqual(result["failed"], 1) self.assertEqual(result["version_comparison_failed"], 1) self.assertEqual(result["version_comparison"]["status"], "error") self.assertEqual(cycle.await_count, 2) comparison.assert_awaited_once_with(comparison_conn) async def test_cycle_derives_then_calls_append_only_stores(self) -> None: rows = { "trajectory_rows": [ { "learner_id": LEARNER_A, "revision_id": UUID("70000000-0000-0000-0000-000000000001"), "anchor_session_id": SESSION_A, "revision_no": 1, "assessment": { "sessions": [ { "session_no": 1, "status": "off_track", "axes": [{"uncertainty": 0.5}], } ] }, } ], "rupture_rows": [], "calibration_rows": [], "transfer_rows": [], "practice_rows": [], } append_attention = AsyncMock( return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)} ) append_gap = AsyncMock( return_value={"submission_id": UUID(int=3), "gap_snapshot_id": UUID(int=4)} ) conn = AsyncMock() with ( patch( "app.services.supervision_research_producer._load_attention_rows", AsyncMock(return_value=rows), ), patch( "app.services.supervision_research_producer._load_manifest_rows", AsyncMock( return_value={ domain: [] for domain in ("alliance", "rupture", "transfer", "calibration") } ), ), patch( "app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot", append_attention, ), patch( "app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap", append_gap, ), ): result = await produce_supervision_cycle(conn, cohort_id="cohort-a") self.assertEqual(result["derived_signal_count"], 1) self.assertFalse(result["raw_transcript_included"]) self.assertFalse(result["clinical_claim_allowed"]) append_attention.assert_awaited_once() append_gap.assert_awaited_once()