544 lines
20 KiB
Python
544 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
from unittest import IsolatedAsyncioTestCase, TestCase
|
|
from unittest.mock import AsyncMock, patch
|
|
from uuid import UUID
|
|
|
|
from .services.supervision_research_producer import (
|
|
_load_attention_rows,
|
|
derive_attention_signals,
|
|
derive_manifest_input,
|
|
produce_all_active_cohorts_once,
|
|
produce_supervision_cycle,
|
|
)
|
|
from .services.supervision_research import build_attention_queue
|
|
|
|
|
|
LEARNER_A = UUID("00000000-0000-0000-0000-0000000000a1")
|
|
LEARNER_B = UUID("00000000-0000-0000-0000-0000000000b2")
|
|
SESSION_A = UUID("10000000-0000-0000-0000-0000000000a1")
|
|
|
|
|
|
class SupervisionResearchProducerDerivationTest(TestCase):
|
|
def test_escalated_safety_event_becomes_metadata_only_priority_signal(
|
|
self,
|
|
) -> None:
|
|
safety_rows = [
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 91,
|
|
"session_id": SESSION_A,
|
|
"turn_id": None,
|
|
"trigger_type": "real_crisis",
|
|
"ko_risk_level": 5,
|
|
"escalated": True,
|
|
"observed_sequence": 7,
|
|
# 호출자가 실수로 전체 row를 넘겨도 민감 원천 payload를
|
|
# G6 신호에 복제하면 안 된다.
|
|
"detail": {"matched": ["resident-010-1234-5678"]},
|
|
"raw_transcript": "resident-010-1234-5678",
|
|
},
|
|
{
|
|
"learner_id": LEARNER_B,
|
|
"safety_event_id": 90,
|
|
"session_id": UUID("10000000-0000-0000-0000-0000000000b2"),
|
|
"turn_id": UUID("90000000-0000-0000-0000-000000000002"),
|
|
"trigger_type": "boundary_violation",
|
|
"ko_risk_level": 3,
|
|
"escalated": True,
|
|
"observed_sequence": 2,
|
|
},
|
|
]
|
|
|
|
first = derive_attention_signals(
|
|
trajectory_rows=[],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
safety_rows=safety_rows,
|
|
)
|
|
second = derive_attention_signals(
|
|
trajectory_rows=[],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
safety_rows=list(reversed(safety_rows)),
|
|
)
|
|
|
|
self.assertEqual(
|
|
[item.signal.model_dump(mode="json") for item in first],
|
|
[item.signal.model_dump(mode="json") for item in second],
|
|
)
|
|
self.assertEqual(
|
|
{item.signal.signal_type for item in first}, {"safety_boundary"}
|
|
)
|
|
self.assertTrue(all(item.signal.severity == "high" for item in first))
|
|
self.assertTrue(all(item.signal.uncertainty == 1.0 for item in first))
|
|
self.assertTrue(
|
|
all(item.competency_id == "competency.safety_boundary" for item in first)
|
|
)
|
|
pointers = [item.signal.evidence[0] for item in first]
|
|
self.assertTrue(all(pointer.ledger == "safety_event" for pointer in pointers))
|
|
self.assertEqual({pointer.event_id for pointer in pointers}, {"90", "91"})
|
|
serialized = str([item.signal.model_dump(mode="json") for item in first])
|
|
self.assertNotIn("resident-010-1234-5678", serialized)
|
|
self.assertNotIn("real_crisis", serialized)
|
|
self.assertNotIn("boundary_violation", serialized)
|
|
self.assertNotIn("transcript", serialized.lower())
|
|
|
|
def test_incomplete_or_non_escalated_safety_rows_fail_closed(self) -> None:
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
safety_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 1,
|
|
"session_id": SESSION_A,
|
|
"trigger_type": "real_crisis",
|
|
"escalated": False,
|
|
"observed_sequence": 1,
|
|
},
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 2,
|
|
"session_id": None,
|
|
"trigger_type": "real_crisis",
|
|
"escalated": True,
|
|
"observed_sequence": 2,
|
|
},
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 3,
|
|
"session_id": SESSION_A,
|
|
"trigger_type": " ",
|
|
"escalated": True,
|
|
"observed_sequence": 3,
|
|
},
|
|
],
|
|
)
|
|
|
|
self.assertEqual(derived, ())
|
|
|
|
def test_derived_safety_event_outranks_other_active_signals(self) -> None:
|
|
distinct_learner = UUID("b0000000-0000-0000-0000-0000000000b2")
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[
|
|
{
|
|
"learner_id": distinct_learner,
|
|
"revision_id": UUID("20000000-0000-0000-0000-000000000001"),
|
|
"anchor_session_id": SESSION_A,
|
|
"revision_no": 1,
|
|
"assessment": {
|
|
"sessions": [
|
|
{
|
|
"session_no": 1,
|
|
"status": "deteriorating",
|
|
"axes": [{"uncertainty": 0.1}],
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
safety_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 91,
|
|
"session_id": SESSION_A,
|
|
"trigger_type": "real_crisis",
|
|
"escalated": True,
|
|
"observed_sequence": 9,
|
|
}
|
|
],
|
|
)
|
|
|
|
queue = build_attention_queue(item.signal for item in derived)
|
|
|
|
self.assertEqual(queue[0].primary_signal, "safety_boundary")
|
|
self.assertEqual(queue[1].primary_signal, "deterioration")
|
|
|
|
def test_derives_only_observed_active_states_without_transcript(self) -> None:
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"revision_id": UUID("20000000-0000-0000-0000-000000000001"),
|
|
"anchor_session_id": SESSION_A,
|
|
"revision_no": 3,
|
|
"assessment": {
|
|
"sessions": [
|
|
{
|
|
"session_no": 3,
|
|
"status": "deteriorating",
|
|
"axes": [{"uncertainty": 0.2}, {"uncertainty": 0.4}],
|
|
}
|
|
]
|
|
},
|
|
},
|
|
{
|
|
"learner_id": LEARNER_B,
|
|
"revision_id": UUID("20000000-0000-0000-0000-000000000002"),
|
|
"anchor_session_id": SESSION_A,
|
|
"revision_no": 2,
|
|
"assessment": {
|
|
"sessions": [{"session_no": 2, "status": "on_track"}]
|
|
},
|
|
},
|
|
],
|
|
rupture_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"observation_id": UUID("30000000-0000-0000-0000-000000000001"),
|
|
"session_id": SESSION_A,
|
|
"sequence_no": 4,
|
|
"to_state": "partial",
|
|
"uncertainty": 0.3,
|
|
},
|
|
{
|
|
"learner_id": LEARNER_B,
|
|
"observation_id": UUID("30000000-0000-0000-0000-000000000002"),
|
|
"session_id": SESSION_A,
|
|
"sequence_no": 5,
|
|
"to_state": "resolved",
|
|
"uncertainty": 0.1,
|
|
},
|
|
],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
)
|
|
|
|
self.assertEqual(
|
|
{item.signal.signal_type for item in derived},
|
|
{"deterioration", "unresolved_rupture"},
|
|
)
|
|
self.assertTrue(all(item.signal.evidence for item in derived))
|
|
serialized = str([item.signal.model_dump(mode="json") for item in derived])
|
|
self.assertNotIn("transcript", serialized.lower())
|
|
|
|
def test_produced_pointer_uses_canonical_session_review_route(self) -> None:
|
|
"""세션이 있는 생산 신호는 교수자 세션 리뷰 경로를 가리킨다."""
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"revision_id": UUID("20000000-0000-0000-0000-000000000003"),
|
|
"anchor_session_id": SESSION_A,
|
|
"revision_no": 1,
|
|
"assessment": {
|
|
"sessions": [
|
|
{
|
|
"session_no": 1,
|
|
"status": "off_track",
|
|
"axes": [{"uncertainty": 0.2}],
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[],
|
|
practice_rows=[],
|
|
)
|
|
|
|
self.assertEqual(len(derived), 1)
|
|
self.assertEqual(
|
|
derived[0].signal.evidence[0].route_hint,
|
|
f"/teach/session/{SESSION_A}/review",
|
|
)
|
|
|
|
def test_produced_pointer_without_session_falls_back_to_teach(self) -> None:
|
|
"""세션 없는 생산 신호는 교수자 첫 화면으로 안전하게 돌아간다."""
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[],
|
|
rupture_rows=[],
|
|
calibration_rows=[],
|
|
transfer_rows=[
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"competency_id": "competency.empathy",
|
|
"transfer_assessment_id": UUID(
|
|
"50000000-0000-0000-0000-000000000003"
|
|
),
|
|
"session_id": None,
|
|
"assessment_payload": {
|
|
"eligible": True,
|
|
"transfer_verified": False,
|
|
"error_interval": {"lower": 0.1, "upper": 0.3},
|
|
},
|
|
"observed_sequence": 1,
|
|
}
|
|
],
|
|
practice_rows=[],
|
|
)
|
|
|
|
self.assertEqual(len(derived), 1)
|
|
self.assertEqual(derived[0].signal.evidence[0].route_hint, "/teach")
|
|
|
|
def test_requires_repeated_overconfidence_and_retry_for_persistent_signals(
|
|
self,
|
|
) -> None:
|
|
calibration_rows = [
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"competency_id": "competency.empathy",
|
|
"assessment_snapshot_id": UUID(
|
|
f"40000000-0000-0000-0000-00000000000{index}"
|
|
),
|
|
"session_id": SESSION_A,
|
|
"snapshot_no": index,
|
|
"assessment_payload": {
|
|
"bias": "overconfident",
|
|
"error_interval": {"lower": 0.1, "upper": 0.4},
|
|
},
|
|
}
|
|
for index in (1, 2)
|
|
]
|
|
practice_rows = [
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"competency_id": "competency.reflection",
|
|
"attempt_record_id": UUID(
|
|
f"50000000-0000-0000-0000-00000000000{index}"
|
|
),
|
|
"session_id": SESSION_A,
|
|
"sequence_no": index,
|
|
"outcome": "needs_retry",
|
|
"uncertainty": 0.25,
|
|
"counterevidence": ["client_response_withdrawn"],
|
|
}
|
|
for index in (1, 2)
|
|
]
|
|
derived = derive_attention_signals(
|
|
trajectory_rows=[],
|
|
rupture_rows=[],
|
|
calibration_rows=calibration_rows,
|
|
transfer_rows=[],
|
|
practice_rows=practice_rows,
|
|
)
|
|
self.assertEqual(
|
|
{item.signal.signal_type for item in derived},
|
|
{"persistent_overconfidence", "growth_stagnation"},
|
|
)
|
|
|
|
def test_manifest_requires_all_domains_and_hashes_event_ids(self) -> None:
|
|
rows = {
|
|
domain: [
|
|
{
|
|
"event_id": UUID(f"60000000-0000-0000-0000-{index:012d}"),
|
|
"learner_id": LEARNER_A,
|
|
"session_id": SESSION_A,
|
|
}
|
|
]
|
|
for index, domain in enumerate(
|
|
("alliance", "rupture", "transfer", "calibration"), start=1
|
|
)
|
|
}
|
|
manifest = derive_manifest_input(rows)
|
|
assert manifest is not None
|
|
self.assertEqual({item.domain for item in manifest.artifacts}, set(rows))
|
|
self.assertTrue(all(item.record_count == 1 for item in manifest.artifacts))
|
|
self.assertEqual(len(manifest.source_fingerprint), 64)
|
|
self.assertIsNone(derive_manifest_input({"alliance": rows["alliance"]}))
|
|
|
|
|
|
class SupervisionResearchProducerCycleTest(IsolatedAsyncioTestCase):
|
|
async def test_loader_reads_safety_metadata_without_sensitive_detail(self) -> None:
|
|
conn = AsyncMock()
|
|
conn.fetch.side_effect = [[], [], [], [], [], []]
|
|
|
|
rows = await _load_attention_rows(conn, "cohort-a")
|
|
|
|
self.assertIn("safety_rows", rows)
|
|
safety_queries = [
|
|
str(call.args[0])
|
|
for call in conn.fetch.call_args_list
|
|
if "app.safety_events" in str(call.args[0])
|
|
]
|
|
self.assertEqual(len(safety_queries), 1)
|
|
query = safety_queries[0].lower()
|
|
self.assertIn("se.id", query)
|
|
self.assertIn("se.session_id", query)
|
|
self.assertIn("se.turn_id", query)
|
|
self.assertIn("se.trigger_type", query)
|
|
self.assertIn("se.ko_risk_level", query)
|
|
self.assertIn("se.escalated = true", query)
|
|
self.assertNotIn("se.detail", query)
|
|
self.assertNotIn("transcript", query)
|
|
self.assertNotIn("matched", query)
|
|
|
|
async def test_safety_attention_does_not_become_a_curriculum_gap(self) -> None:
|
|
rows = {
|
|
"trajectory_rows": [],
|
|
"rupture_rows": [],
|
|
"calibration_rows": [],
|
|
"transfer_rows": [],
|
|
"practice_rows": [],
|
|
"safety_rows": [
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"safety_event_id": 91,
|
|
"session_id": SESSION_A,
|
|
"turn_id": None,
|
|
"trigger_type": "real_crisis",
|
|
"ko_risk_level": 5,
|
|
"escalated": True,
|
|
"observed_sequence": 1,
|
|
}
|
|
],
|
|
}
|
|
append_attention = AsyncMock(
|
|
return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)}
|
|
)
|
|
append_gap = AsyncMock()
|
|
conn = AsyncMock()
|
|
with (
|
|
patch(
|
|
"app.services.supervision_research_producer._load_attention_rows",
|
|
AsyncMock(return_value=rows),
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer._load_manifest_rows",
|
|
AsyncMock(
|
|
return_value={
|
|
domain: []
|
|
for domain in ("alliance", "rupture", "transfer", "calibration")
|
|
}
|
|
),
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot",
|
|
append_attention,
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap",
|
|
append_gap,
|
|
),
|
|
):
|
|
result = await produce_supervision_cycle(conn, cohort_id="cohort-a")
|
|
|
|
self.assertEqual(result["derived_signal_count"], 1)
|
|
signal = append_attention.await_args.kwargs["signals"][0]
|
|
self.assertEqual(signal.signal_type, "safety_boundary")
|
|
append_gap.assert_not_awaited()
|
|
|
|
async def test_all_cohort_cycle_isolates_one_cohort_failure(self) -> None:
|
|
class AcquireContext:
|
|
def __init__(self, conn: AsyncMock) -> None:
|
|
self.conn = conn
|
|
|
|
async def __aenter__(self) -> AsyncMock:
|
|
return self.conn
|
|
|
|
async def __aexit__(self, *_: object) -> None:
|
|
return None
|
|
|
|
list_conn = AsyncMock()
|
|
list_conn.fetch.return_value = [{"cohort": "a"}, {"cohort": "b"}]
|
|
cohort_a_conn = AsyncMock()
|
|
cohort_b_conn = AsyncMock()
|
|
comparison_conn = AsyncMock()
|
|
cycle = AsyncMock(side_effect=[RuntimeError("broken cohort"), {"ok": True}])
|
|
comparison = AsyncMock(side_effect=RuntimeError("broken benchmark anchor"))
|
|
with (
|
|
patch(
|
|
"app.services.supervision_research_producer.db.acquire",
|
|
side_effect=[
|
|
AcquireContext(list_conn),
|
|
AcquireContext(cohort_a_conn),
|
|
AcquireContext(cohort_b_conn),
|
|
AcquireContext(comparison_conn),
|
|
],
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer.produce_supervision_cycle",
|
|
cycle,
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer."
|
|
"supervision_research_version_evaluator."
|
|
"produce_repository_version_comparison",
|
|
comparison,
|
|
),
|
|
):
|
|
result = await produce_all_active_cohorts_once()
|
|
|
|
self.assertEqual(result["cohort_count"], 2)
|
|
self.assertEqual(result["completed"], 1)
|
|
self.assertEqual(result["failed"], 1)
|
|
self.assertEqual(result["version_comparison_failed"], 1)
|
|
self.assertEqual(result["version_comparison"]["status"], "error")
|
|
self.assertEqual(cycle.await_count, 2)
|
|
comparison.assert_awaited_once_with(comparison_conn)
|
|
|
|
async def test_cycle_derives_then_calls_append_only_stores(self) -> None:
|
|
rows = {
|
|
"trajectory_rows": [
|
|
{
|
|
"learner_id": LEARNER_A,
|
|
"revision_id": UUID("70000000-0000-0000-0000-000000000001"),
|
|
"anchor_session_id": SESSION_A,
|
|
"revision_no": 1,
|
|
"assessment": {
|
|
"sessions": [
|
|
{
|
|
"session_no": 1,
|
|
"status": "off_track",
|
|
"axes": [{"uncertainty": 0.5}],
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
"rupture_rows": [],
|
|
"calibration_rows": [],
|
|
"transfer_rows": [],
|
|
"practice_rows": [],
|
|
}
|
|
append_attention = AsyncMock(
|
|
return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)}
|
|
)
|
|
append_gap = AsyncMock(
|
|
return_value={"submission_id": UUID(int=3), "gap_snapshot_id": UUID(int=4)}
|
|
)
|
|
conn = AsyncMock()
|
|
with (
|
|
patch(
|
|
"app.services.supervision_research_producer._load_attention_rows",
|
|
AsyncMock(return_value=rows),
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer._load_manifest_rows",
|
|
AsyncMock(
|
|
return_value={
|
|
domain: []
|
|
for domain in ("alliance", "rupture", "transfer", "calibration")
|
|
}
|
|
),
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot",
|
|
append_attention,
|
|
),
|
|
patch(
|
|
"app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap",
|
|
append_gap,
|
|
),
|
|
):
|
|
result = await produce_supervision_cycle(conn, cohort_id="cohort-a")
|
|
|
|
self.assertEqual(result["derived_signal_count"], 1)
|
|
self.assertFalse(result["raw_transcript_included"])
|
|
self.assertFalse(result["clinical_claim_allowed"])
|
|
append_attention.assert_awaited_once()
|
|
append_gap.assert_awaited_once()
|