G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
484
apps/api/app/test_supervision_research_producer.py
Normal file
484
apps/api/app/test_supervision_research_producer.py
Normal file
|
|
@ -0,0 +1,484 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from unittest import IsolatedAsyncioTestCase, TestCase
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from uuid import UUID
|
||||
|
||||
from .services.supervision_research_producer import (
|
||||
_load_attention_rows,
|
||||
derive_attention_signals,
|
||||
derive_manifest_input,
|
||||
produce_all_active_cohorts_once,
|
||||
produce_supervision_cycle,
|
||||
)
|
||||
from .services.supervision_research import build_attention_queue
|
||||
|
||||
|
||||
LEARNER_A = UUID("00000000-0000-0000-0000-0000000000a1")
|
||||
LEARNER_B = UUID("00000000-0000-0000-0000-0000000000b2")
|
||||
SESSION_A = UUID("10000000-0000-0000-0000-0000000000a1")
|
||||
|
||||
|
||||
class SupervisionResearchProducerDerivationTest(TestCase):
|
||||
def test_escalated_safety_event_becomes_metadata_only_priority_signal(
|
||||
self,
|
||||
) -> None:
|
||||
safety_rows = [
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 91,
|
||||
"session_id": SESSION_A,
|
||||
"turn_id": None,
|
||||
"trigger_type": "real_crisis",
|
||||
"ko_risk_level": 5,
|
||||
"escalated": True,
|
||||
"observed_sequence": 7,
|
||||
# 호출자가 실수로 전체 row를 넘겨도 민감 원천 payload를
|
||||
# G6 신호에 복제하면 안 된다.
|
||||
"detail": {"matched": ["resident-010-1234-5678"]},
|
||||
"raw_transcript": "resident-010-1234-5678",
|
||||
},
|
||||
{
|
||||
"learner_id": LEARNER_B,
|
||||
"safety_event_id": 90,
|
||||
"session_id": UUID("10000000-0000-0000-0000-0000000000b2"),
|
||||
"turn_id": UUID("90000000-0000-0000-0000-000000000002"),
|
||||
"trigger_type": "boundary_violation",
|
||||
"ko_risk_level": 3,
|
||||
"escalated": True,
|
||||
"observed_sequence": 2,
|
||||
},
|
||||
]
|
||||
|
||||
first = derive_attention_signals(
|
||||
trajectory_rows=[],
|
||||
rupture_rows=[],
|
||||
calibration_rows=[],
|
||||
transfer_rows=[],
|
||||
practice_rows=[],
|
||||
safety_rows=safety_rows,
|
||||
)
|
||||
second = derive_attention_signals(
|
||||
trajectory_rows=[],
|
||||
rupture_rows=[],
|
||||
calibration_rows=[],
|
||||
transfer_rows=[],
|
||||
practice_rows=[],
|
||||
safety_rows=list(reversed(safety_rows)),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item.signal.model_dump(mode="json") for item in first],
|
||||
[item.signal.model_dump(mode="json") for item in second],
|
||||
)
|
||||
self.assertEqual(
|
||||
{item.signal.signal_type for item in first}, {"safety_boundary"}
|
||||
)
|
||||
self.assertTrue(all(item.signal.severity == "high" for item in first))
|
||||
self.assertTrue(all(item.signal.uncertainty == 1.0 for item in first))
|
||||
self.assertTrue(
|
||||
all(item.competency_id == "competency.safety_boundary" for item in first)
|
||||
)
|
||||
pointers = [item.signal.evidence[0] for item in first]
|
||||
self.assertTrue(all(pointer.ledger == "safety_event" for pointer in pointers))
|
||||
self.assertEqual({pointer.event_id for pointer in pointers}, {"90", "91"})
|
||||
serialized = str([item.signal.model_dump(mode="json") for item in first])
|
||||
self.assertNotIn("resident-010-1234-5678", serialized)
|
||||
self.assertNotIn("real_crisis", serialized)
|
||||
self.assertNotIn("boundary_violation", serialized)
|
||||
self.assertNotIn("transcript", serialized.lower())
|
||||
|
||||
def test_incomplete_or_non_escalated_safety_rows_fail_closed(self) -> None:
|
||||
derived = derive_attention_signals(
|
||||
trajectory_rows=[],
|
||||
rupture_rows=[],
|
||||
calibration_rows=[],
|
||||
transfer_rows=[],
|
||||
practice_rows=[],
|
||||
safety_rows=[
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 1,
|
||||
"session_id": SESSION_A,
|
||||
"trigger_type": "real_crisis",
|
||||
"escalated": False,
|
||||
"observed_sequence": 1,
|
||||
},
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 2,
|
||||
"session_id": None,
|
||||
"trigger_type": "real_crisis",
|
||||
"escalated": True,
|
||||
"observed_sequence": 2,
|
||||
},
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 3,
|
||||
"session_id": SESSION_A,
|
||||
"trigger_type": " ",
|
||||
"escalated": True,
|
||||
"observed_sequence": 3,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(derived, ())
|
||||
|
||||
def test_derived_safety_event_outranks_other_active_signals(self) -> None:
|
||||
distinct_learner = UUID("b0000000-0000-0000-0000-0000000000b2")
|
||||
derived = derive_attention_signals(
|
||||
trajectory_rows=[
|
||||
{
|
||||
"learner_id": distinct_learner,
|
||||
"revision_id": UUID("20000000-0000-0000-0000-000000000001"),
|
||||
"anchor_session_id": SESSION_A,
|
||||
"revision_no": 1,
|
||||
"assessment": {
|
||||
"sessions": [
|
||||
{
|
||||
"session_no": 1,
|
||||
"status": "deteriorating",
|
||||
"axes": [{"uncertainty": 0.1}],
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
],
|
||||
rupture_rows=[],
|
||||
calibration_rows=[],
|
||||
transfer_rows=[],
|
||||
practice_rows=[],
|
||||
safety_rows=[
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 91,
|
||||
"session_id": SESSION_A,
|
||||
"trigger_type": "real_crisis",
|
||||
"escalated": True,
|
||||
"observed_sequence": 9,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
queue = build_attention_queue(item.signal for item in derived)
|
||||
|
||||
self.assertEqual(queue[0].primary_signal, "safety_boundary")
|
||||
self.assertEqual(queue[1].primary_signal, "deterioration")
|
||||
|
||||
def test_derives_only_observed_active_states_without_transcript(self) -> None:
|
||||
derived = derive_attention_signals(
|
||||
trajectory_rows=[
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"revision_id": UUID("20000000-0000-0000-0000-000000000001"),
|
||||
"anchor_session_id": SESSION_A,
|
||||
"revision_no": 3,
|
||||
"assessment": {
|
||||
"sessions": [
|
||||
{
|
||||
"session_no": 3,
|
||||
"status": "deteriorating",
|
||||
"axes": [{"uncertainty": 0.2}, {"uncertainty": 0.4}],
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
"learner_id": LEARNER_B,
|
||||
"revision_id": UUID("20000000-0000-0000-0000-000000000002"),
|
||||
"anchor_session_id": SESSION_A,
|
||||
"revision_no": 2,
|
||||
"assessment": {
|
||||
"sessions": [{"session_no": 2, "status": "on_track"}]
|
||||
},
|
||||
},
|
||||
],
|
||||
rupture_rows=[
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"observation_id": UUID("30000000-0000-0000-0000-000000000001"),
|
||||
"session_id": SESSION_A,
|
||||
"sequence_no": 4,
|
||||
"to_state": "partial",
|
||||
"uncertainty": 0.3,
|
||||
},
|
||||
{
|
||||
"learner_id": LEARNER_B,
|
||||
"observation_id": UUID("30000000-0000-0000-0000-000000000002"),
|
||||
"session_id": SESSION_A,
|
||||
"sequence_no": 5,
|
||||
"to_state": "resolved",
|
||||
"uncertainty": 0.1,
|
||||
},
|
||||
],
|
||||
calibration_rows=[],
|
||||
transfer_rows=[],
|
||||
practice_rows=[],
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{item.signal.signal_type for item in derived},
|
||||
{"deterioration", "unresolved_rupture"},
|
||||
)
|
||||
self.assertTrue(all(item.signal.evidence for item in derived))
|
||||
serialized = str([item.signal.model_dump(mode="json") for item in derived])
|
||||
self.assertNotIn("transcript", serialized.lower())
|
||||
|
||||
def test_requires_repeated_overconfidence_and_retry_for_persistent_signals(
|
||||
self,
|
||||
) -> None:
|
||||
calibration_rows = [
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"competency_id": "competency.empathy",
|
||||
"assessment_snapshot_id": UUID(
|
||||
f"40000000-0000-0000-0000-00000000000{index}"
|
||||
),
|
||||
"session_id": SESSION_A,
|
||||
"snapshot_no": index,
|
||||
"assessment_payload": {
|
||||
"bias": "overconfident",
|
||||
"error_interval": {"lower": 0.1, "upper": 0.4},
|
||||
},
|
||||
}
|
||||
for index in (1, 2)
|
||||
]
|
||||
practice_rows = [
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"competency_id": "competency.reflection",
|
||||
"attempt_record_id": UUID(
|
||||
f"50000000-0000-0000-0000-00000000000{index}"
|
||||
),
|
||||
"session_id": SESSION_A,
|
||||
"sequence_no": index,
|
||||
"outcome": "needs_retry",
|
||||
"uncertainty": 0.25,
|
||||
"counterevidence": ["client_response_withdrawn"],
|
||||
}
|
||||
for index in (1, 2)
|
||||
]
|
||||
derived = derive_attention_signals(
|
||||
trajectory_rows=[],
|
||||
rupture_rows=[],
|
||||
calibration_rows=calibration_rows,
|
||||
transfer_rows=[],
|
||||
practice_rows=practice_rows,
|
||||
)
|
||||
self.assertEqual(
|
||||
{item.signal.signal_type for item in derived},
|
||||
{"persistent_overconfidence", "growth_stagnation"},
|
||||
)
|
||||
|
||||
def test_manifest_requires_all_domains_and_hashes_event_ids(self) -> None:
|
||||
rows = {
|
||||
domain: [
|
||||
{
|
||||
"event_id": UUID(f"60000000-0000-0000-0000-{index:012d}"),
|
||||
"learner_id": LEARNER_A,
|
||||
"session_id": SESSION_A,
|
||||
}
|
||||
]
|
||||
for index, domain in enumerate(
|
||||
("alliance", "rupture", "transfer", "calibration"), start=1
|
||||
)
|
||||
}
|
||||
manifest = derive_manifest_input(rows)
|
||||
assert manifest is not None
|
||||
self.assertEqual({item.domain for item in manifest.artifacts}, set(rows))
|
||||
self.assertTrue(all(item.record_count == 1 for item in manifest.artifacts))
|
||||
self.assertEqual(len(manifest.source_fingerprint), 64)
|
||||
self.assertIsNone(derive_manifest_input({"alliance": rows["alliance"]}))
|
||||
|
||||
|
||||
class SupervisionResearchProducerCycleTest(IsolatedAsyncioTestCase):
|
||||
async def test_loader_reads_safety_metadata_without_sensitive_detail(self) -> None:
|
||||
conn = AsyncMock()
|
||||
conn.fetch.side_effect = [[], [], [], [], [], []]
|
||||
|
||||
rows = await _load_attention_rows(conn, "cohort-a")
|
||||
|
||||
self.assertIn("safety_rows", rows)
|
||||
safety_queries = [
|
||||
str(call.args[0])
|
||||
for call in conn.fetch.call_args_list
|
||||
if "app.safety_events" in str(call.args[0])
|
||||
]
|
||||
self.assertEqual(len(safety_queries), 1)
|
||||
query = safety_queries[0].lower()
|
||||
self.assertIn("se.id", query)
|
||||
self.assertIn("se.session_id", query)
|
||||
self.assertIn("se.turn_id", query)
|
||||
self.assertIn("se.trigger_type", query)
|
||||
self.assertIn("se.ko_risk_level", query)
|
||||
self.assertIn("se.escalated = true", query)
|
||||
self.assertNotIn("se.detail", query)
|
||||
self.assertNotIn("transcript", query)
|
||||
self.assertNotIn("matched", query)
|
||||
|
||||
async def test_safety_attention_does_not_become_a_curriculum_gap(self) -> None:
|
||||
rows = {
|
||||
"trajectory_rows": [],
|
||||
"rupture_rows": [],
|
||||
"calibration_rows": [],
|
||||
"transfer_rows": [],
|
||||
"practice_rows": [],
|
||||
"safety_rows": [
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"safety_event_id": 91,
|
||||
"session_id": SESSION_A,
|
||||
"turn_id": None,
|
||||
"trigger_type": "real_crisis",
|
||||
"ko_risk_level": 5,
|
||||
"escalated": True,
|
||||
"observed_sequence": 1,
|
||||
}
|
||||
],
|
||||
}
|
||||
append_attention = AsyncMock(
|
||||
return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)}
|
||||
)
|
||||
append_gap = AsyncMock()
|
||||
conn = AsyncMock()
|
||||
with (
|
||||
patch(
|
||||
"app.services.supervision_research_producer._load_attention_rows",
|
||||
AsyncMock(return_value=rows),
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer._load_manifest_rows",
|
||||
AsyncMock(
|
||||
return_value={
|
||||
domain: []
|
||||
for domain in ("alliance", "rupture", "transfer", "calibration")
|
||||
}
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot",
|
||||
append_attention,
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap",
|
||||
append_gap,
|
||||
),
|
||||
):
|
||||
result = await produce_supervision_cycle(conn, cohort_id="cohort-a")
|
||||
|
||||
self.assertEqual(result["derived_signal_count"], 1)
|
||||
signal = append_attention.await_args.kwargs["signals"][0]
|
||||
self.assertEqual(signal.signal_type, "safety_boundary")
|
||||
append_gap.assert_not_awaited()
|
||||
|
||||
async def test_all_cohort_cycle_isolates_one_cohort_failure(self) -> None:
|
||||
class AcquireContext:
|
||||
def __init__(self, conn: AsyncMock) -> None:
|
||||
self.conn = conn
|
||||
|
||||
async def __aenter__(self) -> AsyncMock:
|
||||
return self.conn
|
||||
|
||||
async def __aexit__(self, *_: object) -> None:
|
||||
return None
|
||||
|
||||
list_conn = AsyncMock()
|
||||
list_conn.fetch.return_value = [{"cohort": "a"}, {"cohort": "b"}]
|
||||
cohort_a_conn = AsyncMock()
|
||||
cohort_b_conn = AsyncMock()
|
||||
comparison_conn = AsyncMock()
|
||||
cycle = AsyncMock(side_effect=[RuntimeError("broken cohort"), {"ok": True}])
|
||||
comparison = AsyncMock(side_effect=RuntimeError("broken benchmark anchor"))
|
||||
with (
|
||||
patch(
|
||||
"app.services.supervision_research_producer.db.acquire",
|
||||
side_effect=[
|
||||
AcquireContext(list_conn),
|
||||
AcquireContext(cohort_a_conn),
|
||||
AcquireContext(cohort_b_conn),
|
||||
AcquireContext(comparison_conn),
|
||||
],
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer.produce_supervision_cycle",
|
||||
cycle,
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer."
|
||||
"supervision_research_version_evaluator."
|
||||
"produce_repository_version_comparison",
|
||||
comparison,
|
||||
),
|
||||
):
|
||||
result = await produce_all_active_cohorts_once()
|
||||
|
||||
self.assertEqual(result["cohort_count"], 2)
|
||||
self.assertEqual(result["completed"], 1)
|
||||
self.assertEqual(result["failed"], 1)
|
||||
self.assertEqual(result["version_comparison_failed"], 1)
|
||||
self.assertEqual(result["version_comparison"]["status"], "error")
|
||||
self.assertEqual(cycle.await_count, 2)
|
||||
comparison.assert_awaited_once_with(comparison_conn)
|
||||
|
||||
async def test_cycle_derives_then_calls_append_only_stores(self) -> None:
|
||||
rows = {
|
||||
"trajectory_rows": [
|
||||
{
|
||||
"learner_id": LEARNER_A,
|
||||
"revision_id": UUID("70000000-0000-0000-0000-000000000001"),
|
||||
"anchor_session_id": SESSION_A,
|
||||
"revision_no": 1,
|
||||
"assessment": {
|
||||
"sessions": [
|
||||
{
|
||||
"session_no": 1,
|
||||
"status": "off_track",
|
||||
"axes": [{"uncertainty": 0.5}],
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
],
|
||||
"rupture_rows": [],
|
||||
"calibration_rows": [],
|
||||
"transfer_rows": [],
|
||||
"practice_rows": [],
|
||||
}
|
||||
append_attention = AsyncMock(
|
||||
return_value={"submission_id": UUID(int=1), "snapshot_id": UUID(int=2)}
|
||||
)
|
||||
append_gap = AsyncMock(
|
||||
return_value={"submission_id": UUID(int=3), "gap_snapshot_id": UUID(int=4)}
|
||||
)
|
||||
conn = AsyncMock()
|
||||
with (
|
||||
patch(
|
||||
"app.services.supervision_research_producer._load_attention_rows",
|
||||
AsyncMock(return_value=rows),
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer._load_manifest_rows",
|
||||
AsyncMock(
|
||||
return_value={
|
||||
domain: []
|
||||
for domain in ("alliance", "rupture", "transfer", "calibration")
|
||||
}
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer.supervision_research_store.append_attention_snapshot",
|
||||
append_attention,
|
||||
),
|
||||
patch(
|
||||
"app.services.supervision_research_producer.supervision_research_store.append_curriculum_gap",
|
||||
append_gap,
|
||||
),
|
||||
):
|
||||
result = await produce_supervision_cycle(conn, cohort_id="cohort-a")
|
||||
|
||||
self.assertEqual(result["derived_signal_count"], 1)
|
||||
self.assertFalse(result["raw_transcript_included"])
|
||||
self.assertFalse(result["clinical_claim_allowed"])
|
||||
append_attention.assert_awaited_once()
|
||||
append_gap.assert_awaited_once()
|
||||
Loading…
Add table
Add a link
Reference in a new issue