272 lines
10 KiB
Python
272 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, patch
|
|
from uuid import UUID, uuid4
|
|
|
|
from .routes import calibration_transfer, sessions
|
|
from .services import evaluator, session_learning_producer, state_machine
|
|
from .services.persona import P1
|
|
from .store import InProcSession, TurnRecord
|
|
|
|
|
|
SESSION_ID = UUID("00000000-0000-0000-0000-00000000a401")
|
|
LEARNER_ID = UUID("00000000-0000-0000-0000-00000000a402")
|
|
COUNSELOR_TURN_ID = UUID("00000000-0000-0000-0000-00000000a403")
|
|
CLIENT_TURN_ID = UUID("00000000-0000-0000-0000-00000000a404")
|
|
|
|
|
|
class FakeProducerConnection:
|
|
def __init__(self, *, locked_history: bool = False) -> None:
|
|
self.locked_history = locked_history
|
|
self.executed: list[tuple[str, tuple[Any, ...]]] = []
|
|
self.evaluation = {
|
|
"status": "ready",
|
|
"scope": "session_end",
|
|
"learner_id": LEARNER_ID,
|
|
"payload": {
|
|
"loop": "deep",
|
|
"scope": "session_end",
|
|
"intent_deviations": [
|
|
{
|
|
"dimension": "reflection",
|
|
"expected": "정서를 반영하고 이해를 확인한다",
|
|
"actual": "바로 다음 질문으로 이동했다",
|
|
"severity": "moderate",
|
|
}
|
|
],
|
|
},
|
|
}
|
|
|
|
async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None:
|
|
if "FROM app.session_evaluation" in query:
|
|
return self.evaluation
|
|
if "FROM app.competency_graph_snapshot" in query:
|
|
return None
|
|
raise AssertionError(f"unexpected fetchrow: {query}")
|
|
|
|
async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]:
|
|
if "FROM app.turns t" in query:
|
|
return [
|
|
{
|
|
"turn_id": COUNSELOR_TURN_ID,
|
|
"turn_seq": 3,
|
|
"response_turn_id": CLIENT_TURN_ID,
|
|
"response_turn_seq": 4,
|
|
"intent_deviation": {
|
|
"dimension": "공감적 반영",
|
|
"expected": "정서를 반영하고 이해를 확인한다",
|
|
"actual": "바로 다음 질문으로 이동했다",
|
|
"severity": "moderate",
|
|
},
|
|
}
|
|
]
|
|
if "FROM app.calibration_prediction_history h" in query:
|
|
if not self.locked_history:
|
|
return []
|
|
return [
|
|
{
|
|
"history_id": UUID("00000000-0000-0000-0000-00000000a405"),
|
|
"competency_id": "competency.empathic_reflection",
|
|
"locked_sequence": 2,
|
|
}
|
|
]
|
|
raise AssertionError(f"unexpected fetch: {query}")
|
|
|
|
async def execute(self, query: str, *args: Any) -> str:
|
|
self.executed.append((query, args))
|
|
return "INSERT 0 1"
|
|
|
|
|
|
class SessionLearningProducerTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_g4_uses_durable_turns_and_replays_stable_submission(self) -> None:
|
|
conn = FakeProducerConnection()
|
|
append = AsyncMock(
|
|
side_effect=[
|
|
{"submission_id": uuid4(), "idempotent_replay": False},
|
|
{"submission_id": uuid4(), "idempotent_replay": True},
|
|
]
|
|
)
|
|
with patch.object(
|
|
session_learning_producer.deliberate_practice_store,
|
|
"append_prescription_submission",
|
|
append,
|
|
):
|
|
first = await session_learning_producer._produce_g4(
|
|
conn, session_id=SESSION_ID
|
|
)
|
|
second = await session_learning_producer._produce_g4(
|
|
conn, session_id=SESSION_ID
|
|
)
|
|
|
|
self.assertEqual(first["status"], "ready")
|
|
self.assertEqual(second["status"], "ready")
|
|
self.assertEqual(append.await_count, 2)
|
|
first_call = append.await_args_list[0].kwargs
|
|
second_call = append.await_args_list[1].kwargs
|
|
self.assertEqual(first_call["submission_id"], second_call["submission_id"])
|
|
self.assertEqual(first_call["coaching_cards"], second_call["coaching_cards"])
|
|
self.assertEqual(
|
|
tuple(first_call["evidence_turn_ids"]),
|
|
(COUNSELOR_TURN_ID, CLIENT_TURN_ID),
|
|
)
|
|
card = first_call["coaching_cards"][0]
|
|
self.assertEqual(
|
|
tuple(item.ref_id for item in card.evidence_refs),
|
|
(str(COUNSELOR_TURN_ID), str(CLIENT_TURN_ID)),
|
|
)
|
|
graph = first_call["graph"]
|
|
self.assertEqual(len(graph.states), 4)
|
|
self.assertTrue(all(state.band == "unassessed" for state in graph.states))
|
|
self.assertTrue(all(state.attempt_count == 0 for state in graph.states))
|
|
self.assertTrue(
|
|
all(state.unseen_transfer_demonstrations == 0 for state in graph.states)
|
|
)
|
|
|
|
async def test_g5_does_not_reveal_before_prediction_lock(self) -> None:
|
|
conn = FakeProducerConnection(locked_history=False)
|
|
append = AsyncMock()
|
|
with patch.object(
|
|
session_learning_producer.calibration_transfer_store,
|
|
"append_performance_observation",
|
|
append,
|
|
):
|
|
result = await session_learning_producer._produce_g5(
|
|
conn, session_id=SESSION_ID
|
|
)
|
|
|
|
self.assertEqual(
|
|
result, {"status": "skipped", "reason": "locked_prediction_missing"}
|
|
)
|
|
append.assert_not_awaited()
|
|
self.assertFalse(
|
|
any("INSERT INTO audit.model_run" in query for query, _ in conn.executed)
|
|
)
|
|
|
|
async def test_g5_locked_history_gets_failed_observation_with_real_provenance(
|
|
self,
|
|
) -> None:
|
|
conn = FakeProducerConnection(locked_history=True)
|
|
append = AsyncMock(
|
|
return_value={"observation_id": uuid4(), "idempotent_replay": False}
|
|
)
|
|
with patch.object(
|
|
session_learning_producer.calibration_transfer_store,
|
|
"append_performance_observation",
|
|
append,
|
|
):
|
|
result = await session_learning_producer._produce_g5(
|
|
conn, session_id=SESSION_ID
|
|
)
|
|
|
|
self.assertEqual(result["status"], "ready")
|
|
self.assertTrue(
|
|
any("INSERT INTO audit.model_run" in query for query, _ in conn.executed)
|
|
)
|
|
kwargs = append.await_args.kwargs
|
|
self.assertEqual(kwargs["status"], "failed")
|
|
self.assertEqual(kwargs["source_kind"], "model_inferred")
|
|
self.assertEqual(kwargs["perspective"], "independent_observer")
|
|
self.assertIsInstance(kwargs["model_run_id"], UUID)
|
|
self.assertEqual(
|
|
tuple(kwargs["evidence_turn_ids"]),
|
|
(COUNSELOR_TURN_ID, CLIENT_TURN_ID),
|
|
)
|
|
self.assertEqual(kwargs["revealed_sequence"], 3)
|
|
self.assertNotIn("master", repr(kwargs).lower())
|
|
self.assertNotIn("transfer_verified", repr(kwargs))
|
|
|
|
async def test_ready_evaluation_is_not_rolled_back_when_producer_fails(
|
|
self,
|
|
) -> None:
|
|
state = state_machine.init_state(params=P1.openness_params())
|
|
sess = InProcSession(
|
|
session_id=str(SESSION_ID),
|
|
case_id=str(uuid4()),
|
|
learner_id=str(LEARNER_ID),
|
|
persona_code=P1.code,
|
|
theory_mode="humanistic",
|
|
persona=P1,
|
|
state=state,
|
|
turns=[
|
|
TurnRecord(
|
|
turn_seq=1,
|
|
speaker="counselor",
|
|
stage=state.stage.value,
|
|
text="상담자 발화",
|
|
text_masked="상담자 발화",
|
|
)
|
|
],
|
|
ended=True,
|
|
)
|
|
ready = evaluator.SessionEvaluation(
|
|
session_id=str(SESSION_ID),
|
|
stage=state.stage.value,
|
|
scope="session_end",
|
|
improvements=["정서를 반영한 뒤 이해를 확인한다."],
|
|
)
|
|
save = AsyncMock(return_value=True)
|
|
notify = AsyncMock()
|
|
with (
|
|
patch.object(
|
|
sessions.evaluator, "evaluate_session", AsyncMock(return_value=ready)
|
|
),
|
|
patch.object(sessions.session_evaluation_repository, "save_session_evaluation", save),
|
|
patch.object(
|
|
sessions.session_learning_producer,
|
|
"produce_session_learning_artifacts",
|
|
AsyncMock(side_effect=RuntimeError("producer offline")),
|
|
),
|
|
patch.object(
|
|
sessions, "_enqueue_session_review_ready_notification", notify
|
|
),
|
|
):
|
|
await sessions._generate_and_save_session_evaluation(sess)
|
|
|
|
save.assert_awaited_once()
|
|
self.assertEqual(save.await_args.args[0].status, "ready")
|
|
notify.assert_awaited_once_with(str(SESSION_ID))
|
|
|
|
async def test_prediction_lock_invokes_same_session_worker(self) -> None:
|
|
history_id = uuid4()
|
|
body = calibration_transfer.PredictionLockRequest(
|
|
submission_id=uuid4(),
|
|
lock_id=uuid4(),
|
|
prediction_revision_id=uuid4(),
|
|
locked_sequence=2,
|
|
)
|
|
payload = {
|
|
"submission_id": body.submission_id,
|
|
"history_id": history_id,
|
|
"lock_id": body.lock_id,
|
|
"idempotent_replay": False,
|
|
}
|
|
principal = calibration_transfer.Principal(
|
|
user_id=str(LEARNER_ID),
|
|
role=calibration_transfer.Role.LEARNER,
|
|
cohort_ids=["g5-test"],
|
|
)
|
|
worker = AsyncMock(return_value={"g5": {"status": "ready"}})
|
|
with (
|
|
patch.object(
|
|
calibration_transfer.calibration_transfer_store,
|
|
"append_prediction_lock",
|
|
AsyncMock(return_value=payload),
|
|
),
|
|
patch.object(
|
|
calibration_transfer.session_learning_producer,
|
|
"produce_locked_prediction_history",
|
|
worker,
|
|
),
|
|
):
|
|
response = await calibration_transfer.lock_prediction_history(
|
|
history_id, body, principal
|
|
)
|
|
|
|
self.assertEqual(response.history_id, history_id)
|
|
worker.assert_awaited_once_with(history_id)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|