421 lines
16 KiB
Python
421 lines
16 KiB
Python
"""평가 재시도 라우트의 실패 노출 테스트."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from .config import settings
|
|
from .deps import Principal, Role
|
|
from .services import evaluator
|
|
|
|
eval_routes = importlib.import_module("app.routes.eval")
|
|
|
|
|
|
class ReevaluateSessionRouteTest(unittest.IsolatedAsyncioTestCase):
|
|
async def test_reevaluate_session_saves_result_error_then_raises(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub()
|
|
result = evaluator.SessionEvaluation(
|
|
session_id=session_id,
|
|
stage="정리",
|
|
scope="session_end",
|
|
turns_evaluated=1,
|
|
error="no_structured_output",
|
|
)
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(eval_routes.evaluator, "evaluate_session", AsyncMock(return_value=result)),
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"save_session_evaluation",
|
|
AsyncMock(return_value=True),
|
|
) as save_evaluation,
|
|
):
|
|
with self.assertRaises(HTTPException) as raised:
|
|
await eval_routes.reevaluate_session(
|
|
session_id,
|
|
eval_routes.ReevaluateRequest(scope="session_end"),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(raised.exception.status_code, 502)
|
|
self.assertEqual(
|
|
raised.exception.detail,
|
|
"AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요.",
|
|
)
|
|
self.assertNotIn("no_structured_output", raised.exception.detail)
|
|
saved_write = save_evaluation.await_args.args[0]
|
|
self.assertEqual(saved_write.status, "error")
|
|
self.assertEqual(saved_write.error, "no_structured_output")
|
|
|
|
async def test_reevaluate_session_raises_when_result_cannot_be_saved(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub()
|
|
result = evaluator.SessionEvaluation(
|
|
session_id=session_id,
|
|
stage="정리",
|
|
scope="session_end",
|
|
turns_evaluated=1,
|
|
)
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(eval_routes.evaluator, "evaluate_session", AsyncMock(return_value=result)),
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"save_session_evaluation",
|
|
AsyncMock(return_value=False),
|
|
),
|
|
):
|
|
with self.assertRaises(HTTPException) as raised:
|
|
await eval_routes.reevaluate_session(
|
|
session_id,
|
|
eval_routes.ReevaluateRequest(scope="session_end"),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(raised.exception.status_code, 503)
|
|
self.assertIn("could not be saved", raised.exception.detail)
|
|
|
|
async def test_reevaluate_session_redacts_engine_exception_from_http_response(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub()
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_session",
|
|
AsyncMock(side_effect=eval_routes.EngineError("provider body: [NAME]")),
|
|
),
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"save_session_evaluation",
|
|
AsyncMock(return_value=True),
|
|
) as save_evaluation,
|
|
):
|
|
with self.assertRaises(HTTPException) as raised:
|
|
await eval_routes.reevaluate_session(
|
|
session_id,
|
|
eval_routes.ReevaluateRequest(scope="session_end"),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(raised.exception.status_code, 503)
|
|
self.assertEqual(
|
|
raised.exception.detail,
|
|
"AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요.",
|
|
)
|
|
self.assertNotIn("[NAME]", raised.exception.detail)
|
|
saved_write = save_evaluation.await_args.args[0]
|
|
self.assertIn("provider body: [NAME]", saved_write.error)
|
|
|
|
async def test_reevaluate_session_prefers_session_theory_mode(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub(theory_mode="cbt", theory_target=["humanistic"])
|
|
result = evaluator.SessionEvaluation(
|
|
session_id=session_id,
|
|
stage="정리",
|
|
scope="session_end",
|
|
turns_evaluated=1,
|
|
)
|
|
|
|
with (
|
|
patch.object(settings, "session_evaluation_timeout", 300.0),
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_session",
|
|
AsyncMock(return_value=result),
|
|
) as evaluate_session,
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"save_session_evaluation",
|
|
AsyncMock(return_value=True),
|
|
),
|
|
):
|
|
await eval_routes.reevaluate_session(
|
|
session_id,
|
|
eval_routes.ReevaluateRequest(scope="session_end"),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(evaluate_session.await_args.kwargs["theory_mode"], "cbt")
|
|
self.assertEqual(evaluate_session.await_args.kwargs["timeout"], 330.0)
|
|
|
|
async def test_reevaluate_session_uses_one_based_masked_turn_seq(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub()
|
|
result = evaluator.SessionEvaluation(
|
|
session_id=session_id,
|
|
stage="정리",
|
|
scope="session_end",
|
|
turns_evaluated=2,
|
|
)
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_session",
|
|
AsyncMock(return_value=result),
|
|
) as evaluate_session,
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"save_session_evaluation",
|
|
AsyncMock(return_value=True),
|
|
),
|
|
):
|
|
await eval_routes.reevaluate_session(
|
|
session_id,
|
|
eval_routes.ReevaluateRequest(scope="session_end"),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
masked_turns = evaluate_session.await_args.kwargs["masked_turns"]
|
|
self.assertEqual([turn["seq"] for turn in masked_turns], [1, 2])
|
|
|
|
async def test_reevaluate_turn_preserves_session_theory_mode(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub(theory_mode="cbt", theory_target=["humanistic"], with_turns=True)
|
|
result = evaluator.TurnEvaluation(turn_seq=1, stage="정리", theory_mode="cbt")
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_turn",
|
|
AsyncMock(return_value=result),
|
|
) as evaluate_turn,
|
|
patch.object(
|
|
eval_routes.session_persistence,
|
|
"replace_turn_evaluation",
|
|
AsyncMock(return_value=True),
|
|
) as replace_turn_evaluation,
|
|
):
|
|
await eval_routes.reevaluate_turn(
|
|
session_id,
|
|
eval_routes.TurnReevaluateRequest(turn_seq=1),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
ctx = evaluate_turn.await_args.args[0]
|
|
self.assertEqual(ctx.theory_mode, "cbt")
|
|
replace_turn_evaluation.assert_awaited_once()
|
|
self.assertEqual(
|
|
replace_turn_evaluation.await_args.kwargs["turn_id"],
|
|
"00000000-0000-0000-0000-00000000a001",
|
|
)
|
|
|
|
async def test_reevaluate_turn_persists_result_for_review_hydration(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub(theory_mode="cbt", with_turns=True)
|
|
result = evaluator.TurnEvaluation(
|
|
turn_seq=1,
|
|
stage="정리",
|
|
theory_mode="cbt",
|
|
appropriateness="pos",
|
|
rapport_signal=0.4,
|
|
techniques=[
|
|
evaluator.TechniqueTag(
|
|
code="empathy",
|
|
label_ko="공감",
|
|
category="relational",
|
|
rationale="감정을 반영했다.",
|
|
)
|
|
],
|
|
)
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_turn",
|
|
AsyncMock(return_value=result),
|
|
),
|
|
patch.object(
|
|
eval_routes.session_persistence,
|
|
"replace_turn_evaluation",
|
|
AsyncMock(return_value=True),
|
|
) as replace_turn_evaluation,
|
|
):
|
|
response = await eval_routes.reevaluate_turn(
|
|
session_id,
|
|
eval_routes.TurnReevaluateRequest(turn_seq=1),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(response.appropriateness, "pos")
|
|
saved_payload = replace_turn_evaluation.await_args.kwargs["evaluation"]
|
|
self.assertEqual(saved_payload["turn_seq"], 1)
|
|
self.assertEqual(saved_payload["theory_mode"], "cbt")
|
|
self.assertEqual(saved_payload["rapport_signal"], 0.4)
|
|
self.assertEqual(saved_payload["techniques"][0]["label_ko"], "공감")
|
|
self.assertEqual(sess.turns[0].evaluation, saved_payload)
|
|
|
|
async def test_reevaluate_turn_raises_when_result_cannot_be_saved(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub(with_turns=True)
|
|
result = evaluator.TurnEvaluation(turn_seq=1, stage="정리")
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_turn",
|
|
AsyncMock(return_value=result),
|
|
),
|
|
patch.object(
|
|
eval_routes.session_persistence,
|
|
"replace_turn_evaluation",
|
|
AsyncMock(return_value=False),
|
|
),
|
|
):
|
|
with self.assertRaises(HTTPException) as raised:
|
|
await eval_routes.reevaluate_turn(
|
|
session_id,
|
|
eval_routes.TurnReevaluateRequest(turn_seq=1),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(raised.exception.status_code, 503)
|
|
self.assertIn("could not be saved", raised.exception.detail)
|
|
|
|
async def test_reevaluate_turn_saves_error_marker_then_raises(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub(with_turns=True)
|
|
result = evaluator.TurnEvaluation(
|
|
turn_seq=1,
|
|
stage="정리",
|
|
error="engine_error",
|
|
)
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.evaluator,
|
|
"evaluate_turn",
|
|
AsyncMock(return_value=result),
|
|
),
|
|
patch.object(
|
|
eval_routes.session_persistence,
|
|
"replace_turn_evaluation",
|
|
AsyncMock(return_value=True),
|
|
) as replace_turn_evaluation,
|
|
):
|
|
with self.assertRaises(HTTPException) as raised:
|
|
await eval_routes.reevaluate_turn(
|
|
session_id,
|
|
eval_routes.TurnReevaluateRequest(turn_seq=1),
|
|
_teacher_principal(),
|
|
)
|
|
|
|
self.assertEqual(raised.exception.status_code, 503)
|
|
self.assertEqual(
|
|
raised.exception.detail,
|
|
"AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요.",
|
|
)
|
|
saved_payload = replace_turn_evaluation.await_args.kwargs["evaluation"]
|
|
self.assertEqual(saved_payload["error"], "engine_error")
|
|
|
|
async def test_get_session_evaluation_exposes_status_error_and_durable_flag(self) -> None:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
sess = _session_stub()
|
|
record = {
|
|
"status": "error",
|
|
"source": "engine",
|
|
"scope": "session_end",
|
|
"stage": "정리",
|
|
"payload": {
|
|
"error": "session evaluation timeout after 45s",
|
|
"distribution": {"total": 0},
|
|
},
|
|
"error": "session evaluation timeout after 45s",
|
|
"updated_at": "2026-07-01T00:00:00Z",
|
|
}
|
|
|
|
with (
|
|
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
|
patch.object(
|
|
eval_routes.session_evaluation_repository,
|
|
"load_session_evaluation",
|
|
AsyncMock(return_value=(record, True)),
|
|
),
|
|
):
|
|
response = await eval_routes.get_session_evaluation(session_id, _teacher_principal())
|
|
|
|
self.assertEqual(response.status, "error")
|
|
self.assertEqual(
|
|
response.error,
|
|
"AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요.",
|
|
)
|
|
self.assertIsNotNone(response.failure)
|
|
assert response.failure is not None
|
|
self.assertEqual(response.failure.code, "timeout")
|
|
self.assertNotIn("session evaluation timeout after 45s", response.model_dump_json())
|
|
self.assertTrue(response.durable)
|
|
self.assertEqual(response.deep, {"distribution": {"total": 0}})
|
|
|
|
|
|
def _session_stub(
|
|
*,
|
|
theory_mode: str = "humanistic",
|
|
theory_target: list[str] | None = None,
|
|
with_turns: bool = False,
|
|
):
|
|
selected_theory_mode = theory_mode
|
|
selected_theory_target = theory_target or []
|
|
|
|
class SessionStub:
|
|
session_id = "00000000-0000-0000-0000-00000000e222"
|
|
case_id = "00000000-0000-0000-0000-00000000c333"
|
|
learner_id = "00000000-0000-0000-0000-000000000111"
|
|
theory_mode = selected_theory_mode
|
|
persona = SimpleNamespace(theory_target=selected_theory_target)
|
|
state = SimpleNamespace(stage=SimpleNamespace(value="정리"))
|
|
turns = (
|
|
[
|
|
SimpleNamespace(
|
|
turn_id="00000000-0000-0000-0000-00000000a001",
|
|
turn_seq=1,
|
|
speaker="counselor",
|
|
text="상담자 발화",
|
|
text_masked="상담자 발화",
|
|
),
|
|
SimpleNamespace(
|
|
turn_id="00000000-0000-0000-0000-00000000a002",
|
|
turn_seq=2,
|
|
speaker="client",
|
|
text="내담자 응답",
|
|
text_masked="내담자 응답",
|
|
),
|
|
]
|
|
if with_turns
|
|
else []
|
|
)
|
|
|
|
def masked_turns(self):
|
|
return [
|
|
{"speaker": "counselor", "text_masked": "상담자 발화"},
|
|
{"speaker": "client", "text_masked": "내담자 응답"},
|
|
]
|
|
|
|
return SessionStub()
|
|
|
|
|
|
def _teacher_principal() -> Principal:
|
|
return Principal(
|
|
user_id="00000000-0000-0000-0000-000000000222",
|
|
role=Role.TEACHER,
|
|
cohort_ids=[],
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|