"""평가 라우트 — 교수자/관리자용 평가 조회 + 재평가 트리거 (Features: evaluator). services/evaluator.py 의 2-loop 평가(fast/deep)를 교수자(TEACHER)·관리자(ADMIN)에게 노출한다. 학습자(LEARNER)에겐 평가 결과가 직접 노출되지 않는다(설계서 §4.1 2-레이어: RBAC × AIView). 이 라우터는 RBAC(레이어2)만 강제한다 — require_role(TEACHER, ADMIN). 평가 AI 는 전부 봐도 되므로 (레이어1 AIView.EVALUATOR) CCD/정답 누설 걱정은 client AI 쪽 책임이고 여기선 무관. 엔드포인트: GET /eval/health — 헬스(소유 트랙 전환 확인) POST /eval/sessions/{id}/turn — 단일 턴 fast-loop 재평가 트리거 POST /eval/sessions/{id}/reevaluate — 회기 deep-loop 재평가 트리거(전체 축어록) GET /eval/sessions/{id}/evaluation — 회기 평가 조회(분포 + 최근 deep 결과) 평가 결과는 session_persistence 의 DB-backed evaluation 저장소를 사용한다. DB 미가용 시 in-proc cache/session fallback 은 local dev 에서만 허용한다. """ from __future__ import annotations import logging from typing import Annotated, Any, Optional from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, Field from .. import session_persistence from ..deps import Principal, Role, require_role from ..engine_client import EngineError, engine_client from ..runtime_policy import runtime_fallback_allowed from ..session_evaluation_input import enriched_masked_turns from ..session_evaluation_timeout import ( session_evaluation_transport_timeout_seconds, ) from ..session_read_model import ( ReviewEvaluationFailure, StageLabel, stage_label_or_none, teacher_evaluation_failure, ) from ..services import evaluator from ..services.evaluator import SessionEvaluation, TurnEvaluation from ..store import InProcSession from ..store import store router = APIRouter(prefix="/eval", tags=["eval"]) logger = logging.getLogger(__name__) # 교수자/관리자만 평가 조회·트리거 (학습자 비노출) TeacherOrAdmin = Annotated[Principal, Depends(require_role(Role.TEACHER, Role.ADMIN))] # ── 요청/응답 모델 ────────────────────────────────────── class ReevaluateRequest(BaseModel): scope: str = Field("session_end", description="'session_end' | 'stage_transition'") class TurnReevaluateRequest(BaseModel): turn_seq: int = Field(..., ge=0, description="재평가할 상담자 발화의 turn_seq") class EvaluationSummary(BaseModel): """회기 평가 조회 응답(분포 + deep 결과 합본, provider 오류 원문 제외).""" session_id: str stage: StageLabel | None = None status: str | None = None error: str | None = None failure: ReviewEvaluationFailure | None = None durable: bool = False deep: Optional[dict[str, Any]] = None distribution: dict[str, Any] = Field(default_factory=dict) class TurnEvaluationResponse(TurnEvaluation): stage: StageLabel class SessionEvaluationResponse(SessionEvaluation): stage: StageLabel async def _load_session_or_404(session_id: str, principal: Principal) -> InProcSession: sess = await session_persistence.load_session(session_id, principal, allow_ended=True) if sess is not None: store.put(sess) elif runtime_fallback_allowed(): sess = store.get(session_id) if sess is None: raise HTTPException(status.HTTP_404_NOT_FOUND, detail="session not found") return sess def _theory_mode_of(sess) -> Optional[str]: # 회기에서 학습자가 명시 선택한 이론 모드가 최우선이다. sess_theory = str(getattr(sess, "theory_mode", "") or "").strip() if sess_theory: return sess_theory tt = getattr(sess.persona, "theory_target", None) if isinstance(tt, (list, tuple)) and tt: return ", ".join(str(x) for x in tt) return None def _summary_stage(value: object) -> StageLabel | None: return stage_label_or_none(value) @router.get("/health") async def eval_health() -> dict[str, str]: """평가 라우터 헬스 — Features:evaluator 로 전환됨.""" return {"status": "ok", "owner": "features:evaluator", "loops": "fast,deep"} def _session_evaluation_error_status(error: str) -> int: if error.startswith("engine_error"): return status.HTTP_503_SERVICE_UNAVAILABLE return status.HTTP_502_BAD_GATEWAY def _safe_session_evaluation_retry_detail() -> str: """재시도 HTTP 응답에서는 provider 예외 원문을 내보내지 않는다. 원인은 durable evaluation record에 서버 전용으로 보존하고, 교수자 화면은 review의 안전 분류(evaluationFailure)로 다음 행동만 안내한다. """ return "AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요." # ════════════════════════════════════════════════════════════════════════════ # 회기 deep-loop 재평가 트리거 (교수자/관리자) # ════════════════════════════════════════════════════════════════════════════ @router.post("/sessions/{session_id}/reevaluate", response_model=SessionEvaluationResponse) async def reevaluate_session( session_id: str, body: ReevaluateRequest, principal: TeacherOrAdmin, ) -> SessionEvaluation: """회기 전체 deep-loop 재평가(슈퍼바이저 rationale/critique + 개선점 + 대안발화). 저장된 마스킹 축어록을 evaluator.evaluate_session 으로 평가한다. 엔진 장애는 503으로 변환한다. provider 예외 원문은 durable 기록에만 남기고 HTTP에는 안전한 안내만 반환한다. """ sess = await _load_session_or_404(session_id, principal) counselor_identity = getattr(sess, "learner_label", None) client_identity = getattr(sess.persona, "display_name", None) enriched = enriched_masked_turns( sess.masked_turns(), counselor_identity=counselor_identity, client_identity=client_identity, ) # 누적 기법 코드 — DB 미가용이라 fast 결과가 없으면 빈 분포(deep LLM 정성 평가는 그대로 유효). technique_codes: list[str] = [] try: result = await evaluator.evaluate_session( session_id=session_id, stage=sess.state.stage.value, masked_turns=enriched, engine=engine_client, technique_codes=technique_codes, theory_mode=_theory_mode_of(sess), scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end", audit_hook=session_persistence.record_llm_call_audit, timeout=session_evaluation_transport_timeout_seconds(), ) except EngineError as e: detail = f"engine unavailable: {e}" write = session_persistence.SessionEvaluationWrite.from_error( session_id=session_id, learner_id=sess.learner_id, scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end", stage=sess.state.stage.value, error=detail, counselor_identity=counselor_identity, client_identity=client_identity, ) saved = await session_persistence.save_session_evaluation(write) if not saved: logger.error( "session evaluation retry error record did not reach durable store: session_id=%s error=%s", session_id, write.error, ) raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, detail=_safe_session_evaluation_retry_detail(), ) write = session_persistence.SessionEvaluationWrite.from_result( session_id=session_id, learner_id=sess.learner_id, result=result, counselor_identity=counselor_identity, client_identity=client_identity, ) saved = await session_persistence.save_session_evaluation(write) if not saved: detail = "session evaluation retry result was generated but could not be saved" logger.error("%s: session_id=%s status=%s", detail, session_id, write.status) raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail) if result.error: raise HTTPException( _session_evaluation_error_status(result.error), detail=_safe_session_evaluation_retry_detail(), ) return result # ════════════════════════════════════════════════════════════════════════════ # 단일 턴 fast-loop 재평가 트리거 (교수자/관리자) # ════════════════════════════════════════════════════════════════════════════ @router.post("/sessions/{session_id}/turn", response_model=TurnEvaluationResponse) async def reevaluate_turn( session_id: str, body: TurnReevaluateRequest, principal: TeacherOrAdmin, ) -> TurnEvaluation: """단일 상담자 발화 fast-loop 재평가(기법/내담자상태/적절성/의도이탈). 저장된 축어록에서 해당 turn_seq 상담자 발화 + 직후 내담자 응답을 재구성해 경량 TurnContext 로 evaluator.evaluate_turn 을 호출한다. """ sess = await _load_session_or_404(session_id, principal) # 대상 상담자 발화 + 직후 내담자 응답 찾기 target_idx: Optional[int] = None for i, tr in enumerate(sess.turns): if tr.speaker == "counselor" and tr.turn_seq == body.turn_seq: target_idx = i break if target_idx is None: raise HTTPException( status.HTTP_404_NOT_FOUND, detail=f"counselor turn_seq {body.turn_seq} not found" ) learner = sess.turns[target_idx] client_reply = "" if target_idx + 1 < len(sess.turns) and sess.turns[target_idx + 1].speaker == "client": client_reply = sess.turns[target_idx + 1].text_masked # 평가용 경량 TurnContext 재구성(prepare_turn 의 결정론 산출과 동형). 엔진 호출 없음. from ..services.orchestrator import TurnContext, TurnMemory # 지연 import(소유권 경계) recent = [ {"speaker": tr.speaker, "text": tr.text_masked} for tr in sess.turns[max(0, target_idx - 4):target_idx] ] ctx = TurnContext( session_id=session_id, case_id=sess.case_id, persona=sess.persona, state_before=sess.state, learner_text_raw=learner.text, learner_text_masked=learner.text_masked, state_after=sess.state, # 조회 시점 상태(정밀 재현은 DB 스냅샷 도입 시) memory=TurnMemory(recent_turns=recent), theory_mode=_theory_mode_of(sess), ) result = await evaluator.evaluate_turn( ctx, client_reply, engine=engine_client, audit_hook=session_persistence.record_llm_call_audit, ) result_payload = result.to_hook_dict() turn_id = getattr(learner, "turn_id", None) if not turn_id: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, detail="turn evaluation retry result was generated but the target turn has no durable id", ) saved = await session_persistence.replace_turn_evaluation( turn_id=turn_id, evaluation=result_payload, ) if not saved: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, detail="turn evaluation retry result was generated but could not be saved", ) learner.evaluation = result_payload if result.error: raise HTTPException( _session_evaluation_error_status(result.error), detail=_safe_session_evaluation_retry_detail(), ) return result # ════════════════════════════════════════════════════════════════════════════ # 회기 평가 조회 (교수자/관리자) — 마지막 deep 결과 + 분포 # ════════════════════════════════════════════════════════════════════════════ @router.get("/sessions/{session_id}/evaluation", response_model=EvaluationSummary) async def get_session_evaluation( session_id: str, principal: TeacherOrAdmin, ) -> EvaluationSummary: """회기 평가 조회(읽기) — 저장된 마지막 deep 재평가 결과 + 기법 분포. 아직 평가 트리거가 없었다면 deep=None + 빈 분포. """ await _load_session_or_404(session_id, principal) record, durable = await session_persistence.load_session_evaluation(session_id, principal) if record is None: return EvaluationSummary( session_id=session_id, stage=None, status=None, error=None, durable=durable, deep=None, distribution={}, ) payload = record.get("payload") deep = dict(payload) if isinstance(payload, dict) else {} # SessionEvaluation.to_dict()에는 server-side error가 함께 저장된다. deep 객체도 # 교수자 API 경계에서는 같은 원칙으로 제거한다. deep.pop("error", None) distribution = deep.get("distribution") failure = teacher_evaluation_failure(record) return EvaluationSummary( session_id=session_id, stage=_summary_stage(record.get("stage") or deep.get("stage")), status=str(record.get("status") or "") or None, error=_safe_session_evaluation_retry_detail() if failure is not None else None, failure=failure, durable=durable, deep=deep, distribution=distribution if isinstance(distribution, dict) else {}, )