332 lines
14 KiB
Python
332 lines
14 KiB
Python
"""평가 라우트 — 교수자/관리자용 평가 조회 + 재평가 트리거 (Features: evaluator).
|
||
|
||
services/evaluator.py 의 2-loop 평가(fast/deep)를 교수자(TEACHER)·관리자(ADMIN)에게 노출한다.
|
||
학습자(LEARNER)에겐 평가 결과가 직접 노출되지 않는다(설계서 §4.1 2-레이어: RBAC × AIView).
|
||
이 라우터는 RBAC(레이어2)만 강제한다 — require_role(TEACHER, ADMIN). 평가 AI 는 전부 봐도 되므로
|
||
(레이어1 AIView.EVALUATOR) CCD/정답 누설 걱정은 client AI 쪽 책임이고 여기선 무관.
|
||
|
||
엔드포인트:
|
||
GET /eval/health — 헬스(소유 트랙 전환 확인)
|
||
POST /eval/sessions/{id}/turn — 단일 턴 fast-loop 재평가 트리거
|
||
POST /eval/sessions/{id}/reevaluate — 회기 deep-loop 재평가 트리거(전체 축어록)
|
||
GET /eval/sessions/{id}/evaluation — 회기 평가 조회(분포 + 최근 deep 결과)
|
||
|
||
평가 결과는 session_persistence 의 DB-backed evaluation 저장소를 사용한다. DB 미가용 시
|
||
in-proc cache/session fallback 은 local dev 에서만 허용한다.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from typing import Annotated, Any, Optional
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException, status
|
||
from pydantic import BaseModel, Field
|
||
|
||
from .. import session_persistence
|
||
from ..deps import Principal, Role, require_role
|
||
from ..engine_client import EngineError, engine_client
|
||
from ..runtime_policy import runtime_fallback_allowed
|
||
from ..session_evaluation_input import enriched_masked_turns
|
||
from ..session_evaluation_timeout import (
|
||
session_evaluation_transport_timeout_seconds,
|
||
)
|
||
from ..session_read_model import (
|
||
ReviewEvaluationFailure,
|
||
StageLabel,
|
||
stage_label_or_none,
|
||
teacher_evaluation_failure,
|
||
)
|
||
from ..services import evaluator
|
||
from ..services.evaluator import SessionEvaluation, TurnEvaluation
|
||
from ..store import InProcSession
|
||
from ..store import store
|
||
|
||
router = APIRouter(prefix="/eval", tags=["eval"])
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 교수자/관리자만 평가 조회·트리거 (학습자 비노출)
|
||
TeacherOrAdmin = Annotated[Principal, Depends(require_role(Role.TEACHER, Role.ADMIN))]
|
||
|
||
|
||
# ── 요청/응답 모델 ──────────────────────────────────────
|
||
class ReevaluateRequest(BaseModel):
|
||
scope: str = Field("session_end", description="'session_end' | 'stage_transition'")
|
||
|
||
|
||
class TurnReevaluateRequest(BaseModel):
|
||
turn_seq: int = Field(..., ge=0, description="재평가할 상담자 발화의 turn_seq")
|
||
|
||
|
||
class EvaluationSummary(BaseModel):
|
||
"""회기 평가 조회 응답(분포 + deep 결과 합본, provider 오류 원문 제외)."""
|
||
|
||
session_id: str
|
||
stage: StageLabel | None = None
|
||
status: str | None = None
|
||
error: str | None = None
|
||
failure: ReviewEvaluationFailure | None = None
|
||
durable: bool = False
|
||
deep: Optional[dict[str, Any]] = None
|
||
distribution: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class TurnEvaluationResponse(TurnEvaluation):
|
||
stage: StageLabel
|
||
|
||
|
||
class SessionEvaluationResponse(SessionEvaluation):
|
||
stage: StageLabel
|
||
|
||
|
||
async def _load_session_or_404(session_id: str, principal: Principal) -> InProcSession:
|
||
sess = await session_persistence.load_session(session_id, principal, allow_ended=True)
|
||
if sess is not None:
|
||
store.put(sess)
|
||
elif runtime_fallback_allowed():
|
||
sess = store.get(session_id)
|
||
if sess is None:
|
||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="session not found")
|
||
return sess
|
||
|
||
|
||
def _theory_mode_of(sess) -> Optional[str]:
|
||
# 회기에서 학습자가 명시 선택한 이론 모드가 최우선이다.
|
||
sess_theory = str(getattr(sess, "theory_mode", "") or "").strip()
|
||
if sess_theory:
|
||
return sess_theory
|
||
tt = getattr(sess.persona, "theory_target", None)
|
||
if isinstance(tt, (list, tuple)) and tt:
|
||
return ", ".join(str(x) for x in tt)
|
||
return None
|
||
|
||
|
||
def _summary_stage(value: object) -> StageLabel | None:
|
||
return stage_label_or_none(value)
|
||
|
||
|
||
@router.get("/health")
|
||
async def eval_health() -> dict[str, str]:
|
||
"""평가 라우터 헬스 — Features:evaluator 로 전환됨."""
|
||
return {"status": "ok", "owner": "features:evaluator", "loops": "fast,deep"}
|
||
|
||
|
||
def _session_evaluation_error_status(error: str) -> int:
|
||
if error.startswith("engine_error"):
|
||
return status.HTTP_503_SERVICE_UNAVAILABLE
|
||
return status.HTTP_502_BAD_GATEWAY
|
||
|
||
|
||
def _safe_session_evaluation_retry_detail() -> str:
|
||
"""재시도 HTTP 응답에서는 provider 예외 원문을 내보내지 않는다.
|
||
|
||
원인은 durable evaluation record에 서버 전용으로 보존하고, 교수자 화면은 review의
|
||
안전 분류(evaluationFailure)로 다음 행동만 안내한다.
|
||
"""
|
||
return "AI 평가를 완료하지 못했습니다. 최신 평가 상태를 확인해 주세요."
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# 회기 deep-loop 재평가 트리거 (교수자/관리자)
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
@router.post("/sessions/{session_id}/reevaluate", response_model=SessionEvaluationResponse)
|
||
async def reevaluate_session(
|
||
session_id: str,
|
||
body: ReevaluateRequest,
|
||
principal: TeacherOrAdmin,
|
||
) -> SessionEvaluation:
|
||
"""회기 전체 deep-loop 재평가(슈퍼바이저 rationale/critique + 개선점 + 대안발화).
|
||
|
||
저장된 마스킹 축어록을 evaluator.evaluate_session 으로 평가한다.
|
||
엔진 장애는 503으로 변환한다. provider 예외 원문은 durable 기록에만 남기고 HTTP에는
|
||
안전한 안내만 반환한다.
|
||
"""
|
||
sess = await _load_session_or_404(session_id, principal)
|
||
counselor_identity = getattr(sess, "learner_label", None)
|
||
client_identity = getattr(sess.persona, "display_name", None)
|
||
enriched = enriched_masked_turns(
|
||
sess.masked_turns(),
|
||
counselor_identity=counselor_identity,
|
||
client_identity=client_identity,
|
||
)
|
||
|
||
# 누적 기법 코드 — DB 미가용이라 fast 결과가 없으면 빈 분포(deep LLM 정성 평가는 그대로 유효).
|
||
technique_codes: list[str] = []
|
||
|
||
try:
|
||
result = await evaluator.evaluate_session(
|
||
session_id=session_id,
|
||
stage=sess.state.stage.value,
|
||
masked_turns=enriched,
|
||
engine=engine_client,
|
||
technique_codes=technique_codes,
|
||
theory_mode=_theory_mode_of(sess),
|
||
scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end",
|
||
audit_hook=session_persistence.record_llm_call_audit,
|
||
timeout=session_evaluation_transport_timeout_seconds(),
|
||
)
|
||
except EngineError as e:
|
||
detail = f"engine unavailable: {e}"
|
||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||
session_id=session_id,
|
||
learner_id=sess.learner_id,
|
||
scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end",
|
||
stage=sess.state.stage.value,
|
||
error=detail,
|
||
counselor_identity=counselor_identity,
|
||
client_identity=client_identity,
|
||
)
|
||
saved = await session_persistence.save_session_evaluation(write)
|
||
if not saved:
|
||
logger.error(
|
||
"session evaluation retry error record did not reach durable store: session_id=%s error=%s",
|
||
session_id,
|
||
write.error,
|
||
)
|
||
raise HTTPException(
|
||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||
detail=_safe_session_evaluation_retry_detail(),
|
||
)
|
||
|
||
write = session_persistence.SessionEvaluationWrite.from_result(
|
||
session_id=session_id,
|
||
learner_id=sess.learner_id,
|
||
result=result,
|
||
counselor_identity=counselor_identity,
|
||
client_identity=client_identity,
|
||
)
|
||
saved = await session_persistence.save_session_evaluation(write)
|
||
if not saved:
|
||
detail = "session evaluation retry result was generated but could not be saved"
|
||
logger.error("%s: session_id=%s status=%s", detail, session_id, write.status)
|
||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail)
|
||
if result.error:
|
||
raise HTTPException(
|
||
_session_evaluation_error_status(result.error),
|
||
detail=_safe_session_evaluation_retry_detail(),
|
||
)
|
||
return result
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# 단일 턴 fast-loop 재평가 트리거 (교수자/관리자)
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
@router.post("/sessions/{session_id}/turn", response_model=TurnEvaluationResponse)
|
||
async def reevaluate_turn(
|
||
session_id: str,
|
||
body: TurnReevaluateRequest,
|
||
principal: TeacherOrAdmin,
|
||
) -> TurnEvaluation:
|
||
"""단일 상담자 발화 fast-loop 재평가(기법/내담자상태/적절성/의도이탈).
|
||
|
||
저장된 축어록에서 해당 turn_seq 상담자 발화 + 직후 내담자 응답을 재구성해
|
||
경량 TurnContext 로 evaluator.evaluate_turn 을 호출한다.
|
||
"""
|
||
sess = await _load_session_or_404(session_id, principal)
|
||
|
||
# 대상 상담자 발화 + 직후 내담자 응답 찾기
|
||
target_idx: Optional[int] = None
|
||
for i, tr in enumerate(sess.turns):
|
||
if tr.speaker == "counselor" and tr.turn_seq == body.turn_seq:
|
||
target_idx = i
|
||
break
|
||
if target_idx is None:
|
||
raise HTTPException(
|
||
status.HTTP_404_NOT_FOUND, detail=f"counselor turn_seq {body.turn_seq} not found"
|
||
)
|
||
|
||
learner = sess.turns[target_idx]
|
||
client_reply = ""
|
||
if target_idx + 1 < len(sess.turns) and sess.turns[target_idx + 1].speaker == "client":
|
||
client_reply = sess.turns[target_idx + 1].text_masked
|
||
|
||
# 평가용 경량 TurnContext 재구성(prepare_turn 의 결정론 산출과 동형). 엔진 호출 없음.
|
||
from ..services.orchestrator import TurnContext, TurnMemory # 지연 import(소유권 경계)
|
||
|
||
recent = [
|
||
{"speaker": tr.speaker, "text": tr.text_masked} for tr in sess.turns[max(0, target_idx - 4):target_idx]
|
||
]
|
||
ctx = TurnContext(
|
||
session_id=session_id,
|
||
case_id=sess.case_id,
|
||
persona=sess.persona,
|
||
state_before=sess.state,
|
||
learner_text_raw=learner.text,
|
||
learner_text_masked=learner.text_masked,
|
||
state_after=sess.state, # 조회 시점 상태(정밀 재현은 DB 스냅샷 도입 시)
|
||
memory=TurnMemory(recent_turns=recent),
|
||
theory_mode=_theory_mode_of(sess),
|
||
)
|
||
|
||
result = await evaluator.evaluate_turn(
|
||
ctx,
|
||
client_reply,
|
||
engine=engine_client,
|
||
audit_hook=session_persistence.record_llm_call_audit,
|
||
)
|
||
result_payload = result.to_hook_dict()
|
||
turn_id = getattr(learner, "turn_id", None)
|
||
if not turn_id:
|
||
raise HTTPException(
|
||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||
detail="turn evaluation retry result was generated but the target turn has no durable id",
|
||
)
|
||
saved = await session_persistence.replace_turn_evaluation(
|
||
turn_id=turn_id,
|
||
evaluation=result_payload,
|
||
)
|
||
if not saved:
|
||
raise HTTPException(
|
||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||
detail="turn evaluation retry result was generated but could not be saved",
|
||
)
|
||
learner.evaluation = result_payload
|
||
if result.error:
|
||
raise HTTPException(
|
||
_session_evaluation_error_status(result.error),
|
||
detail=_safe_session_evaluation_retry_detail(),
|
||
)
|
||
return result
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# 회기 평가 조회 (교수자/관리자) — 마지막 deep 결과 + 분포
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
@router.get("/sessions/{session_id}/evaluation", response_model=EvaluationSummary)
|
||
async def get_session_evaluation(
|
||
session_id: str,
|
||
principal: TeacherOrAdmin,
|
||
) -> EvaluationSummary:
|
||
"""회기 평가 조회(읽기) — 저장된 마지막 deep 재평가 결과 + 기법 분포.
|
||
|
||
아직 평가 트리거가 없었다면 deep=None + 빈 분포.
|
||
"""
|
||
await _load_session_or_404(session_id, principal)
|
||
record, durable = await session_persistence.load_session_evaluation(session_id, principal)
|
||
if record is None:
|
||
return EvaluationSummary(
|
||
session_id=session_id,
|
||
stage=None,
|
||
status=None,
|
||
error=None,
|
||
durable=durable,
|
||
deep=None,
|
||
distribution={},
|
||
)
|
||
payload = record.get("payload")
|
||
deep = dict(payload) if isinstance(payload, dict) else {}
|
||
# SessionEvaluation.to_dict()에는 server-side error가 함께 저장된다. deep 객체도
|
||
# 교수자 API 경계에서는 같은 원칙으로 제거한다.
|
||
deep.pop("error", None)
|
||
distribution = deep.get("distribution")
|
||
failure = teacher_evaluation_failure(record)
|
||
return EvaluationSummary(
|
||
session_id=session_id,
|
||
stage=_summary_stage(record.get("stage") or deep.get("stage")),
|
||
status=str(record.get("status") or "") or None,
|
||
error=_safe_session_evaluation_retry_detail() if failure is not None else None,
|
||
failure=failure,
|
||
durable=durable,
|
||
deep=deep,
|
||
distribution=distribution if isinstance(distribution, dict) else {},
|
||
)
|