yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

세션 평가와 교수자 분석 보강

Browse source
This commit is contained in:
Yun Chan 2026-07-01 12:10:52 +09:00
parent 5c4ac04e06
commit fe2796f05a
51 changed files with 4928 additions and 240 deletions
Download patch file Download diff file Expand all files Collapse all files

67
apps/api/app/routes/eval.py
View file

@ -17,6 +17,7 @@ in-proc cache/session fallback 은 local dev 에서만 허용한다.
from __future__ import annotations
import logging
from typing import Annotated, Any, Optional
from fastapi import APIRouter, Depends, HTTPException, status
@ -33,6 +34,7 @@ from ..store import InProcSession
from ..store import store
router = APIRouter(prefix="/eval", tags=["eval"])
logger = logging.getLogger(__name__)
# 교수자/관리자만 평가 조회·트리거 (학습자 비노출)
TeacherOrAdmin = Annotated[Principal, Depends(require_role(Role.TEACHER, Role.ADMIN))]
@ -52,6 +54,9 @@ class EvaluationSummary(BaseModel):
session_id: str
stage: StageLabel | None = None
status: str | None = None
error: str | None = None
durable: bool = False
deep: Optional[dict[str, Any]] = None
distribution: dict[str, Any] = Field(default_factory=dict)
@ -76,11 +81,14 @@ async def _load_session_or_404(session_id: str, principal: Principal) -> InProcS
def _theory_mode_of(sess) -> Optional[str]:
# 회기에서 학습자가 명시 선택한 이론 모드가 최우선이다.
sess_theory = str(getattr(sess, "theory_mode", "") or "").strip()
if sess_theory:
return sess_theory
tt = getattr(sess.persona, "theory_target", None)
if isinstance(tt, (list, tuple)) and tt:
return ", ".join(str(x) for x in tt)
# store 가 theory_mode 문자열도 보유(InProcSession.theory_mode)
return getattr(sess, "theory_mode", None)
return None
def _summary_stage(value: object) -> StageLabel | None:
@ -93,6 +101,12 @@ async def eval_health() -> dict[str, str]:
return {"status": "ok", "owner": "features:evaluator", "loops": "fast,deep"}
def _session_evaluation_error_status(error: str) -> int:
if error.startswith("engine_error"):
return status.HTTP_503_SERVICE_UNAVAILABLE
return status.HTTP_502_BAD_GATEWAY
# ════════════════════════════════════════════════════════════════════════════
# 회기 deep-loop 재평가 트리거 (교수자/관리자)
# ════════════════════════════════════════════════════════════════════════════
@ -131,18 +145,35 @@ async def reevaluate_session(
audit_hook=session_persistence.record_llm_call_audit,
)
except EngineError as e:
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"engine unavailable: {e}")
if result.error and result.error.startswith("engine_error"):
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=result.error)
await session_persistence.save_session_evaluation(
session_persistence.SessionEvaluationWrite.from_result(
detail = f"engine unavailable: {e}"
write = session_persistence.SessionEvaluationWrite.from_error(
session_id=session_id,
learner_id=sess.learner_id,
result=result,
scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end",
stage=sess.state.stage.value,
error=detail,
)
saved = await session_persistence.save_session_evaluation(write)
if not saved:
logger.error(
"session evaluation retry error record did not reach durable store: session_id=%s error=%s",
session_id,
write.error,
)
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail)
write = session_persistence.SessionEvaluationWrite.from_result(
session_id=session_id,
learner_id=sess.learner_id,
result=result,
)
saved = await session_persistence.save_session_evaluation(write)
if not saved:
detail = "session evaluation retry result was generated but could not be saved"
logger.error("%s: session_id=%s status=%s", detail, session_id, write.status)
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail)
if result.error:
raise HTTPException(_session_evaluation_error_status(result.error), detail=result.error)
return result
@ -193,6 +224,7 @@ async def reevaluate_turn(
learner_text_masked=learner.text_masked,
state_after=sess.state, # 조회 시점 상태(정밀 재현은 DB 스냅샷 도입 시)
memory=TurnMemory(recent_turns=recent),
theory_mode=_theory_mode_of(sess),
)
result = await evaluator.evaluate_turn(
@ -219,15 +251,26 @@ async def get_session_evaluation(
아직 평가 트리거가 없었다면 deep=None + 빈 분포.
"""
await _load_session_or_404(session_id, principal)
record, _durable = await session_persistence.load_session_evaluation(session_id, principal)
record, durable = await session_persistence.load_session_evaluation(session_id, principal)
if record is None:
return EvaluationSummary(session_id=session_id, stage=None, deep=None, distribution={})
return EvaluationSummary(
session_id=session_id,
stage=None,
status=None,
error=None,
durable=durable,
deep=None,
distribution={},
)
payload = record.get("payload")
deep = payload if isinstance(payload, dict) else {}
distribution = deep.get("distribution")
return EvaluationSummary(
session_id=session_id,
stage=_summary_stage(record.get("stage") or deep.get("stage")),
status=str(record.get("status") or "") or None,
error=str(record.get("error") or "") or None,
durable=durable,
deep=deep,
distribution=distribution if isinstance(distribution, dict) else {},
)

188
apps/api/app/routes/sessions.py
View file

@ -482,22 +482,20 @@ async def _load_session_or_404(
return sess
async def _load_review_session_or_404(
def _review_supervisor_principal(principal: Principal) -> Principal | None:
if principal.role in {Role.TEACHER, Role.ADMIN}:
return principal
if principal.super_admin:
return principal.with_role(Role.ADMIN)
return None
async def _load_supervisor_review_session_or_404(
session_id: str,
principal: Principal,
*,
include_turn_evaluation: bool = False,
) -> InProcSession:
if principal.role == Role.LEARNER:
return await _load_session_or_404(
session_id,
principal,
allow_ended=True,
include_turn_evaluation=include_turn_evaluation,
)
if principal.role not in {Role.TEACHER, Role.ADMIN}:
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
sess = await session_persistence.load_session(
session_id,
principal,
@ -511,6 +509,50 @@ async def _load_review_session_or_404(
return sess
async def _load_review_session_or_404(
session_id: str,
principal: Principal,
*,
include_turn_evaluation: bool = False,
) -> tuple[InProcSession, Principal]:
if principal.role == Role.LEARNER:
try:
sess = await _load_session_or_404(
session_id,
principal,
allow_ended=True,
include_turn_evaluation=include_turn_evaluation,
)
return sess, principal
except HTTPException as exc:
supervisor = _review_supervisor_principal(principal)
if supervisor is None or exc.status_code not in {
status.HTTP_403_FORBIDDEN,
status.HTTP_404_NOT_FOUND,
}:
raise
return (
await _load_supervisor_review_session_or_404(
session_id,
supervisor,
include_turn_evaluation=include_turn_evaluation,
),
supervisor,
)
supervisor = _review_supervisor_principal(principal)
if supervisor is None:
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
return (
await _load_supervisor_review_session_or_404(
session_id,
supervisor,
include_turn_evaluation=include_turn_evaluation,
),
supervisor,
)
async def _end_persisted_session(sess: InProcSession, carry: memory.CarryOver) -> None:
if await session_persistence.end_session(sess, carry):
sess.ended = True
@ -635,6 +677,7 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
if not sess.turns:
return
timeout_seconds = _session_evaluation_timeout_seconds()
enriched: list[dict[str, object]] = []
for index, turn in enumerate(sess.masked_turns(), start=1):
item: dict[str, object] = dict(turn)
@ -653,31 +696,89 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
scope="session_end",
audit_hook=session_persistence.record_llm_call_audit,
),
timeout=min(float(settings.engine_timeout), 45.0),
timeout=timeout_seconds,
)
saved = await session_persistence.save_session_evaluation(
session_persistence.SessionEvaluationWrite.from_result(
session_id=sess.session_id,
learner_id=sess.learner_id,
result=result,
write = session_persistence.SessionEvaluationWrite.from_result(
session_id=sess.session_id,
learner_id=sess.learner_id,
result=result,
)
saved = await session_persistence.save_session_evaluation(write)
if not saved:
logger.error(
"session evaluation save did not reach durable store: session_id=%s status=%s scope=%s",
sess.session_id,
write.status,
write.scope,
)
if saved:
await _enqueue_session_review_ready_notification(sess.session_id)
except asyncio.TimeoutError:
message = f"session evaluation timeout after {timeout_seconds:g}s"
logger.exception("%s: session_id=%s", message, sess.session_id)
write = session_persistence.SessionEvaluationWrite.from_error(
session_id=sess.session_id,
learner_id=sess.learner_id,
scope="session_end",
stage=_stage_label(sess.state.stage),
error=message,
)
saved = await session_persistence.save_session_evaluation(write)
if not saved:
logger.error(
"session evaluation error save did not reach durable store: session_id=%s error=%s",
sess.session_id,
write.error,
)
if saved:
await _enqueue_session_review_ready_notification(sess.session_id)
except Exception as exc:
saved = await session_persistence.save_session_evaluation(
session_persistence.SessionEvaluationWrite.from_error(
session_id=sess.session_id,
learner_id=sess.learner_id,
scope="session_end",
stage=_stage_label(sess.state.stage),
error=str(exc),
)
logger.exception("session evaluation failed: session_id=%s", sess.session_id)
write = session_persistence.SessionEvaluationWrite.from_error(
session_id=sess.session_id,
learner_id=sess.learner_id,
scope="session_end",
stage=_stage_label(sess.state.stage),
error=exc,
)
saved = await session_persistence.save_session_evaluation(write)
if not saved:
logger.error(
"session evaluation failure record did not reach durable store: session_id=%s error=%s",
sess.session_id,
write.error,
)
if saved:
await _enqueue_session_review_ready_notification(sess.session_id)
def _observe_session_evaluation_task(task: asyncio.Task[None], session_id: str) -> None:
try:
task.result()
except asyncio.CancelledError:
logger.warning("session evaluation background task cancelled: session_id=%s", session_id)
except Exception:
logger.exception("session evaluation background task crashed: session_id=%s", session_id)
def _schedule_session_evaluation(sess: InProcSession) -> asyncio.Task[None] | None:
if not sess.turns:
return None
task = asyncio.create_task(
_generate_and_save_session_evaluation(sess),
name=f"session-evaluation:{sess.session_id}",
)
task.add_done_callback(
lambda done, session_id=sess.session_id: _observe_session_evaluation_task(done, session_id)
)
return task
def _session_evaluation_timeout_seconds() -> float:
configured = float(settings.session_evaluation_timeout or settings.engine_timeout)
return max(configured, 1.0)
async def _enqueue_session_review_ready_notification(session_id: str) -> None:
try:
await notifications.enqueue_session_review_ready(session_id=session_id)
@ -685,12 +786,6 @@ async def _enqueue_session_review_ready_notification(session_id: str) -> None:
logger.warning("session review notification enqueue failed: %s", exc)
def _schedule_session_evaluation(sess: InProcSession) -> None:
if not sess.turns:
return
asyncio.create_task(_generate_and_save_session_evaluation(sess))
async def _load_learner_sessions(
principal: Principal,
*,
@ -978,25 +1073,25 @@ async def get_session_review(
principal: CurrentPrincipal,
) -> SessionReviewResponse:
"""Return a role-safe review built only from the stored session transcript."""
sess = await _load_review_session_or_404(
sess, review_principal = await _load_review_session_or_404(
session_id,
principal,
include_turn_evaluation=True,
)
evaluation_record, evaluation_durable = await session_persistence.load_session_evaluation(
session_id,
principal,
review_principal,
)
saved_worksheet_payload, _ = await session_persistence.load_case_worksheet(
session_id,
principal,
review_principal,
)
include_teacher_review = principal.role in {Role.TEACHER, Role.ADMIN}
include_teacher_review = review_principal.role in {Role.TEACHER, Role.ADMIN}
teacher_review_record = None
if include_teacher_review:
teacher_review_record, _ = await session_persistence.load_session_review_status(
session_id,
principal,
review_principal,
)
return build_session_review(
@ -1201,18 +1296,15 @@ async def live_coach_turn(
grounding=grounding,
audit_hook=session_persistence.record_llm_call_audit,
)
try:
await session_persistence.save_live_coach_event(
session_id=sess.session_id,
learner_id=sess.learner_id,
turn_seq=turn_seq,
stage=stage,
learner_text=body.learner_text,
client_reply=body.client_reply,
suggestion=suggestion,
)
except Exception:
pass
await session_persistence.save_live_coach_event(
session_id=sess.session_id,
learner_id=sess.learner_id,
turn_seq=turn_seq,
stage=stage,
learner_text=body.learner_text,
client_reply=body.client_reply,
suggestion=suggestion,
)
return suggestion

111
apps/api/app/routes/teacher.py
View file

@ -12,6 +12,7 @@ from ..deps import Principal, Role, require_role
from ..runtime_policy import require_runtime_fallback_allowed
from ..session_read_model import StageLabel, stage_label
from ..services import session_metrics
from ..stage_contract import STAGE_LABEL_VALUES
from ..store import InProcSession, store
router = APIRouter(prefix="/teacher", tags=["teacher"])
@ -86,6 +87,28 @@ class TeacherLearnerGrowth(BaseModel):
points: list[TeacherGrowthPoint] = Field(default_factory=list)
class TeacherStageBreakdown(BaseModel):
stage: StageLabel
sessions: int = 0
turns: int = 0
class TeacherLearnerAnalysisResponse(BaseModel):
source: str = "in_memory"
learner_id: str
learner_label: str
total_sessions: int
active_sessions: int
ended_sessions: int
pending_reviews: int
closed_reviews: int
summary: TeacherLearnerGrowth
points: list[TeacherGrowthPoint] = Field(default_factory=list)
stage_breakdown: list[TeacherStageBreakdown] = Field(default_factory=list)
sessions: list[TeacherSessionSummary] = Field(default_factory=list)
message: str
class TeacherSafetyAlert(BaseModel):
id: str
session_id: str
@ -134,11 +157,17 @@ def _growth_point(point: session_metrics.SessionGrowthPoint) -> TeacherGrowthPoi
)
def _build_learner_growth(sessions: list[InProcSession]) -> list[TeacherLearnerGrowth]:
def _build_learner_growth(
sessions: list[InProcSession],
*,
limit: int | None = 12,
point_limit: int | None = 6,
) -> list[TeacherLearnerGrowth]:
metrics = session_metrics.build_learner_growth(
sessions,
learner_label=_learner_label,
limit=12,
limit=limit,
point_limit=point_limit,
)
return [
TeacherLearnerGrowth(
@ -160,6 +189,26 @@ def _build_learner_growth(sessions: list[InProcSession]) -> list[TeacherLearnerG
]
def _stage_breakdown(sessions: list[InProcSession]) -> list[TeacherStageBreakdown]:
buckets = {
stage: {"sessions": 0, "turns": 0}
for stage in STAGE_LABEL_VALUES
}
for sess in sessions:
label = stage_label(sess.state.stage)
bucket = buckets[label]
bucket["sessions"] += 1
bucket["turns"] += len(sess.turns)
return [
TeacherStageBreakdown(
stage=stage,
sessions=buckets[stage]["sessions"],
turns=buckets[stage]["turns"],
)
for stage in STAGE_LABEL_VALUES
]
def _review_status_value(record: dict[str, object] | None) -> Literal["pending", "viewed", "closed"]:
value = str((record or {}).get("status") or "pending")
if value in {"viewed", "closed"}:
@ -274,6 +323,64 @@ async def teacher_dashboard(principal: TeacherPrincipal) -> TeacherDashboardResp
)
@router.get(
"/learners/{learner_id}/analysis",
response_model=TeacherLearnerAnalysisResponse,
)
async def learner_analysis(
learner_id: str,
principal: TeacherPrincipal,
) -> TeacherLearnerAnalysisResponse:
"""Return one learner's full teacher-visible session timeline and trend."""
sessions, durable = await session_persistence.list_sessions(
principal,
include_turn_evaluation=True,
)
if not durable:
require_runtime_fallback_allowed("teacher learner analysis")
sessions = sorted(store.list(), key=lambda sess: sess.created_at, reverse=True)
learner_sessions = [sess for sess in sessions if sess.learner_id == learner_id]
if not learner_sessions:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="learner not found")
ordered = sorted(
learner_sessions,
key=lambda sess: (sess.created_at, sess.session_no, sess.session_id),
)
review_statuses, _ = await session_persistence.list_session_review_statuses(
[sess.session_id for sess in ordered if sess.ended],
principal,
)
summaries = [
_summary(sess, review_statuses.get(sess.session_id))
for sess in ordered
]
growth = _build_learner_growth(ordered, limit=None, point_limit=None)[0]
pending_reviews = sum(
1
for item in summaries
if item.status == "ended" and item.review_status != "closed"
)
closed_reviews = sum(1 for item in summaries if item.review_status == "closed")
return TeacherLearnerAnalysisResponse(
source="database" if durable else "runtime",
learner_id=learner_id,
learner_label=growth.learner_label,
total_sessions=len(ordered),
active_sessions=sum(1 for sess in ordered if not sess.ended),
ended_sessions=sum(1 for sess in ordered if sess.ended),
pending_reviews=pending_reviews,
closed_reviews=closed_reviews,
summary=growth,
points=growth.points,
stage_breakdown=_stage_breakdown(ordered),
sessions=summaries,
message=f"{growth.learner_label}의 실제 회기 {len(ordered)}건을 오래된 순서로 표시합니다.",
)
@router.put(
"/sessions/{session_id}/review-status",
response_model=TeacherSessionReviewStatusResponse,