세션 평가와 교수자 분석 보강
This commit is contained in:
Yun Chan
parent
5c4ac04e06
commit
fe2796f05a
51 changed files with 4928 additions and 240 deletions
|
|
@ -482,22 +482,20 @@ async def _load_session_or_404(
|
|||
return sess
|
||||
|
||||
|
||||
async def _load_review_session_or_404(
|
||||
def _review_supervisor_principal(principal: Principal) -> Principal | None:
|
||||
if principal.role in {Role.TEACHER, Role.ADMIN}:
|
||||
return principal
|
||||
if principal.super_admin:
|
||||
return principal.with_role(Role.ADMIN)
|
||||
return None
|
||||
|
||||
|
||||
async def _load_supervisor_review_session_or_404(
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
*,
|
||||
include_turn_evaluation: bool = False,
|
||||
) -> InProcSession:
|
||||
if principal.role == Role.LEARNER:
|
||||
return await _load_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
allow_ended=True,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
)
|
||||
if principal.role not in {Role.TEACHER, Role.ADMIN}:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
|
||||
|
||||
sess = await session_persistence.load_session(
|
||||
session_id,
|
||||
principal,
|
||||
|
|
@ -511,6 +509,50 @@ async def _load_review_session_or_404(
|
|||
return sess
|
||||
|
||||
|
||||
async def _load_review_session_or_404(
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
*,
|
||||
include_turn_evaluation: bool = False,
|
||||
) -> tuple[InProcSession, Principal]:
|
||||
if principal.role == Role.LEARNER:
|
||||
try:
|
||||
sess = await _load_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
allow_ended=True,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
)
|
||||
return sess, principal
|
||||
except HTTPException as exc:
|
||||
supervisor = _review_supervisor_principal(principal)
|
||||
if supervisor is None or exc.status_code not in {
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
}:
|
||||
raise
|
||||
return (
|
||||
await _load_supervisor_review_session_or_404(
|
||||
session_id,
|
||||
supervisor,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
),
|
||||
supervisor,
|
||||
)
|
||||
|
||||
supervisor = _review_supervisor_principal(principal)
|
||||
if supervisor is None:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
|
||||
return (
|
||||
await _load_supervisor_review_session_or_404(
|
||||
session_id,
|
||||
supervisor,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
),
|
||||
supervisor,
|
||||
)
|
||||
|
||||
|
||||
async def _end_persisted_session(sess: InProcSession, carry: memory.CarryOver) -> None:
|
||||
if await session_persistence.end_session(sess, carry):
|
||||
sess.ended = True
|
||||
|
|
@ -635,6 +677,7 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
|
|||
if not sess.turns:
|
||||
return
|
||||
|
||||
timeout_seconds = _session_evaluation_timeout_seconds()
|
||||
enriched: list[dict[str, object]] = []
|
||||
for index, turn in enumerate(sess.masked_turns(), start=1):
|
||||
item: dict[str, object] = dict(turn)
|
||||
|
|
@ -653,31 +696,89 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
|
|||
scope="session_end",
|
||||
audit_hook=session_persistence.record_llm_call_audit,
|
||||
),
|
||||
timeout=min(float(settings.engine_timeout), 45.0),
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(
|
||||
session_persistence.SessionEvaluationWrite.from_result(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
write = session_persistence.SessionEvaluationWrite.from_result(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation save did not reach durable store: session_id=%s status=%s scope=%s",
|
||||
sess.session_id,
|
||||
write.status,
|
||||
write.scope,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
except asyncio.TimeoutError:
|
||||
message = f"session evaluation timeout after {timeout_seconds:g}s"
|
||||
logger.exception("%s: session_id=%s", message, sess.session_id)
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=message,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation error save did not reach durable store: session_id=%s error=%s",
|
||||
sess.session_id,
|
||||
write.error,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
except Exception as exc:
|
||||
saved = await session_persistence.save_session_evaluation(
|
||||
session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=str(exc),
|
||||
)
|
||||
logger.exception("session evaluation failed: session_id=%s", sess.session_id)
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=exc,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation failure record did not reach durable store: session_id=%s error=%s",
|
||||
sess.session_id,
|
||||
write.error,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
|
||||
|
||||
def _observe_session_evaluation_task(task: asyncio.Task[None], session_id: str) -> None:
|
||||
try:
|
||||
task.result()
|
||||
except asyncio.CancelledError:
|
||||
logger.warning("session evaluation background task cancelled: session_id=%s", session_id)
|
||||
except Exception:
|
||||
logger.exception("session evaluation background task crashed: session_id=%s", session_id)
|
||||
|
||||
|
||||
def _schedule_session_evaluation(sess: InProcSession) -> asyncio.Task[None] | None:
|
||||
if not sess.turns:
|
||||
return None
|
||||
task = asyncio.create_task(
|
||||
_generate_and_save_session_evaluation(sess),
|
||||
name=f"session-evaluation:{sess.session_id}",
|
||||
)
|
||||
task.add_done_callback(
|
||||
lambda done, session_id=sess.session_id: _observe_session_evaluation_task(done, session_id)
|
||||
)
|
||||
return task
|
||||
|
||||
|
||||
def _session_evaluation_timeout_seconds() -> float:
|
||||
configured = float(settings.session_evaluation_timeout or settings.engine_timeout)
|
||||
return max(configured, 1.0)
|
||||
|
||||
|
||||
async def _enqueue_session_review_ready_notification(session_id: str) -> None:
|
||||
try:
|
||||
await notifications.enqueue_session_review_ready(session_id=session_id)
|
||||
|
|
@ -685,12 +786,6 @@ async def _enqueue_session_review_ready_notification(session_id: str) -> None:
|
|||
logger.warning("session review notification enqueue failed: %s", exc)
|
||||
|
||||
|
||||
def _schedule_session_evaluation(sess: InProcSession) -> None:
|
||||
if not sess.turns:
|
||||
return
|
||||
asyncio.create_task(_generate_and_save_session_evaluation(sess))
|
||||
|
||||
|
||||
async def _load_learner_sessions(
|
||||
principal: Principal,
|
||||
*,
|
||||
|
|
@ -978,25 +1073,25 @@ async def get_session_review(
|
|||
principal: CurrentPrincipal,
|
||||
) -> SessionReviewResponse:
|
||||
"""Return a role-safe review built only from the stored session transcript."""
|
||||
sess = await _load_review_session_or_404(
|
||||
sess, review_principal = await _load_review_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
include_turn_evaluation=True,
|
||||
)
|
||||
evaluation_record, evaluation_durable = await session_persistence.load_session_evaluation(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
saved_worksheet_payload, _ = await session_persistence.load_case_worksheet(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
include_teacher_review = principal.role in {Role.TEACHER, Role.ADMIN}
|
||||
include_teacher_review = review_principal.role in {Role.TEACHER, Role.ADMIN}
|
||||
teacher_review_record = None
|
||||
if include_teacher_review:
|
||||
teacher_review_record, _ = await session_persistence.load_session_review_status(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
|
||||
return build_session_review(
|
||||
|
|
@ -1201,18 +1296,15 @@ async def live_coach_turn(
|
|||
grounding=grounding,
|
||||
audit_hook=session_persistence.record_llm_call_audit,
|
||||
)
|
||||
try:
|
||||
await session_persistence.save_live_coach_event(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
turn_seq=turn_seq,
|
||||
stage=stage,
|
||||
learner_text=body.learner_text,
|
||||
client_reply=body.client_reply,
|
||||
suggestion=suggestion,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
await session_persistence.save_live_coach_event(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
turn_seq=turn_seq,
|
||||
stage=stage,
|
||||
learner_text=body.learner_text,
|
||||
client_reply=body.client_reply,
|
||||
suggestion=suggestion,
|
||||
)
|
||||
return suggestion
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue