세션 평가와 교수자 분석 보강
This commit is contained in:
Yun Chan
parent
5c4ac04e06
commit
fe2796f05a
51 changed files with 4928 additions and 240 deletions
|
|
@ -359,8 +359,16 @@ async def _runtime_tables_ready(conn) -> bool:
|
|||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'app'
|
||||
AND table_name = 'turns'
|
||||
AND column_name = 'provider_events'
|
||||
) AS has_turn_provider_events,
|
||||
AND column_name IN (
|
||||
'audio_ref',
|
||||
'silence_ms',
|
||||
'speech_rate',
|
||||
'barge_in',
|
||||
'provider_events'
|
||||
)
|
||||
GROUP BY table_schema, table_name
|
||||
HAVING count(*) = 5
|
||||
) AS has_turn_voice_metadata_columns,
|
||||
EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'app'
|
||||
|
|
@ -485,7 +493,7 @@ async def _runtime_tables_ready(conn) -> bool:
|
|||
and row["has_engine_config"]
|
||||
and row["has_session_columns"]
|
||||
and row["has_state_columns"]
|
||||
and row["has_turn_provider_events"]
|
||||
and row["has_turn_voice_metadata_columns"]
|
||||
and row["has_session_review_worksheet_columns"]
|
||||
and row["has_stage_defs"]
|
||||
and row["has_admin_health_event"]
|
||||
|
|
@ -908,6 +916,10 @@ async def ensure_runtime_tables() -> None:
|
|||
await conn.execute(
|
||||
"""
|
||||
ALTER TABLE app.turns
|
||||
ADD COLUMN IF NOT EXISTS audio_ref TEXT,
|
||||
ADD COLUMN IF NOT EXISTS silence_ms INT,
|
||||
ADD COLUMN IF NOT EXISTS speech_rate REAL,
|
||||
ADD COLUMN IF NOT EXISTS barge_in BOOLEAN,
|
||||
ADD COLUMN IF NOT EXISTS provider_events JSONB NOT NULL DEFAULT '[]'::jsonb
|
||||
"""
|
||||
)
|
||||
|
|
|
|||
|
|
@ -104,6 +104,10 @@ class Settings(BaseSettings):
|
|||
default=256,
|
||||
validation_alias="EVALUATOR_SEMANTIC_CACHE_MAX_ENTRIES",
|
||||
)
|
||||
session_evaluation_timeout: float = Field(
|
||||
default=0.0,
|
||||
validation_alias="SESSION_EVALUATION_TIMEOUT",
|
||||
)
|
||||
session_digest_worker_enabled: bool = Field(
|
||||
default=False,
|
||||
validation_alias="SESSION_DIGEST_WORKER_ENABLED",
|
||||
|
|
|
|||
|
|
@ -161,8 +161,16 @@ async def healthcheck() -> bool:
|
|||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'app'
|
||||
AND table_name = 'turns'
|
||||
AND column_name = 'provider_events'
|
||||
) AS has_turn_provider_events,
|
||||
AND column_name IN (
|
||||
'audio_ref',
|
||||
'silence_ms',
|
||||
'speech_rate',
|
||||
'barge_in',
|
||||
'provider_events'
|
||||
)
|
||||
GROUP BY table_schema, table_name
|
||||
HAVING count(*) = 5
|
||||
) AS has_turn_voice_metadata_columns,
|
||||
EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'app'
|
||||
|
|
@ -191,7 +199,7 @@ async def healthcheck() -> bool:
|
|||
and row["has_sessions"]
|
||||
and row["has_turns"]
|
||||
and row["has_session_review_status"]
|
||||
and row["has_turn_provider_events"]
|
||||
and row["has_turn_voice_metadata_columns"]
|
||||
and row["has_session_review_worksheet_columns"]
|
||||
)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ in-proc cache/session fallback 은 local dev 에서만 허용한다.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Any, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
|
@ -33,6 +34,7 @@ from ..store import InProcSession
|
|||
from ..store import store
|
||||
|
||||
router = APIRouter(prefix="/eval", tags=["eval"])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 교수자/관리자만 평가 조회·트리거 (학습자 비노출)
|
||||
TeacherOrAdmin = Annotated[Principal, Depends(require_role(Role.TEACHER, Role.ADMIN))]
|
||||
|
|
@ -52,6 +54,9 @@ class EvaluationSummary(BaseModel):
|
|||
|
||||
session_id: str
|
||||
stage: StageLabel | None = None
|
||||
status: str | None = None
|
||||
error: str | None = None
|
||||
durable: bool = False
|
||||
deep: Optional[dict[str, Any]] = None
|
||||
distribution: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
|
@ -76,11 +81,14 @@ async def _load_session_or_404(session_id: str, principal: Principal) -> InProcS
|
|||
|
||||
|
||||
def _theory_mode_of(sess) -> Optional[str]:
|
||||
# 회기에서 학습자가 명시 선택한 이론 모드가 최우선이다.
|
||||
sess_theory = str(getattr(sess, "theory_mode", "") or "").strip()
|
||||
if sess_theory:
|
||||
return sess_theory
|
||||
tt = getattr(sess.persona, "theory_target", None)
|
||||
if isinstance(tt, (list, tuple)) and tt:
|
||||
return ", ".join(str(x) for x in tt)
|
||||
# store 가 theory_mode 문자열도 보유(InProcSession.theory_mode)
|
||||
return getattr(sess, "theory_mode", None)
|
||||
return None
|
||||
|
||||
|
||||
def _summary_stage(value: object) -> StageLabel | None:
|
||||
|
|
@ -93,6 +101,12 @@ async def eval_health() -> dict[str, str]:
|
|||
return {"status": "ok", "owner": "features:evaluator", "loops": "fast,deep"}
|
||||
|
||||
|
||||
def _session_evaluation_error_status(error: str) -> int:
|
||||
if error.startswith("engine_error"):
|
||||
return status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
return status.HTTP_502_BAD_GATEWAY
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 회기 deep-loop 재평가 트리거 (교수자/관리자)
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
|
@ -131,18 +145,35 @@ async def reevaluate_session(
|
|||
audit_hook=session_persistence.record_llm_call_audit,
|
||||
)
|
||||
except EngineError as e:
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"engine unavailable: {e}")
|
||||
|
||||
if result.error and result.error.startswith("engine_error"):
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=result.error)
|
||||
|
||||
await session_persistence.save_session_evaluation(
|
||||
session_persistence.SessionEvaluationWrite.from_result(
|
||||
detail = f"engine unavailable: {e}"
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
scope=body.scope if body.scope in ("session_end", "stage_transition") else "session_end",
|
||||
stage=sess.state.stage.value,
|
||||
error=detail,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation retry error record did not reach durable store: session_id=%s error=%s",
|
||||
session_id,
|
||||
write.error,
|
||||
)
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail)
|
||||
|
||||
write = session_persistence.SessionEvaluationWrite.from_result(
|
||||
session_id=session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
detail = "session evaluation retry result was generated but could not be saved"
|
||||
logger.error("%s: session_id=%s status=%s", detail, session_id, write.status)
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=detail)
|
||||
if result.error:
|
||||
raise HTTPException(_session_evaluation_error_status(result.error), detail=result.error)
|
||||
return result
|
||||
|
||||
|
||||
|
|
@ -193,6 +224,7 @@ async def reevaluate_turn(
|
|||
learner_text_masked=learner.text_masked,
|
||||
state_after=sess.state, # 조회 시점 상태(정밀 재현은 DB 스냅샷 도입 시)
|
||||
memory=TurnMemory(recent_turns=recent),
|
||||
theory_mode=_theory_mode_of(sess),
|
||||
)
|
||||
|
||||
result = await evaluator.evaluate_turn(
|
||||
|
|
@ -219,15 +251,26 @@ async def get_session_evaluation(
|
|||
아직 평가 트리거가 없었다면 deep=None + 빈 분포.
|
||||
"""
|
||||
await _load_session_or_404(session_id, principal)
|
||||
record, _durable = await session_persistence.load_session_evaluation(session_id, principal)
|
||||
record, durable = await session_persistence.load_session_evaluation(session_id, principal)
|
||||
if record is None:
|
||||
return EvaluationSummary(session_id=session_id, stage=None, deep=None, distribution={})
|
||||
return EvaluationSummary(
|
||||
session_id=session_id,
|
||||
stage=None,
|
||||
status=None,
|
||||
error=None,
|
||||
durable=durable,
|
||||
deep=None,
|
||||
distribution={},
|
||||
)
|
||||
payload = record.get("payload")
|
||||
deep = payload if isinstance(payload, dict) else {}
|
||||
distribution = deep.get("distribution")
|
||||
return EvaluationSummary(
|
||||
session_id=session_id,
|
||||
stage=_summary_stage(record.get("stage") or deep.get("stage")),
|
||||
status=str(record.get("status") or "") or None,
|
||||
error=str(record.get("error") or "") or None,
|
||||
durable=durable,
|
||||
deep=deep,
|
||||
distribution=distribution if isinstance(distribution, dict) else {},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -482,22 +482,20 @@ async def _load_session_or_404(
|
|||
return sess
|
||||
|
||||
|
||||
async def _load_review_session_or_404(
|
||||
def _review_supervisor_principal(principal: Principal) -> Principal | None:
|
||||
if principal.role in {Role.TEACHER, Role.ADMIN}:
|
||||
return principal
|
||||
if principal.super_admin:
|
||||
return principal.with_role(Role.ADMIN)
|
||||
return None
|
||||
|
||||
|
||||
async def _load_supervisor_review_session_or_404(
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
*,
|
||||
include_turn_evaluation: bool = False,
|
||||
) -> InProcSession:
|
||||
if principal.role == Role.LEARNER:
|
||||
return await _load_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
allow_ended=True,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
)
|
||||
if principal.role not in {Role.TEACHER, Role.ADMIN}:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
|
||||
|
||||
sess = await session_persistence.load_session(
|
||||
session_id,
|
||||
principal,
|
||||
|
|
@ -511,6 +509,50 @@ async def _load_review_session_or_404(
|
|||
return sess
|
||||
|
||||
|
||||
async def _load_review_session_or_404(
|
||||
session_id: str,
|
||||
principal: Principal,
|
||||
*,
|
||||
include_turn_evaluation: bool = False,
|
||||
) -> tuple[InProcSession, Principal]:
|
||||
if principal.role == Role.LEARNER:
|
||||
try:
|
||||
sess = await _load_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
allow_ended=True,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
)
|
||||
return sess, principal
|
||||
except HTTPException as exc:
|
||||
supervisor = _review_supervisor_principal(principal)
|
||||
if supervisor is None or exc.status_code not in {
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
}:
|
||||
raise
|
||||
return (
|
||||
await _load_supervisor_review_session_or_404(
|
||||
session_id,
|
||||
supervisor,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
),
|
||||
supervisor,
|
||||
)
|
||||
|
||||
supervisor = _review_supervisor_principal(principal)
|
||||
if supervisor is None:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="session review access denied")
|
||||
return (
|
||||
await _load_supervisor_review_session_or_404(
|
||||
session_id,
|
||||
supervisor,
|
||||
include_turn_evaluation=include_turn_evaluation,
|
||||
),
|
||||
supervisor,
|
||||
)
|
||||
|
||||
|
||||
async def _end_persisted_session(sess: InProcSession, carry: memory.CarryOver) -> None:
|
||||
if await session_persistence.end_session(sess, carry):
|
||||
sess.ended = True
|
||||
|
|
@ -635,6 +677,7 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
|
|||
if not sess.turns:
|
||||
return
|
||||
|
||||
timeout_seconds = _session_evaluation_timeout_seconds()
|
||||
enriched: list[dict[str, object]] = []
|
||||
for index, turn in enumerate(sess.masked_turns(), start=1):
|
||||
item: dict[str, object] = dict(turn)
|
||||
|
|
@ -653,31 +696,89 @@ async def _generate_and_save_session_evaluation(sess: InProcSession) -> None:
|
|||
scope="session_end",
|
||||
audit_hook=session_persistence.record_llm_call_audit,
|
||||
),
|
||||
timeout=min(float(settings.engine_timeout), 45.0),
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(
|
||||
session_persistence.SessionEvaluationWrite.from_result(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
write = session_persistence.SessionEvaluationWrite.from_result(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
result=result,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation save did not reach durable store: session_id=%s status=%s scope=%s",
|
||||
sess.session_id,
|
||||
write.status,
|
||||
write.scope,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
except asyncio.TimeoutError:
|
||||
message = f"session evaluation timeout after {timeout_seconds:g}s"
|
||||
logger.exception("%s: session_id=%s", message, sess.session_id)
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=message,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation error save did not reach durable store: session_id=%s error=%s",
|
||||
sess.session_id,
|
||||
write.error,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
except Exception as exc:
|
||||
saved = await session_persistence.save_session_evaluation(
|
||||
session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=str(exc),
|
||||
)
|
||||
logger.exception("session evaluation failed: session_id=%s", sess.session_id)
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
scope="session_end",
|
||||
stage=_stage_label(sess.state.stage),
|
||||
error=exc,
|
||||
)
|
||||
saved = await session_persistence.save_session_evaluation(write)
|
||||
if not saved:
|
||||
logger.error(
|
||||
"session evaluation failure record did not reach durable store: session_id=%s error=%s",
|
||||
sess.session_id,
|
||||
write.error,
|
||||
)
|
||||
if saved:
|
||||
await _enqueue_session_review_ready_notification(sess.session_id)
|
||||
|
||||
|
||||
def _observe_session_evaluation_task(task: asyncio.Task[None], session_id: str) -> None:
|
||||
try:
|
||||
task.result()
|
||||
except asyncio.CancelledError:
|
||||
logger.warning("session evaluation background task cancelled: session_id=%s", session_id)
|
||||
except Exception:
|
||||
logger.exception("session evaluation background task crashed: session_id=%s", session_id)
|
||||
|
||||
|
||||
def _schedule_session_evaluation(sess: InProcSession) -> asyncio.Task[None] | None:
|
||||
if not sess.turns:
|
||||
return None
|
||||
task = asyncio.create_task(
|
||||
_generate_and_save_session_evaluation(sess),
|
||||
name=f"session-evaluation:{sess.session_id}",
|
||||
)
|
||||
task.add_done_callback(
|
||||
lambda done, session_id=sess.session_id: _observe_session_evaluation_task(done, session_id)
|
||||
)
|
||||
return task
|
||||
|
||||
|
||||
def _session_evaluation_timeout_seconds() -> float:
|
||||
configured = float(settings.session_evaluation_timeout or settings.engine_timeout)
|
||||
return max(configured, 1.0)
|
||||
|
||||
|
||||
async def _enqueue_session_review_ready_notification(session_id: str) -> None:
|
||||
try:
|
||||
await notifications.enqueue_session_review_ready(session_id=session_id)
|
||||
|
|
@ -685,12 +786,6 @@ async def _enqueue_session_review_ready_notification(session_id: str) -> None:
|
|||
logger.warning("session review notification enqueue failed: %s", exc)
|
||||
|
||||
|
||||
def _schedule_session_evaluation(sess: InProcSession) -> None:
|
||||
if not sess.turns:
|
||||
return
|
||||
asyncio.create_task(_generate_and_save_session_evaluation(sess))
|
||||
|
||||
|
||||
async def _load_learner_sessions(
|
||||
principal: Principal,
|
||||
*,
|
||||
|
|
@ -978,25 +1073,25 @@ async def get_session_review(
|
|||
principal: CurrentPrincipal,
|
||||
) -> SessionReviewResponse:
|
||||
"""Return a role-safe review built only from the stored session transcript."""
|
||||
sess = await _load_review_session_or_404(
|
||||
sess, review_principal = await _load_review_session_or_404(
|
||||
session_id,
|
||||
principal,
|
||||
include_turn_evaluation=True,
|
||||
)
|
||||
evaluation_record, evaluation_durable = await session_persistence.load_session_evaluation(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
saved_worksheet_payload, _ = await session_persistence.load_case_worksheet(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
include_teacher_review = principal.role in {Role.TEACHER, Role.ADMIN}
|
||||
include_teacher_review = review_principal.role in {Role.TEACHER, Role.ADMIN}
|
||||
teacher_review_record = None
|
||||
if include_teacher_review:
|
||||
teacher_review_record, _ = await session_persistence.load_session_review_status(
|
||||
session_id,
|
||||
principal,
|
||||
review_principal,
|
||||
)
|
||||
|
||||
return build_session_review(
|
||||
|
|
@ -1201,18 +1296,15 @@ async def live_coach_turn(
|
|||
grounding=grounding,
|
||||
audit_hook=session_persistence.record_llm_call_audit,
|
||||
)
|
||||
try:
|
||||
await session_persistence.save_live_coach_event(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
turn_seq=turn_seq,
|
||||
stage=stage,
|
||||
learner_text=body.learner_text,
|
||||
client_reply=body.client_reply,
|
||||
suggestion=suggestion,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
await session_persistence.save_live_coach_event(
|
||||
session_id=sess.session_id,
|
||||
learner_id=sess.learner_id,
|
||||
turn_seq=turn_seq,
|
||||
stage=stage,
|
||||
learner_text=body.learner_text,
|
||||
client_reply=body.client_reply,
|
||||
suggestion=suggestion,
|
||||
)
|
||||
return suggestion
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from ..deps import Principal, Role, require_role
|
|||
from ..runtime_policy import require_runtime_fallback_allowed
|
||||
from ..session_read_model import StageLabel, stage_label
|
||||
from ..services import session_metrics
|
||||
from ..stage_contract import STAGE_LABEL_VALUES
|
||||
from ..store import InProcSession, store
|
||||
|
||||
router = APIRouter(prefix="/teacher", tags=["teacher"])
|
||||
|
|
@ -86,6 +87,28 @@ class TeacherLearnerGrowth(BaseModel):
|
|||
points: list[TeacherGrowthPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
class TeacherStageBreakdown(BaseModel):
|
||||
stage: StageLabel
|
||||
sessions: int = 0
|
||||
turns: int = 0
|
||||
|
||||
|
||||
class TeacherLearnerAnalysisResponse(BaseModel):
|
||||
source: str = "in_memory"
|
||||
learner_id: str
|
||||
learner_label: str
|
||||
total_sessions: int
|
||||
active_sessions: int
|
||||
ended_sessions: int
|
||||
pending_reviews: int
|
||||
closed_reviews: int
|
||||
summary: TeacherLearnerGrowth
|
||||
points: list[TeacherGrowthPoint] = Field(default_factory=list)
|
||||
stage_breakdown: list[TeacherStageBreakdown] = Field(default_factory=list)
|
||||
sessions: list[TeacherSessionSummary] = Field(default_factory=list)
|
||||
message: str
|
||||
|
||||
|
||||
class TeacherSafetyAlert(BaseModel):
|
||||
id: str
|
||||
session_id: str
|
||||
|
|
@ -134,11 +157,17 @@ def _growth_point(point: session_metrics.SessionGrowthPoint) -> TeacherGrowthPoi
|
|||
)
|
||||
|
||||
|
||||
def _build_learner_growth(sessions: list[InProcSession]) -> list[TeacherLearnerGrowth]:
|
||||
def _build_learner_growth(
|
||||
sessions: list[InProcSession],
|
||||
*,
|
||||
limit: int | None = 12,
|
||||
point_limit: int | None = 6,
|
||||
) -> list[TeacherLearnerGrowth]:
|
||||
metrics = session_metrics.build_learner_growth(
|
||||
sessions,
|
||||
learner_label=_learner_label,
|
||||
limit=12,
|
||||
limit=limit,
|
||||
point_limit=point_limit,
|
||||
)
|
||||
return [
|
||||
TeacherLearnerGrowth(
|
||||
|
|
@ -160,6 +189,26 @@ def _build_learner_growth(sessions: list[InProcSession]) -> list[TeacherLearnerG
|
|||
]
|
||||
|
||||
|
||||
def _stage_breakdown(sessions: list[InProcSession]) -> list[TeacherStageBreakdown]:
|
||||
buckets = {
|
||||
stage: {"sessions": 0, "turns": 0}
|
||||
for stage in STAGE_LABEL_VALUES
|
||||
}
|
||||
for sess in sessions:
|
||||
label = stage_label(sess.state.stage)
|
||||
bucket = buckets[label]
|
||||
bucket["sessions"] += 1
|
||||
bucket["turns"] += len(sess.turns)
|
||||
return [
|
||||
TeacherStageBreakdown(
|
||||
stage=stage,
|
||||
sessions=buckets[stage]["sessions"],
|
||||
turns=buckets[stage]["turns"],
|
||||
)
|
||||
for stage in STAGE_LABEL_VALUES
|
||||
]
|
||||
|
||||
|
||||
def _review_status_value(record: dict[str, object] | None) -> Literal["pending", "viewed", "closed"]:
|
||||
value = str((record or {}).get("status") or "pending")
|
||||
if value in {"viewed", "closed"}:
|
||||
|
|
@ -274,6 +323,64 @@ async def teacher_dashboard(principal: TeacherPrincipal) -> TeacherDashboardResp
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/learners/{learner_id}/analysis",
|
||||
response_model=TeacherLearnerAnalysisResponse,
|
||||
)
|
||||
async def learner_analysis(
|
||||
learner_id: str,
|
||||
principal: TeacherPrincipal,
|
||||
) -> TeacherLearnerAnalysisResponse:
|
||||
"""Return one learner's full teacher-visible session timeline and trend."""
|
||||
sessions, durable = await session_persistence.list_sessions(
|
||||
principal,
|
||||
include_turn_evaluation=True,
|
||||
)
|
||||
if not durable:
|
||||
require_runtime_fallback_allowed("teacher learner analysis")
|
||||
sessions = sorted(store.list(), key=lambda sess: sess.created_at, reverse=True)
|
||||
|
||||
learner_sessions = [sess for sess in sessions if sess.learner_id == learner_id]
|
||||
if not learner_sessions:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="learner not found")
|
||||
|
||||
ordered = sorted(
|
||||
learner_sessions,
|
||||
key=lambda sess: (sess.created_at, sess.session_no, sess.session_id),
|
||||
)
|
||||
review_statuses, _ = await session_persistence.list_session_review_statuses(
|
||||
[sess.session_id for sess in ordered if sess.ended],
|
||||
principal,
|
||||
)
|
||||
summaries = [
|
||||
_summary(sess, review_statuses.get(sess.session_id))
|
||||
for sess in ordered
|
||||
]
|
||||
growth = _build_learner_growth(ordered, limit=None, point_limit=None)[0]
|
||||
pending_reviews = sum(
|
||||
1
|
||||
for item in summaries
|
||||
if item.status == "ended" and item.review_status != "closed"
|
||||
)
|
||||
closed_reviews = sum(1 for item in summaries if item.review_status == "closed")
|
||||
|
||||
return TeacherLearnerAnalysisResponse(
|
||||
source="database" if durable else "runtime",
|
||||
learner_id=learner_id,
|
||||
learner_label=growth.learner_label,
|
||||
total_sessions=len(ordered),
|
||||
active_sessions=sum(1 for sess in ordered if not sess.ended),
|
||||
ended_sessions=sum(1 for sess in ordered if sess.ended),
|
||||
pending_reviews=pending_reviews,
|
||||
closed_reviews=closed_reviews,
|
||||
summary=growth,
|
||||
points=growth.points,
|
||||
stage_breakdown=_stage_breakdown(ordered),
|
||||
sessions=summaries,
|
||||
message=f"{growth.learner_label}의 실제 회기 {len(ordered)}건을 오래된 순서로 표시합니다.",
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/sessions/{session_id}/review-status",
|
||||
response_model=TeacherSessionReviewStatusResponse,
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ _KOREAN_SURNAME_CHARS = (
|
|||
)
|
||||
_KOREAN_FULL_NAME = rf"[{_KOREAN_SURNAME_CHARS}][가-힣]{{1,3}}"
|
||||
_KOREAN_FULL_NAME_BEFORE_SUFFIX = rf"[{_KOREAN_SURNAME_CHARS}][가-힣]{{1,3}}?"
|
||||
_KOREAN_CONTEXTLESS_NAME = rf"[{_KOREAN_SURNAME_CHARS}][가-힣]{{2,3}}"
|
||||
_KOREAN_NAME_STOPWORDS = {
|
||||
"연락",
|
||||
"연락처",
|
||||
|
|
@ -126,7 +127,7 @@ _PII_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_CONTEXTLESS_NAME})"
|
||||
r"(?P<suffix>(?:은|는|이|가|을|를|와|과|에게|한테|라고|이라는))"
|
||||
),
|
||||
),
|
||||
|
|
@ -372,6 +373,71 @@ _MEANS_TERMS = [
|
|||
"약을 얼마나", "몇 알", "몇 정", "투신", "목을 매", "목 매", "번개탄", "농약",
|
||||
"손목을", "칼로", "뛰어내리", "수면제 ", "독", "방법은", "어떻게 죽",
|
||||
]
|
||||
_DISPLAY_PLACEHOLDER_LABELS = {
|
||||
"NAME": "그 이름",
|
||||
"ORG": "그 기관",
|
||||
"PHONE": "연락처",
|
||||
"EMAIL": "이메일",
|
||||
"RRN": "식별번호",
|
||||
"NUMID": "식별번호",
|
||||
"DATE": "그 날짜",
|
||||
"MONEY": "그 금액",
|
||||
"ADDR": "그 주소",
|
||||
}
|
||||
_DISPLAY_PLACEHOLDER_RE = re.compile(
|
||||
r"\[(?P<label>NAME|ORG|PHONE|EMAIL|RRN|NUMID|DATE|MONEY|ADDR)\]"
|
||||
r"(?P<particle>[은는이가을를와과])?"
|
||||
)
|
||||
_DISPLAY_PLACEHOLDER_STREAM_TAIL = 16
|
||||
|
||||
|
||||
def _adjust_particle(value: str, particle: str | None) -> str:
|
||||
if particle is None:
|
||||
return value
|
||||
if particle in ("은", "는"):
|
||||
return value + "은"
|
||||
if particle in ("이", "가"):
|
||||
return value + "이"
|
||||
if particle in ("을", "를"):
|
||||
return value + "을"
|
||||
if particle in ("와", "과"):
|
||||
return value + "과"
|
||||
return value + particle
|
||||
|
||||
|
||||
def humanize_pii_placeholders(text: str) -> str:
|
||||
"""사용자에게 보이는 내담자 응답에서 PII placeholder 토큰을 자연어로 낮춘다."""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
label = match.group("label")
|
||||
replacement = _DISPLAY_PLACEHOLDER_LABELS.get(label, "그 정보")
|
||||
return _adjust_particle(replacement, match.group("particle"))
|
||||
|
||||
return _DISPLAY_PLACEHOLDER_RE.sub(_replace, text)
|
||||
|
||||
|
||||
class PiiPlaceholderStreamSanitizer:
|
||||
"""SSE 토큰 경계를 가로질러 나온 PII placeholder를 사용자 표시 전에 치환한다."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tail = ""
|
||||
|
||||
def feed(self, chunk: str) -> str:
|
||||
if not chunk:
|
||||
return ""
|
||||
self._tail += chunk
|
||||
if len(self._tail) <= _DISPLAY_PLACEHOLDER_STREAM_TAIL:
|
||||
return ""
|
||||
ready = self._tail[:-_DISPLAY_PLACEHOLDER_STREAM_TAIL]
|
||||
self._tail = self._tail[-_DISPLAY_PLACEHOLDER_STREAM_TAIL:]
|
||||
return humanize_pii_placeholders(ready)
|
||||
|
||||
def flush(self) -> str:
|
||||
tail = self._tail
|
||||
self._tail = ""
|
||||
return humanize_pii_placeholders(tail)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
|
@ -406,7 +472,7 @@ def sanitize_client_reply(text: str, *, ideation_stage: int) -> OutputGuardResul
|
|||
blocked = True
|
||||
|
||||
return OutputGuardResult(
|
||||
text=text,
|
||||
text=humanize_pii_placeholders(text),
|
||||
blocked=blocked,
|
||||
needs_regeneration=needs_regen,
|
||||
reasons=reasons,
|
||||
|
|
@ -440,6 +506,8 @@ __all__ = [
|
|||
"CrisisResult",
|
||||
"classify_crisis",
|
||||
"OutputGuardResult",
|
||||
"PiiPlaceholderStreamSanitizer",
|
||||
"humanize_pii_placeholders",
|
||||
"sanitize_client_reply",
|
||||
"clamp_ideation",
|
||||
"crisis_resource",
|
||||
|
|
|
|||
|
|
@ -237,6 +237,7 @@ async def run_turn_generate(
|
|||
|
||||
# 5) 출력 가드레일 — 수단 차단 + ideation 상한
|
||||
guard = guardrail.sanitize_client_reply(reply, ideation_stage=st.ideation_stage)
|
||||
reply = guard.text
|
||||
safety_flagged = guard.blocked or (ctx.crisis is not None and ctx.crisis.escalate)
|
||||
if guard.needs_regeneration:
|
||||
# 수단정보 누출 → 안전 대체 응답으로 치환(1차). 재생성 루프는 후속.
|
||||
|
|
@ -320,6 +321,7 @@ async def run_turn_stream(
|
|||
accumulated = ""
|
||||
flagged = False
|
||||
stream_meta: dict[str, Any] = {}
|
||||
display_sanitizer = guardrail.PiiPlaceholderStreamSanitizer()
|
||||
if ctx.crisis is not None and ctx.crisis.escalate:
|
||||
flagged = True
|
||||
resource = guardrail.crisis_resource()
|
||||
|
|
@ -353,6 +355,9 @@ async def run_turn_stream(
|
|||
if packet.event == ENGINE_GATEWAY_SSE_ERROR:
|
||||
payload = packet.payload
|
||||
detail = payload.detail if isinstance(payload, StreamErrorEvent) else "engine stream error"
|
||||
display_tail = display_sanitizer.flush()
|
||||
if display_tail:
|
||||
yield StreamEvent("token", {"text": display_tail})
|
||||
yield StreamEvent("error", {"detail": detail})
|
||||
return
|
||||
if packet.event == ENGINE_GATEWAY_SSE_DONE:
|
||||
|
|
@ -376,7 +381,14 @@ async def run_turn_stream(
|
|||
accumulated = "…(말을 잇지 못하고 잠시 침묵한다)"
|
||||
break
|
||||
|
||||
yield StreamEvent("token", {"text": text_piece})
|
||||
display_piece = display_sanitizer.feed(text_piece)
|
||||
if display_piece:
|
||||
yield StreamEvent("token", {"text": display_piece})
|
||||
|
||||
if not flagged:
|
||||
display_tail = display_sanitizer.flush()
|
||||
if display_tail:
|
||||
yield StreamEvent("token", {"text": display_tail})
|
||||
|
||||
latency_ms = int((time.perf_counter() - started) * 1000)
|
||||
await _record_llm_audit(
|
||||
|
|
@ -415,8 +427,14 @@ async def run_turn_stream(
|
|||
},
|
||||
)
|
||||
except EngineGatewaySseDecodeError as e:
|
||||
display_tail = display_sanitizer.flush()
|
||||
if display_tail:
|
||||
yield StreamEvent("token", {"text": display_tail})
|
||||
yield StreamEvent("error", {"detail": str(e)})
|
||||
except EngineError as e:
|
||||
display_tail = display_sanitizer.flush()
|
||||
if display_tail:
|
||||
yield StreamEvent("token", {"text": display_tail})
|
||||
yield StreamEvent("error", {"detail": str(e)})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ def build_learner_growth(
|
|||
*,
|
||||
learner_label: Callable[[str], str],
|
||||
limit: int | None = None,
|
||||
point_limit: int | None = 6,
|
||||
) -> list[LearnerGrowthMetrics]:
|
||||
grouped: dict[str, list[InProcSession]] = {}
|
||||
for sess in sessions:
|
||||
|
|
@ -222,7 +223,7 @@ def build_learner_growth(
|
|||
avg_rapport=avg([value for value in rapport_values if value is not None]),
|
||||
trend=trend,
|
||||
top_techniques=top_techniques,
|
||||
points=points[-6:],
|
||||
points=points if point_limit is None else points[-point_limit:],
|
||||
)
|
||||
)
|
||||
sorted_result = sorted(result, key=lambda item: item.latest_at, reverse=True)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,13 @@ _APPROPRIATENESS_SCORE = {
|
|||
}
|
||||
|
||||
|
||||
def _coerce_error_message(error: BaseException | str) -> str:
|
||||
if isinstance(error, BaseException):
|
||||
message = str(error).strip()
|
||||
return message or error.__class__.__name__
|
||||
return str(error).strip() or "unknown session evaluation error"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CaseContext:
|
||||
case_id: str
|
||||
|
|
@ -114,7 +121,7 @@ class SessionEvaluationWrite:
|
|||
scope=scope,
|
||||
stage=stage,
|
||||
payload={},
|
||||
error=str(error),
|
||||
error=_coerce_error_message(error),
|
||||
)
|
||||
|
||||
def cache_record(self) -> dict[str, Any]:
|
||||
|
|
@ -1254,9 +1261,8 @@ async def load_session_evaluation(
|
|||
session_id,
|
||||
)
|
||||
if row is None:
|
||||
return (
|
||||
_EVALUATION_CACHE.get(session_id) if runtime_fallback_allowed() else None
|
||||
), False
|
||||
cached = _EVALUATION_CACHE.get(session_id) if runtime_fallback_allowed() else None
|
||||
return cached, cached is None
|
||||
return {
|
||||
"status": row["status"],
|
||||
"source": row["source"],
|
||||
|
|
|
|||
|
|
@ -1201,7 +1201,8 @@ def _review_provider_event(event: dict[str, object]) -> ReviewNonverbalEvent | N
|
|||
|
||||
def _review_nonverbal_events(turn: TurnRecord) -> list[ReviewNonverbalEvent]:
|
||||
events: list[ReviewNonverbalEvent] = []
|
||||
if turn.silence_ms is not None and turn.silence_ms >= 1000:
|
||||
has_turn_level_silence = turn.silence_ms is not None and turn.silence_ms >= 1000
|
||||
if has_turn_level_silence:
|
||||
events.append(
|
||||
ReviewNonverbalEvent(
|
||||
kind="silence",
|
||||
|
|
@ -1236,6 +1237,8 @@ def _review_nonverbal_events(turn: TurnRecord) -> list[ReviewNonverbalEvent]:
|
|||
for event in turn.provider_events:
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
if has_turn_level_silence and str(event.get("event_type") or "").strip() == "silence":
|
||||
continue
|
||||
review_event = _review_provider_event(event)
|
||||
if review_event is not None:
|
||||
events.append(review_event)
|
||||
|
|
|
|||
213
apps/api/app/test_eval_routes.py
Normal file
213
apps/api/app/test_eval_routes.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
"""평가 재시도 라우트의 실패 노출 테스트."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from .deps import Principal, Role
|
||||
from .services import evaluator
|
||||
|
||||
eval_routes = importlib.import_module("app.routes.eval")
|
||||
|
||||
|
||||
class ReevaluateSessionRouteTest(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_reevaluate_session_saves_result_error_then_raises(self) -> None:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
sess = _session_stub()
|
||||
result = evaluator.SessionEvaluation(
|
||||
session_id=session_id,
|
||||
stage="정리",
|
||||
scope="session_end",
|
||||
turns_evaluated=1,
|
||||
error="no_structured_output",
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
||||
patch.object(eval_routes.evaluator, "evaluate_session", AsyncMock(return_value=result)),
|
||||
patch.object(
|
||||
eval_routes.session_persistence,
|
||||
"save_session_evaluation",
|
||||
AsyncMock(return_value=True),
|
||||
) as save_evaluation,
|
||||
):
|
||||
with self.assertRaises(HTTPException) as raised:
|
||||
await eval_routes.reevaluate_session(
|
||||
session_id,
|
||||
eval_routes.ReevaluateRequest(scope="session_end"),
|
||||
_teacher_principal(),
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.status_code, 502)
|
||||
self.assertEqual(raised.exception.detail, "no_structured_output")
|
||||
saved_write = save_evaluation.await_args.args[0]
|
||||
self.assertEqual(saved_write.status, "error")
|
||||
self.assertEqual(saved_write.error, "no_structured_output")
|
||||
|
||||
async def test_reevaluate_session_raises_when_result_cannot_be_saved(self) -> None:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
sess = _session_stub()
|
||||
result = evaluator.SessionEvaluation(
|
||||
session_id=session_id,
|
||||
stage="정리",
|
||||
scope="session_end",
|
||||
turns_evaluated=1,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
||||
patch.object(eval_routes.evaluator, "evaluate_session", AsyncMock(return_value=result)),
|
||||
patch.object(
|
||||
eval_routes.session_persistence,
|
||||
"save_session_evaluation",
|
||||
AsyncMock(return_value=False),
|
||||
),
|
||||
):
|
||||
with self.assertRaises(HTTPException) as raised:
|
||||
await eval_routes.reevaluate_session(
|
||||
session_id,
|
||||
eval_routes.ReevaluateRequest(scope="session_end"),
|
||||
_teacher_principal(),
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.status_code, 503)
|
||||
self.assertIn("could not be saved", raised.exception.detail)
|
||||
|
||||
async def test_reevaluate_session_prefers_session_theory_mode(self) -> None:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
sess = _session_stub(theory_mode="cbt", theory_target=["humanistic"])
|
||||
result = evaluator.SessionEvaluation(
|
||||
session_id=session_id,
|
||||
stage="정리",
|
||||
scope="session_end",
|
||||
turns_evaluated=1,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
||||
patch.object(
|
||||
eval_routes.evaluator,
|
||||
"evaluate_session",
|
||||
AsyncMock(return_value=result),
|
||||
) as evaluate_session,
|
||||
patch.object(
|
||||
eval_routes.session_persistence,
|
||||
"save_session_evaluation",
|
||||
AsyncMock(return_value=True),
|
||||
),
|
||||
):
|
||||
await eval_routes.reevaluate_session(
|
||||
session_id,
|
||||
eval_routes.ReevaluateRequest(scope="session_end"),
|
||||
_teacher_principal(),
|
||||
)
|
||||
|
||||
self.assertEqual(evaluate_session.await_args.kwargs["theory_mode"], "cbt")
|
||||
|
||||
async def test_reevaluate_turn_preserves_session_theory_mode(self) -> None:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
sess = _session_stub(theory_mode="cbt", theory_target=["humanistic"], with_turns=True)
|
||||
result = evaluator.TurnEvaluation(turn_seq=1, stage="정리", theory_mode="cbt")
|
||||
|
||||
with (
|
||||
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
||||
patch.object(
|
||||
eval_routes.evaluator,
|
||||
"evaluate_turn",
|
||||
AsyncMock(return_value=result),
|
||||
) as evaluate_turn,
|
||||
):
|
||||
await eval_routes.reevaluate_turn(
|
||||
session_id,
|
||||
eval_routes.TurnReevaluateRequest(turn_seq=1),
|
||||
_teacher_principal(),
|
||||
)
|
||||
|
||||
ctx = evaluate_turn.await_args.args[0]
|
||||
self.assertEqual(ctx.theory_mode, "cbt")
|
||||
|
||||
async def test_get_session_evaluation_exposes_status_error_and_durable_flag(self) -> None:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
sess = _session_stub()
|
||||
record = {
|
||||
"status": "error",
|
||||
"source": "engine",
|
||||
"scope": "session_end",
|
||||
"stage": "정리",
|
||||
"payload": {},
|
||||
"error": "session evaluation timeout after 45s",
|
||||
"updated_at": "2026-07-01T00:00:00Z",
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(eval_routes, "_load_session_or_404", AsyncMock(return_value=sess)),
|
||||
patch.object(
|
||||
eval_routes.session_persistence,
|
||||
"load_session_evaluation",
|
||||
AsyncMock(return_value=(record, True)),
|
||||
),
|
||||
):
|
||||
response = await eval_routes.get_session_evaluation(session_id, _teacher_principal())
|
||||
|
||||
self.assertEqual(response.status, "error")
|
||||
self.assertEqual(response.error, "session evaluation timeout after 45s")
|
||||
self.assertTrue(response.durable)
|
||||
self.assertEqual(response.deep, {})
|
||||
|
||||
|
||||
def _session_stub(
|
||||
*,
|
||||
theory_mode: str = "humanistic",
|
||||
theory_target: list[str] | None = None,
|
||||
with_turns: bool = False,
|
||||
):
|
||||
selected_theory_mode = theory_mode
|
||||
selected_theory_target = theory_target or []
|
||||
|
||||
class SessionStub:
|
||||
session_id = "00000000-0000-0000-0000-00000000e222"
|
||||
case_id = "00000000-0000-0000-0000-00000000c333"
|
||||
learner_id = "00000000-0000-0000-0000-000000000111"
|
||||
theory_mode = selected_theory_mode
|
||||
persona = SimpleNamespace(theory_target=selected_theory_target)
|
||||
state = SimpleNamespace(stage=SimpleNamespace(value="정리"))
|
||||
turns = (
|
||||
[
|
||||
SimpleNamespace(
|
||||
turn_seq=1,
|
||||
speaker="counselor",
|
||||
text="상담자 발화",
|
||||
text_masked="상담자 발화",
|
||||
),
|
||||
SimpleNamespace(
|
||||
turn_seq=2,
|
||||
speaker="client",
|
||||
text="내담자 응답",
|
||||
text_masked="내담자 응답",
|
||||
),
|
||||
]
|
||||
if with_turns
|
||||
else []
|
||||
)
|
||||
|
||||
def masked_turns(self):
|
||||
return [{"speaker": "counselor", "text_masked": "상담자 발화"}]
|
||||
|
||||
return SessionStub()
|
||||
|
||||
|
||||
def _teacher_principal() -> Principal:
|
||||
return Principal(
|
||||
user_id="00000000-0000-0000-0000-000000000222",
|
||||
role=Role.TEACHER,
|
||||
cohort_ids=[],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -175,6 +175,17 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
|
|||
self.assertEqual(write.payload, {})
|
||||
self.assertEqual(write.error, "engine timeout")
|
||||
|
||||
def test_session_evaluation_write_from_error_names_empty_exception(self) -> None:
|
||||
write = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id="session-1",
|
||||
learner_id="learner-1",
|
||||
scope="session_end",
|
||||
stage="explore",
|
||||
error=TimeoutError(),
|
||||
)
|
||||
|
||||
self.assertEqual(write.error, "TimeoutError")
|
||||
|
||||
def test_rebuild_turn_evaluation_restores_review_shape(self) -> None:
|
||||
rebuilt = session_persistence._rebuild_turn_evaluations(
|
||||
[("11111111-1111-1111-1111-111111111111", 2, "탐색")],
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
|
@ -181,6 +182,52 @@ class NotificationTriggerTest(unittest.IsolatedAsyncioTestCase):
|
|||
save_evaluation.assert_awaited_once()
|
||||
enqueue_review.assert_awaited_once_with(session_id=sess.session_id)
|
||||
|
||||
async def test_session_evaluation_timeout_records_actionable_error(self) -> None:
|
||||
sess = _ended_session()
|
||||
|
||||
with (
|
||||
patched_settings(session_evaluation_timeout=1.0),
|
||||
patch.object(
|
||||
sessions.evaluator,
|
||||
"evaluate_session",
|
||||
AsyncMock(side_effect=asyncio.TimeoutError()),
|
||||
),
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"save_session_evaluation",
|
||||
AsyncMock(return_value=True),
|
||||
) as save_evaluation,
|
||||
patch.object(
|
||||
sessions.notifications,
|
||||
"enqueue_session_review_ready",
|
||||
AsyncMock(),
|
||||
),
|
||||
):
|
||||
await sessions._generate_and_save_session_evaluation(sess)
|
||||
|
||||
saved_write = save_evaluation.await_args.args[0]
|
||||
self.assertEqual(saved_write.status, "error")
|
||||
self.assertEqual(saved_write.error, "session evaluation timeout after 1s")
|
||||
|
||||
async def test_scheduled_session_evaluation_task_crash_is_observed(self) -> None:
|
||||
sess = _ended_session()
|
||||
|
||||
async def boom(_sess: InProcSession) -> None:
|
||||
raise RuntimeError("background crash")
|
||||
|
||||
with patch.object(sessions, "_generate_and_save_session_evaluation", boom):
|
||||
with self.assertLogs(sessions.logger, level="ERROR") as logs:
|
||||
task = sessions._schedule_session_evaluation(sess)
|
||||
self.assertIsNotNone(task)
|
||||
assert task is not None
|
||||
with self.assertRaises(RuntimeError):
|
||||
await task
|
||||
|
||||
self.assertTrue(
|
||||
any("session evaluation background task crashed" in line for line in logs.output),
|
||||
logs.output,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -87,16 +87,17 @@ def _assert_masked_ko_pii_present(test: unittest.TestCase, value: object) -> Non
|
|||
|
||||
|
||||
class CaptureGenerateEngine:
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, text: str = "Masked engine reply.") -> None:
|
||||
self.request = None
|
||||
self.payload: dict[str, Any] | None = None
|
||||
self._payload_builder = EngineClient(base_url="http://engine.test")
|
||||
self.text = text
|
||||
|
||||
async def generate(self, req):
|
||||
self.request = req
|
||||
self.payload = self._payload_builder._payload(req)
|
||||
return GenerateResponse(
|
||||
text="Masked engine reply.",
|
||||
text=self.text,
|
||||
model="fake-model",
|
||||
provider="fake-provider",
|
||||
tokens_in=3,
|
||||
|
|
@ -109,16 +110,18 @@ class CaptureStreamEngine:
|
|||
engine_mode = "fake-provider"
|
||||
default_model = "fake-model"
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, chunks: list[str] | None = None) -> None:
|
||||
self.request = None
|
||||
self.payload: dict[str, Any] | None = None
|
||||
self._payload_builder = EngineClient(base_url="http://engine.test")
|
||||
self.chunks = chunks or ["Masked stream reply."]
|
||||
|
||||
async def stream(self, req):
|
||||
self.request = req
|
||||
self.payload = self._payload_builder._payload(req)
|
||||
yield "event: token"
|
||||
yield 'data: {"text":"Masked stream reply."}'
|
||||
for chunk in self.chunks:
|
||||
yield "event: token"
|
||||
yield "data: " + json.dumps({"text": chunk}, ensure_ascii=False)
|
||||
yield "event: done"
|
||||
yield (
|
||||
'data: {"provider":"fake-provider","model":"fake-model",'
|
||||
|
|
@ -200,6 +203,15 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertNotIn("NAME", masked.entities)
|
||||
self.assertNotIn("ORG", masked.entities)
|
||||
|
||||
def test_mask_pii_does_not_mask_career_topic_as_name(self) -> None:
|
||||
masked = guardrail.mask_pii("아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요.")
|
||||
|
||||
self.assertEqual(
|
||||
masked.text_masked,
|
||||
"아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요.",
|
||||
)
|
||||
self.assertNotIn("NAME", masked.entities)
|
||||
|
||||
def test_prepare_turn_masks_korean_pii_from_engine_messages(self) -> None:
|
||||
ctx = orchestrator.prepare_turn(
|
||||
session_id="masking-session",
|
||||
|
|
@ -316,7 +328,11 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase):
|
|||
)
|
||||
]
|
||||
|
||||
self.assertEqual([event.event for event in events], ["token", "done"])
|
||||
self.assertEqual(events[-1].event, "done")
|
||||
self.assertEqual(
|
||||
"".join(str(event.data.get("text", "")) for event in events if event.event == "token"),
|
||||
"Masked stream reply.",
|
||||
)
|
||||
self.assertIsNotNone(engine.request)
|
||||
self.assertIsNotNone(engine.payload)
|
||||
_assert_no_raw_pii(self, engine.request.messages)
|
||||
|
|
@ -359,6 +375,53 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase):
|
|||
_assert_masked_ko_pii_present(self, engine.payload)
|
||||
_assert_no_raw_ko_pii(self, audit_payloads)
|
||||
|
||||
async def test_run_turn_generate_humanizes_masked_placeholder_reply(self) -> None:
|
||||
ctx = orchestrator.prepare_turn(
|
||||
session_id="masking-session",
|
||||
case_id="masking-case",
|
||||
card=persona.P1,
|
||||
state=_initial_state(),
|
||||
learner_text="진로 이야기를 이어가고 싶어요.",
|
||||
)
|
||||
engine = CaptureGenerateEngine(
|
||||
text="아직 정해진 건 하나도 없어요. [NAME]는 그대로고, 엄마한테 뭐라고 말할지도 모르겠고요."
|
||||
)
|
||||
|
||||
result = await orchestrator.run_turn_generate(
|
||||
ctx,
|
||||
engine, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
self.assertNotIn("[NAME]", result.client_reply or "")
|
||||
self.assertIn("그 이름은 그대로고", result.client_reply or "")
|
||||
|
||||
async def test_run_turn_stream_humanizes_split_masked_placeholder_reply(self) -> None:
|
||||
ctx = orchestrator.prepare_turn(
|
||||
session_id="masking-session",
|
||||
case_id="masking-case",
|
||||
card=persona.P1,
|
||||
state=_initial_state(),
|
||||
learner_text="진로 이야기를 이어가고 싶어요.",
|
||||
)
|
||||
engine = CaptureStreamEngine(
|
||||
chunks=["아직 정해진 건 하나도 없어요. ", "[NA", "ME]는", " 그대로고요."]
|
||||
)
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in orchestrator.run_turn_stream(
|
||||
ctx,
|
||||
engine, # type: ignore[arg-type]
|
||||
)
|
||||
]
|
||||
streamed = "".join(
|
||||
str(event.data.get("text", "")) for event in events if event.event == "token"
|
||||
)
|
||||
|
||||
self.assertNotIn("[NAME]", streamed)
|
||||
self.assertIn("그 이름은 그대로고요.", streamed)
|
||||
self.assertEqual(events[-1].event, "done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class PiiMaskingEvalTests(unittest.TestCase):
|
|||
def test_fixture_cases_are_valid_json_list(self) -> None:
|
||||
cases = load_cases(FIXTURE_PATH)
|
||||
|
||||
self.assertEqual(len(cases), 15)
|
||||
self.assertEqual(len(cases), 16)
|
||||
self.assertTrue(all(case.get("id") for case in cases))
|
||||
self.assertTrue(all(case.get("text") for case in cases))
|
||||
self.assertTrue(all(case.get("locale") == "ko-KR" for case in cases))
|
||||
|
|
@ -80,13 +80,13 @@ class PiiMaskingEvalTests(unittest.TestCase):
|
|||
self.assertEqual(report["data_source"], "local_fixture")
|
||||
self.assertFalse(report["evidence_text_included"])
|
||||
self.assertTrue(report["passed"], report)
|
||||
self.assertEqual(report["cases_total"], 15)
|
||||
self.assertEqual(report["cases_passed"], 15)
|
||||
self.assertEqual(report["cases_total"], 16)
|
||||
self.assertEqual(report["cases_passed"], 16)
|
||||
self.assertEqual(report["cases_failed"], 0)
|
||||
self.assertEqual(report["expected_entity_recall"], 1.0)
|
||||
self.assertEqual(report["forbidden_substring_removal"], 1.0)
|
||||
self.assertEqual(report["unexpected_entity_violations"], 0)
|
||||
self.assertEqual(report["by_source"]["synthetic"]["cases_passed"], 15)
|
||||
self.assertEqual(report["by_source"]["synthetic"]["cases_passed"], 16)
|
||||
self.assertEqual(set(report["by_category"]), EXPECTED_CATEGORIES)
|
||||
self.assertEqual(report["by_severity"]["critical"]["cases_passed"], 2)
|
||||
for result in report["results"]:
|
||||
|
|
@ -105,6 +105,9 @@ class PiiMaskingEvalTests(unittest.TestCase):
|
|||
"저는 최하늘입니다.": "저는 [NAME]입니다.",
|
||||
"김서연입니다.": "[NAME]입니다.",
|
||||
"이름은 중요하지 않고 상담 내용만 이야기하고 싶어요.": "이름은 중요하지 않고 상담 내용만 이야기하고 싶어요.",
|
||||
"아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요.": (
|
||||
"아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요."
|
||||
),
|
||||
}
|
||||
|
||||
for raw, expected in cases.items():
|
||||
|
|
@ -176,7 +179,7 @@ class PiiMaskingEvalTests(unittest.TestCase):
|
|||
|
||||
report = json.loads(completed.stdout)
|
||||
self.assertTrue(report["passed"])
|
||||
self.assertEqual(report["cases_total"], 15)
|
||||
self.assertEqual(report["cases_total"], 16)
|
||||
self.assertFalse(report["evidence_text_included"])
|
||||
self.assertEqual(set(report["by_category"]), EXPECTED_CATEGORIES)
|
||||
self.assertTrue(all("masked_text" not in result for result in report["results"]))
|
||||
|
|
|
|||
|
|
@ -160,6 +160,152 @@ class LearnerSessionIdorTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertIn("does not belong", caught.exception.detail)
|
||||
review_ready.assert_not_awaited()
|
||||
|
||||
async def test_super_admin_primary_learner_can_review_other_learner_session(self) -> None:
|
||||
owner = _principal(
|
||||
user_id="00000000-0000-0000-0000-000000000103",
|
||||
)
|
||||
super_admin = Principal(
|
||||
user_id="00000000-0000-0000-0000-000000000303",
|
||||
role=Role.LEARNER,
|
||||
admin_access=True,
|
||||
super_admin=True,
|
||||
email="yunchan@twentyoz.kr",
|
||||
display_name="Yun Chan",
|
||||
)
|
||||
sess = _session(
|
||||
session_id="00000000-0000-0000-0000-00000000a103",
|
||||
learner_id=owner.user_id,
|
||||
)
|
||||
store.put(sess)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session",
|
||||
AsyncMock(return_value=None),
|
||||
) as load_session,
|
||||
patch.object(sessions.turn_runtime, "runtime_fallback_allowed", return_value=True),
|
||||
):
|
||||
loaded, review_principal = await sessions._load_review_session_or_404(
|
||||
sess.session_id,
|
||||
super_admin,
|
||||
)
|
||||
|
||||
self.assertIs(loaded, sess)
|
||||
self.assertEqual(review_principal.role, Role.ADMIN)
|
||||
self.assertEqual(load_session.await_count, 2)
|
||||
|
||||
async def test_super_admin_primary_learner_review_loaders_use_admin_principal(self) -> None:
|
||||
owner = _principal(
|
||||
user_id="00000000-0000-0000-0000-000000000105",
|
||||
)
|
||||
super_admin = Principal(
|
||||
user_id="00000000-0000-0000-0000-000000000305",
|
||||
role=Role.LEARNER,
|
||||
admin_access=True,
|
||||
super_admin=True,
|
||||
email="yunchan@twentyoz.kr",
|
||||
display_name="Yun Chan",
|
||||
)
|
||||
sess = _session(
|
||||
session_id="00000000-0000-0000-0000-00000000a105",
|
||||
learner_id=owner.user_id,
|
||||
)
|
||||
sess.ended = True
|
||||
sess.ended_at = 1_800_000_120.0
|
||||
sess.turns.extend(
|
||||
[
|
||||
_turn(seq=1, speaker="counselor", text="admin-visible learner turn"),
|
||||
_turn(seq=2, speaker="client", text="admin-visible client turn"),
|
||||
]
|
||||
)
|
||||
store.put(sess)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(sessions.turn_runtime, "runtime_fallback_allowed", return_value=True),
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session_evaluation",
|
||||
AsyncMock(return_value=(None, False)),
|
||||
) as load_session_evaluation,
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_case_worksheet",
|
||||
AsyncMock(return_value=(None, False)),
|
||||
) as load_case_worksheet,
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session_review_status",
|
||||
AsyncMock(
|
||||
return_value=(
|
||||
{
|
||||
"status": "viewed",
|
||||
"reviewer_id": "00000000-0000-0000-0000-000000000999",
|
||||
},
|
||||
True,
|
||||
)
|
||||
),
|
||||
) as load_session_review_status,
|
||||
):
|
||||
response = await sessions.get_session_review(sess.session_id, super_admin)
|
||||
|
||||
load_session_evaluation.assert_awaited_once()
|
||||
load_case_worksheet.assert_awaited_once()
|
||||
load_session_review_status.assert_awaited_once()
|
||||
|
||||
evaluation_principal = load_session_evaluation.await_args.args[1]
|
||||
worksheet_principal = load_case_worksheet.await_args.args[1]
|
||||
review_status_principal = load_session_review_status.await_args.args[1]
|
||||
|
||||
self.assertEqual(load_session_evaluation.await_args.args[0], sess.session_id)
|
||||
self.assertEqual(load_case_worksheet.await_args.args[0], sess.session_id)
|
||||
self.assertEqual(load_session_review_status.await_args.args[0], sess.session_id)
|
||||
self.assertIs(evaluation_principal, worksheet_principal)
|
||||
self.assertIs(evaluation_principal, review_status_principal)
|
||||
self.assertEqual(evaluation_principal.user_id, super_admin.user_id)
|
||||
self.assertEqual(evaluation_principal.role, Role.ADMIN)
|
||||
self.assertTrue(evaluation_principal.super_admin)
|
||||
self.assertEqual(response.session_id, sess.session_id)
|
||||
self.assertIsNotNone(response.teacherReview)
|
||||
self.assertEqual(response.teacherReview.status, "viewed")
|
||||
|
||||
async def test_admin_access_flag_alone_does_not_review_other_learner_session(self) -> None:
|
||||
owner = _principal(
|
||||
user_id="00000000-0000-0000-0000-000000000104",
|
||||
)
|
||||
delegated_admin = Principal(
|
||||
user_id="00000000-0000-0000-0000-000000000304",
|
||||
role=Role.LEARNER,
|
||||
admin_access=True,
|
||||
super_admin=False,
|
||||
email="delegate@example.test",
|
||||
display_name="Delegated Admin",
|
||||
)
|
||||
sess = _session(
|
||||
session_id="00000000-0000-0000-0000-00000000a104",
|
||||
learner_id=owner.user_id,
|
||||
)
|
||||
store.put(sess)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(sessions.turn_runtime, "runtime_fallback_allowed", return_value=True),
|
||||
):
|
||||
with self.assertRaises(HTTPException) as caught:
|
||||
await sessions._load_review_session_or_404(sess.session_id, delegated_admin)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 403)
|
||||
self.assertIn("does not belong", caught.exception.detail)
|
||||
|
||||
async def test_session_detail_filters_evaluator_only_turns(self) -> None:
|
||||
owner = _principal(
|
||||
user_id="00000000-0000-0000-0000-000000000111",
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ class RuntimeFallbackPolicyTest(unittest.IsolatedAsyncioTestCase):
|
|||
"has_engine_config": False,
|
||||
"has_session_columns": False,
|
||||
"has_state_columns": False,
|
||||
"has_turn_provider_events": False,
|
||||
"has_turn_voice_metadata_columns": False,
|
||||
"has_session_review_worksheet_columns": False,
|
||||
"has_stage_defs": False,
|
||||
"has_admin_health_event": False,
|
||||
|
|
@ -185,6 +185,50 @@ class RuntimeFallbackPolicyTest(unittest.IsolatedAsyncioTestCase):
|
|||
|
||||
self.assertIn("runtime DB schema is incomplete", str(caught.exception))
|
||||
|
||||
async def test_runtime_readiness_requires_all_turn_voice_metadata_columns(self) -> None:
|
||||
class VoiceMetadataDriftConn:
|
||||
def __init__(self) -> None:
|
||||
self.query = ""
|
||||
|
||||
async def fetchrow(self, query: str, *args, **kwargs):
|
||||
self.query = query
|
||||
return {
|
||||
"has_user_columns": True,
|
||||
"has_persona_triggers": True,
|
||||
"has_persona_voice_map": True,
|
||||
"has_auth_session": True,
|
||||
"has_preferences": True,
|
||||
"has_engine_config": True,
|
||||
"has_session_columns": True,
|
||||
"has_state_columns": True,
|
||||
"has_turn_voice_metadata_columns": False,
|
||||
"has_session_review_worksheet_columns": True,
|
||||
"has_stage_defs": True,
|
||||
"has_admin_health_event": True,
|
||||
"has_admin_health_daily_rollup": True,
|
||||
"has_admin_health_daily_rollup_columns": True,
|
||||
"has_support_ticket": True,
|
||||
"has_support_ticket_duplicate_columns": True,
|
||||
"has_learner_prepost_measure": True,
|
||||
"has_admin_health_event_policies": True,
|
||||
"has_admin_health_daily_rollup_policies": True,
|
||||
"has_support_ticket_policies": True,
|
||||
"has_learner_prepost_measure_policies": True,
|
||||
"has_session_write_policies": True,
|
||||
"removed_old_session_policy": True,
|
||||
"has_turn_write_policies": True,
|
||||
"removed_old_turn_policy": True,
|
||||
}
|
||||
|
||||
conn = VoiceMetadataDriftConn()
|
||||
|
||||
ready = await auth_sessions._runtime_tables_ready(conn)
|
||||
|
||||
self.assertFalse(ready)
|
||||
for column in ("audio_ref", "silence_ms", "speech_rate", "barge_in", "provider_events"):
|
||||
self.assertIn(column, conn.query)
|
||||
self.assertIn("HAVING count(*) = 5", conn.query)
|
||||
|
||||
async def test_staging_uses_env_engine_config_when_row_missing(self) -> None:
|
||||
class EmptyConfigConn:
|
||||
async def fetchrow(self, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -447,6 +447,49 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase):
|
|||
legacy_history = await sessions.list_live_coach_history(sess.session_id, principal)
|
||||
self.assertIsNone(legacy_history.events[0].stage)
|
||||
|
||||
async def test_live_coach_persistence_failure_is_not_swallowed(self) -> None:
|
||||
principal = _principal()
|
||||
sess = _session(principal)
|
||||
suggestion = live_coach.LiveCoachSuggestion(
|
||||
status="ready",
|
||||
tone="pos",
|
||||
focus="emotion",
|
||||
title="감정 반영이 선명합니다",
|
||||
message="학습자가 내담자의 감정을 먼저 되짚었습니다.",
|
||||
)
|
||||
save_error = sessions.HTTPException(
|
||||
status_code=sessions.status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="live coach event save persistence unavailable; runtime fallback is disabled in prod",
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
sessions,
|
||||
"_retrieve_live_coach_grounding",
|
||||
AsyncMock(return_value=[]),
|
||||
), patch.object(
|
||||
sessions.live_coach,
|
||||
"generate_live_coaching",
|
||||
AsyncMock(return_value=suggestion),
|
||||
), patch.object(
|
||||
session_persistence,
|
||||
"save_live_coach_event",
|
||||
AsyncMock(side_effect=save_error),
|
||||
) as save_event:
|
||||
with self.assertRaises(sessions.HTTPException) as caught:
|
||||
await sessions.live_coach_turn(
|
||||
sess.session_id,
|
||||
sessions.LiveCoachRequest(
|
||||
learner_text="그 마음이 컸겠네요.",
|
||||
client_reply="네, 아무도 몰라주는 것 같았어요.",
|
||||
turn_seq=1,
|
||||
),
|
||||
principal,
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 503)
|
||||
self.assertIn("live coach event save", str(caught.exception.detail))
|
||||
save_event.assert_awaited_once()
|
||||
|
||||
async def test_live_coach_event_normalizes_legacy_stage_values(self) -> None:
|
||||
suggestion = live_coach.LiveCoachSuggestion(
|
||||
status="degraded",
|
||||
|
|
@ -987,6 +1030,11 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase):
|
|||
"category": "speech_activity",
|
||||
"start_ms": 100,
|
||||
},
|
||||
{
|
||||
"event_type": "silence",
|
||||
"category": "timing",
|
||||
"duration_ms": 1234,
|
||||
},
|
||||
{
|
||||
"event_type": "background_noise",
|
||||
"category": "audio_quality",
|
||||
|
|
@ -1021,6 +1069,7 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase):
|
|||
)
|
||||
self.assertEqual(learner_turn.nonverbal[0].label, "침묵")
|
||||
self.assertEqual(learner_turn.nonverbal[0].detail, "1.2초")
|
||||
self.assertEqual([event.kind for event in learner_turn.nonverbal].count("silence"), 1)
|
||||
self.assertEqual(learner_turn.nonverbal[1].detail, "분당 420자")
|
||||
self.assertEqual(learner_turn.nonverbal[4].label, "음성 단서")
|
||||
self.assertEqual(learner_turn.nonverbal[4].detail, "한숨 감지 · 신뢰도 82%")
|
||||
|
|
@ -1191,6 +1240,66 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertIn("주호소", response.teacherReview.worksheetNote)
|
||||
self.assertEqual(response.teacherReview.worksheetReviewedAt, "2026-06-27T10:05:00Z")
|
||||
|
||||
async def test_review_surfaces_session_evaluation_error_record(self) -> None:
|
||||
principal = _principal()
|
||||
sess = _session(principal)
|
||||
sess.ended = True
|
||||
sess.ended_at = sess.created_at + 180
|
||||
sess.turns.extend(
|
||||
[
|
||||
TurnRecord(
|
||||
turn_seq=1,
|
||||
speaker="learner",
|
||||
stage=sess.state.stage.value,
|
||||
text="많이 지쳐 보였어요. 지금 제일 버거운 마음이 뭔가요?",
|
||||
text_masked="많이 지쳐 보였어요. 지금 제일 버거운 마음이 뭔가요?",
|
||||
created_at=sess.created_at + 1,
|
||||
),
|
||||
TurnRecord(
|
||||
turn_seq=2,
|
||||
speaker="client",
|
||||
stage=sess.state.stage.value,
|
||||
text="그냥 아무것도 하고 싶지 않아요.",
|
||||
text_masked="그냥 아무것도 하고 싶지 않아요.",
|
||||
created_at=sess.created_at + 2,
|
||||
),
|
||||
]
|
||||
)
|
||||
error_record = session_persistence.SessionEvaluationWrite.from_error(
|
||||
session_id=sess.session_id,
|
||||
learner_id=principal.user_id,
|
||||
scope="session_end",
|
||||
stage="정리",
|
||||
error="session evaluation timeout after 45s",
|
||||
).cache_record()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session",
|
||||
AsyncMock(return_value=sess),
|
||||
),
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_case_worksheet",
|
||||
AsyncMock(return_value=(None, False)),
|
||||
),
|
||||
patch.object(
|
||||
sessions.session_persistence,
|
||||
"load_session_evaluation",
|
||||
AsyncMock(return_value=(error_record, True)),
|
||||
),
|
||||
):
|
||||
response = await sessions.get_session_review(sess.session_id, principal)
|
||||
|
||||
self.assertFalse(response.reviewReady)
|
||||
self.assertTrue(response.degraded)
|
||||
self.assertEqual(response.supervisorState, "평가 실패")
|
||||
self.assertIn("session evaluation timeout after 45s", response.summary)
|
||||
self.assertEqual(response.rubric, [])
|
||||
self.assertEqual(response.goodMoments, [])
|
||||
self.assertEqual(response.growthPoints, [])
|
||||
|
||||
async def test_learner_can_save_case_formulation_worksheet(self) -> None:
|
||||
principal = _principal()
|
||||
sess = _session(principal)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from __future__ import annotations
|
|||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from .deps import Principal, Role
|
||||
from .routes import teacher
|
||||
from .services import state_machine
|
||||
|
|
@ -244,6 +246,88 @@ class TeacherDashboardGrowthTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertIn("보호요인", response.worksheet_note)
|
||||
self.assertEqual(response.worksheet_reviewed_at, "2026-06-27T10:05:00Z")
|
||||
|
||||
async def test_learner_analysis_returns_full_ordered_history(self) -> None:
|
||||
learner_id = "00000000-0000-0000-0000-000000000111"
|
||||
other_learner_id = "00000000-0000-0000-0000-000000000222"
|
||||
learner_sessions = [
|
||||
_session(
|
||||
session_id=f"analysis-session-{idx}",
|
||||
session_no=idx,
|
||||
learner_id=learner_id,
|
||||
score="pos" if idx >= 5 else "neutral",
|
||||
rapport=0.1 * idx,
|
||||
technique="reflection" if idx % 2 else "open question",
|
||||
created_at=1_000.0 + idx,
|
||||
)
|
||||
for idx in range(1, 8)
|
||||
]
|
||||
sessions = [
|
||||
learner_sessions[6],
|
||||
_session(
|
||||
session_id="analysis-other-session",
|
||||
session_no=1,
|
||||
learner_id=other_learner_id,
|
||||
score="warn",
|
||||
rapport=-0.2,
|
||||
technique="advice",
|
||||
created_at=1_500.0,
|
||||
),
|
||||
*learner_sessions[:6],
|
||||
]
|
||||
principal = _principal()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
teacher.session_persistence,
|
||||
"list_sessions",
|
||||
AsyncMock(return_value=(sessions, True)),
|
||||
) as list_sessions,
|
||||
patch.object(
|
||||
teacher.session_persistence,
|
||||
"list_session_review_statuses",
|
||||
AsyncMock(
|
||||
return_value=(
|
||||
{
|
||||
"analysis-session-2": {
|
||||
"session_id": "analysis-session-2",
|
||||
"status": "closed",
|
||||
"note": "확인 완료",
|
||||
"reviewed_at": "2026-06-27T10:00:00Z",
|
||||
}
|
||||
},
|
||||
True,
|
||||
)
|
||||
),
|
||||
),
|
||||
):
|
||||
response = await teacher.learner_analysis(learner_id, principal)
|
||||
|
||||
list_sessions.assert_awaited_once_with(principal, include_turn_evaluation=True)
|
||||
self.assertEqual(response.source, "database")
|
||||
self.assertEqual(response.learner_id, learner_id)
|
||||
self.assertEqual(response.total_sessions, 7)
|
||||
self.assertEqual(response.ended_sessions, 7)
|
||||
self.assertEqual(response.pending_reviews, 6)
|
||||
self.assertEqual(response.closed_reviews, 1)
|
||||
self.assertEqual([session.session_no for session in response.sessions], [1, 2, 3, 4, 5, 6, 7])
|
||||
self.assertEqual([point.session_no for point in response.points], [1, 2, 3, 4, 5, 6, 7])
|
||||
self.assertEqual(response.sessions[1].review_status, "closed")
|
||||
explore = next(item for item in response.stage_breakdown if item.stage == "탐색")
|
||||
self.assertEqual(explore.sessions, 7)
|
||||
self.assertEqual(explore.turns, 14)
|
||||
|
||||
async def test_learner_analysis_returns_404_for_out_of_scope_learner(self) -> None:
|
||||
principal = _principal()
|
||||
with patch.object(
|
||||
teacher.session_persistence,
|
||||
"list_sessions",
|
||||
AsyncMock(return_value=([], True)),
|
||||
):
|
||||
with self.assertRaises(HTTPException) as raised:
|
||||
await teacher.learner_analysis("missing-learner", principal)
|
||||
|
||||
self.assertEqual(raised.exception.status_code, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue