269 lines
8.5 KiB
Python
269 lines
8.5 KiB
Python
"""Session-level learning metrics shared by learner and teacher dashboards."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Callable
|
|
|
|
from ..store import InProcSession
|
|
from .evaluation_contract import GROWTH_APPROPRIATENESS_SCORE_01
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SessionGrowthPoint:
|
|
session_id: str
|
|
session_no: int
|
|
persona_code: str
|
|
stage: str
|
|
started_at: str
|
|
ended_at: str | None
|
|
score: float | None = None
|
|
rapport: float | None = None
|
|
technique_count: int = 0
|
|
watch_count: int = 0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LearnerGrowthMetrics:
|
|
learner_id: str
|
|
learner_label: str
|
|
sessions: int
|
|
ended_sessions: int
|
|
latest_at: str
|
|
first_score: float | None = None
|
|
latest_score: float | None = None
|
|
score_delta: float | None = None
|
|
avg_score: float | None = None
|
|
avg_rapport: float | None = None
|
|
trend: str = "insufficient"
|
|
top_techniques: list[str] = field(default_factory=list)
|
|
points: list[SessionGrowthPoint] = field(default_factory=list)
|
|
|
|
|
|
def iso_datetime(ts: float | None) -> str | None:
|
|
if ts is None:
|
|
return None
|
|
return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def session_activity_time(sess: InProcSession) -> float:
|
|
return sess.ended_at or sess.created_at
|
|
|
|
|
|
def safe_float(value: object) -> float | None:
|
|
try:
|
|
return float(value) # type: ignore[arg-type]
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def avg(values: list[float]) -> float | None:
|
|
if not values:
|
|
return None
|
|
return round(sum(values) / len(values), 3)
|
|
|
|
|
|
def turn_eval(turn: Any) -> dict[str, Any] | None:
|
|
ev = getattr(turn, "evaluation", None)
|
|
return ev if isinstance(ev, dict) else None
|
|
|
|
|
|
def turn_score(ev: dict[str, Any]) -> float | None:
|
|
if str(ev.get("error") or "").strip():
|
|
return None
|
|
raw = str(ev.get("appropriateness") or "").strip().lower()
|
|
return GROWTH_APPROPRIATENESS_SCORE_01.get(raw)
|
|
|
|
|
|
def turn_rapport(ev: dict[str, Any]) -> float | None:
|
|
value = safe_float(ev.get("rapport_signal"))
|
|
if value is None:
|
|
return None
|
|
return max(-1.0, min(1.0, value))
|
|
|
|
|
|
def turn_technique_label(item: object) -> str | None:
|
|
if isinstance(item, dict):
|
|
label = (
|
|
item.get("label_ko")
|
|
or item.get("label")
|
|
or item.get("name")
|
|
or item.get("id")
|
|
or item.get("code")
|
|
)
|
|
else:
|
|
label = item
|
|
if label is None:
|
|
return None
|
|
text = str(label).strip()
|
|
return text or None
|
|
|
|
|
|
def turn_techniques(ev: dict[str, Any]) -> list[str]:
|
|
raw = ev.get("techniques")
|
|
if not isinstance(raw, list):
|
|
return []
|
|
labels: list[str] = []
|
|
for item in raw:
|
|
label = turn_technique_label(item)
|
|
if label:
|
|
labels.append(label)
|
|
return labels
|
|
|
|
|
|
def turn_feedback_note(ev: dict[str, Any]) -> str | None:
|
|
if str(ev.get("error") or "").strip():
|
|
return None
|
|
raw = ev.get("appropriateness_note")
|
|
if raw is None:
|
|
return None
|
|
text = str(raw).strip()
|
|
return text or None
|
|
|
|
|
|
def session_growth_point(sess: InProcSession) -> SessionGrowthPoint:
|
|
scores: list[float] = []
|
|
rapports: list[float] = []
|
|
technique_count = 0
|
|
watch_count = 0
|
|
for turn in sess.turns:
|
|
if turn.speaker != "counselor":
|
|
continue
|
|
ev = turn_eval(turn)
|
|
if ev is None:
|
|
continue
|
|
score = turn_score(ev)
|
|
if score is not None:
|
|
scores.append(score)
|
|
if score < 1.0:
|
|
watch_count += 1
|
|
rapport = turn_rapport(ev)
|
|
if rapport is not None:
|
|
rapports.append(rapport)
|
|
technique_count += len(turn_techniques(ev))
|
|
return SessionGrowthPoint(
|
|
session_id=sess.session_id,
|
|
session_no=sess.session_no,
|
|
persona_code=sess.persona_code,
|
|
stage=sess.state.stage.value,
|
|
started_at=iso_datetime(sess.created_at) or "",
|
|
ended_at=iso_datetime(sess.ended_at),
|
|
score=avg(scores),
|
|
rapport=avg(rapports),
|
|
technique_count=technique_count,
|
|
watch_count=watch_count,
|
|
)
|
|
|
|
|
|
def build_learner_growth(
|
|
sessions: list[InProcSession],
|
|
*,
|
|
learner_label: Callable[[str], str],
|
|
limit: int | None = None,
|
|
point_limit: int | None = 6,
|
|
) -> list[LearnerGrowthMetrics]:
|
|
grouped: dict[str, list[InProcSession]] = {}
|
|
for sess in sessions:
|
|
grouped.setdefault(sess.learner_id, []).append(sess)
|
|
|
|
result: list[LearnerGrowthMetrics] = []
|
|
for learner_id, learner_sessions in grouped.items():
|
|
ordered = sorted(learner_sessions, key=lambda sess: sess.created_at)
|
|
points = [session_growth_point(sess) for sess in ordered]
|
|
scored = [point for point in points if point.score is not None]
|
|
rapport_values = [
|
|
point.rapport for point in points if point.rapport is not None
|
|
]
|
|
technique_counts: dict[str, int] = {}
|
|
for sess in ordered:
|
|
for turn in sess.turns:
|
|
if turn.speaker != "counselor":
|
|
continue
|
|
ev = turn_eval(turn)
|
|
if ev is None:
|
|
continue
|
|
for label in turn_techniques(ev):
|
|
technique_counts[label] = technique_counts.get(label, 0) + 1
|
|
|
|
first_score = scored[0].score if scored else None
|
|
latest_score = scored[-1].score if scored else None
|
|
score_delta: float | None = None
|
|
trend = "insufficient"
|
|
if first_score is not None and latest_score is not None:
|
|
score_delta = round(latest_score - first_score, 3)
|
|
if len(scored) >= 2:
|
|
if score_delta >= 0.1:
|
|
trend = "up"
|
|
elif score_delta <= -0.1:
|
|
trend = "down"
|
|
else:
|
|
trend = "flat"
|
|
|
|
latest_session = ordered[-1]
|
|
top_techniques = [
|
|
label
|
|
for label, _count in sorted(
|
|
technique_counts.items(),
|
|
key=lambda item: (-item[1], item[0]),
|
|
)[:3]
|
|
]
|
|
result.append(
|
|
LearnerGrowthMetrics(
|
|
learner_id=learner_id,
|
|
learner_label=learner_label(learner_id),
|
|
sessions=len(ordered),
|
|
ended_sessions=sum(1 for sess in ordered if sess.ended),
|
|
latest_at=iso_datetime(session_activity_time(latest_session)) or "",
|
|
first_score=first_score,
|
|
latest_score=latest_score,
|
|
score_delta=score_delta,
|
|
avg_score=avg(
|
|
[point.score for point in scored if point.score is not None]
|
|
),
|
|
avg_rapport=avg(
|
|
[value for value in rapport_values if value is not None]
|
|
),
|
|
trend=trend,
|
|
top_techniques=top_techniques,
|
|
points=points if point_limit is None else points[-point_limit:],
|
|
)
|
|
)
|
|
sorted_result = sorted(result, key=lambda item: item.latest_at, reverse=True)
|
|
return sorted_result if limit is None else sorted_result[:limit]
|
|
|
|
|
|
def recent_feedback_notes(
|
|
sessions: list[InProcSession], *, limit: int = 5
|
|
) -> list[dict[str, object]]:
|
|
notes: list[dict[str, object]] = []
|
|
for sess in sorted(sessions, key=session_activity_time, reverse=True):
|
|
for turn in reversed(sess.turns):
|
|
if turn.speaker != "counselor":
|
|
continue
|
|
ev = turn_eval(turn)
|
|
if ev is None:
|
|
continue
|
|
note = turn_feedback_note(ev)
|
|
if note is None:
|
|
continue
|
|
notes.append(
|
|
{
|
|
"session_id": sess.session_id,
|
|
"persona_code": sess.persona_code,
|
|
"persona_name": sess.persona.display_name,
|
|
"session_no": sess.session_no,
|
|
"stage": turn.stage,
|
|
"turn_seq": turn.turn_seq,
|
|
"created_at": iso_datetime(turn.created_at)
|
|
or iso_datetime(session_activity_time(sess))
|
|
or "",
|
|
"score": turn_score(ev),
|
|
"rapport": turn_rapport(ev),
|
|
"note": note,
|
|
"techniques": turn_techniques(ev)[:3],
|
|
}
|
|
)
|
|
if len(notes) >= limit:
|
|
return notes
|
|
return notes
|