현재 작업 상태 저장
This commit is contained in:
Yun Chan
parent
07cc67761e
commit
6bd91b0d5e
674 changed files with 8726 additions and 298 deletions
|
|
@ -18,6 +18,11 @@ from .store import DEFAULT_TURN_VISIBLE_TO, InProcSession, TurnRecord
|
|||
|
||||
_EVALUATION_CACHE: dict[str, dict[str, Any]] = {}
|
||||
_SESSION_AUDIT_ROLES = {"teacher", "admin"}
|
||||
_APPROPRIATENESS_SCORE = {
|
||||
"warn": 1.0,
|
||||
"neutral": 3.0,
|
||||
"pos": 5.0,
|
||||
}
|
||||
|
||||
|
||||
_JOINED_CARD_COLUMNS = (
|
||||
|
|
@ -71,6 +76,248 @@ def _stage(stage: object) -> str:
|
|||
return getattr(stage, "value", str(stage))
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _safe_float(value: Any) -> float | None:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def _evaluation_loop(evaluation: dict[str, Any]) -> str:
|
||||
loop = _clean_text(evaluation.get("loop")) or "fast"
|
||||
return loop if loop in {"fast", "deep"} else "fast"
|
||||
|
||||
|
||||
def _dict_items(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _evaluation_feedback_rows(evaluation: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""Normalize scalar/rationale turn-evaluation fields into feedback_scores rows."""
|
||||
if not isinstance(evaluation, dict):
|
||||
return []
|
||||
loop = _evaluation_loop(evaluation)
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
def add(
|
||||
dimension: str,
|
||||
*,
|
||||
score: float | None = None,
|
||||
rationale: str | None = None,
|
||||
top1_score: float | None = None,
|
||||
) -> None:
|
||||
dim = _clean_text(dimension)
|
||||
if not dim:
|
||||
return
|
||||
rows.append(
|
||||
{
|
||||
"dimension": dim,
|
||||
"score": score,
|
||||
"rationale": rationale,
|
||||
"top1_score": top1_score,
|
||||
"loop": loop,
|
||||
}
|
||||
)
|
||||
|
||||
appropriateness = _clean_text(evaluation.get("appropriateness")) or "neutral"
|
||||
if appropriateness not in _APPROPRIATENESS_SCORE:
|
||||
appropriateness = "neutral"
|
||||
add(
|
||||
"appropriateness",
|
||||
score=_APPROPRIATENESS_SCORE[appropriateness],
|
||||
rationale=_clean_text(evaluation.get("appropriateness_note")),
|
||||
)
|
||||
|
||||
rapport = _safe_float(evaluation.get("rapport_signal"))
|
||||
if rapport is not None:
|
||||
add("rapport_signal", score=max(-1.0, min(1.0, rapport)))
|
||||
|
||||
theory_mode = _clean_text(evaluation.get("theory_mode"))
|
||||
if theory_mode:
|
||||
add("theory_mode", rationale=theory_mode)
|
||||
|
||||
error = _clean_text(evaluation.get("error"))
|
||||
if error:
|
||||
add("error", rationale=error)
|
||||
|
||||
for tag in _dict_items(evaluation.get("techniques")):
|
||||
code = _clean_text(tag.get("code"))
|
||||
rationale = _clean_text(tag.get("rationale"))
|
||||
if code and rationale:
|
||||
add(f"technique:{code}", rationale=rationale)
|
||||
|
||||
for state in _dict_items(evaluation.get("client_state_read")):
|
||||
code = _clean_text(state.get("code"))
|
||||
rationale = _clean_text(state.get("rationale"))
|
||||
if code and rationale:
|
||||
add(f"client_state:{code}", rationale=rationale)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _evaluation_technique_rows(evaluation: dict[str, Any] | None) -> list[dict[str, str]]:
|
||||
if not isinstance(evaluation, dict):
|
||||
return []
|
||||
rows: list[dict[str, str]] = []
|
||||
for tag in _dict_items(evaluation.get("techniques")):
|
||||
code = _clean_text(tag.get("code"))
|
||||
if not code:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"code": code,
|
||||
"label_ko": _clean_text(tag.get("label_ko")) or code,
|
||||
"category": _clean_text(tag.get("category")) or "",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _evaluation_client_state_rows(evaluation: dict[str, Any] | None) -> list[dict[str, str]]:
|
||||
if not isinstance(evaluation, dict):
|
||||
return []
|
||||
rows: list[dict[str, str]] = []
|
||||
for state in _dict_items(evaluation.get("client_state_read")):
|
||||
code = _clean_text(state.get("code"))
|
||||
if not code:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"code": code,
|
||||
"label_ko": _clean_text(state.get("label_ko")) or code,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _evaluation_comment_rows(evaluation: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
if not isinstance(evaluation, dict):
|
||||
return []
|
||||
deviation = evaluation.get("intent_deviation")
|
||||
if not isinstance(deviation, dict):
|
||||
return []
|
||||
note = _clean_text(evaluation.get("appropriateness_note")) or "의도와 다른 부분"
|
||||
return [{"kind": "critique", "text": note, "intent_deviation": deviation}]
|
||||
|
||||
|
||||
def _appropriateness_from_score(score: Any) -> str:
|
||||
value = _safe_float(score)
|
||||
if value is None:
|
||||
return "neutral"
|
||||
if value >= 4.0:
|
||||
return "pos"
|
||||
if value <= 2.0:
|
||||
return "warn"
|
||||
return "neutral"
|
||||
|
||||
|
||||
def _base_turn_evaluation(turn_seq: int, stage: str) -> dict[str, Any]:
|
||||
return {
|
||||
"loop": "fast",
|
||||
"turn_seq": turn_seq,
|
||||
"stage": stage,
|
||||
"techniques": [],
|
||||
"client_state_read": [],
|
||||
"appropriateness": "neutral",
|
||||
}
|
||||
|
||||
|
||||
def _rebuild_turn_evaluations(
|
||||
turn_refs: list[tuple[str, int, str]],
|
||||
*,
|
||||
feedback_rows: Iterable[Any],
|
||||
technique_rows: Iterable[Any],
|
||||
client_state_rows: Iterable[Any],
|
||||
comment_rows: Iterable[Any],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Rehydrate normalized DB rows back into the TurnRecord.evaluation shape."""
|
||||
refs = {turn_id: (turn_seq, stage) for turn_id, turn_seq, stage in turn_refs}
|
||||
evaluations: dict[str, dict[str, Any]] = {}
|
||||
rationale_by_dimension: dict[str, dict[str, str]] = {}
|
||||
|
||||
def ensure(turn_id: str) -> dict[str, Any]:
|
||||
if turn_id not in evaluations:
|
||||
turn_seq, stage = refs[turn_id]
|
||||
evaluations[turn_id] = _base_turn_evaluation(turn_seq, stage)
|
||||
return evaluations[turn_id]
|
||||
|
||||
for row in feedback_rows:
|
||||
turn_id = str(row["turn_id"])
|
||||
if turn_id not in refs:
|
||||
continue
|
||||
ev = ensure(turn_id)
|
||||
loop = _clean_text(row["loop"])
|
||||
if loop in {"fast", "deep"}:
|
||||
ev["loop"] = loop
|
||||
dimension = _clean_text(row["dimension"]) or ""
|
||||
rationale = _clean_text(row["rationale"])
|
||||
if rationale:
|
||||
rationale_by_dimension.setdefault(turn_id, {})[dimension] = rationale
|
||||
if dimension == "appropriateness":
|
||||
ev["appropriateness"] = _appropriateness_from_score(row["score"])
|
||||
if rationale:
|
||||
ev["appropriateness_note"] = rationale
|
||||
elif dimension == "rapport_signal":
|
||||
score = _safe_float(row["score"])
|
||||
if score is not None:
|
||||
ev["rapport_signal"] = max(-1.0, min(1.0, score))
|
||||
elif dimension == "theory_mode" and rationale:
|
||||
ev["theory_mode"] = rationale
|
||||
elif dimension == "error" and rationale:
|
||||
ev["error"] = rationale
|
||||
|
||||
for row in technique_rows:
|
||||
turn_id = str(row["turn_id"])
|
||||
if turn_id not in refs:
|
||||
continue
|
||||
code = _clean_text(row["code"])
|
||||
if not code:
|
||||
continue
|
||||
item = {
|
||||
"code": code,
|
||||
"label_ko": _clean_text(row["label_ko"]) or code,
|
||||
"category": _clean_text(row["category"]) or "",
|
||||
}
|
||||
rationale = rationale_by_dimension.get(turn_id, {}).get(f"technique:{code}")
|
||||
if rationale:
|
||||
item["rationale"] = rationale
|
||||
ensure(turn_id)["techniques"].append(item)
|
||||
|
||||
for row in client_state_rows:
|
||||
turn_id = str(row["turn_id"])
|
||||
if turn_id not in refs:
|
||||
continue
|
||||
code = _clean_text(row["code"])
|
||||
if not code:
|
||||
continue
|
||||
item = {
|
||||
"code": code,
|
||||
"label_ko": _clean_text(row["label_ko"]) or code,
|
||||
}
|
||||
rationale = rationale_by_dimension.get(turn_id, {}).get(f"client_state:{code}")
|
||||
if rationale:
|
||||
item["rationale"] = rationale
|
||||
ensure(turn_id)["client_state_read"].append(item)
|
||||
|
||||
for row in comment_rows:
|
||||
turn_id = str(row["turn_id"])
|
||||
if turn_id not in refs:
|
||||
continue
|
||||
deviation = row["intent_deviation"]
|
||||
if isinstance(deviation, dict):
|
||||
ensure(turn_id)["intent_deviation"] = deviation
|
||||
|
||||
return evaluations
|
||||
|
||||
|
||||
async def _record_session_read_audit(
|
||||
conn: Any,
|
||||
principal: Principal,
|
||||
|
|
@ -113,7 +360,7 @@ def _state_from_row(row, card: PersonaCard) -> state_machine.SessionState:
|
|||
)
|
||||
|
||||
|
||||
def _turn_from_row(row) -> TurnRecord:
|
||||
def _turn_from_row(row, evaluation: dict[str, Any] | None = None) -> TurnRecord:
|
||||
created_at = _ts(row["created_at"]) or time.time()
|
||||
return TurnRecord(
|
||||
turn_seq=int(row["seq"]),
|
||||
|
|
@ -121,6 +368,7 @@ def _turn_from_row(row) -> TurnRecord:
|
|||
stage=row["stage"],
|
||||
text=row["text"] or row["text_masked"] or "",
|
||||
text_masked=row["text_masked"] or row["text"] or "",
|
||||
turn_id=str(_row_value(row, "id")) if _row_value(row, "id") is not None else None,
|
||||
created_at=created_at,
|
||||
llm_provider=_row_value(row, "llm_provider"),
|
||||
model=_row_value(row, "model"),
|
||||
|
|
@ -131,10 +379,179 @@ def _turn_from_row(row) -> TurnRecord:
|
|||
silence_ms=_row_value(row, "silence_ms"),
|
||||
speech_rate=_row_value(row, "speech_rate"),
|
||||
barge_in=_row_value(row, "barge_in"),
|
||||
evaluation=evaluation,
|
||||
visible_to=tuple(_row_value(row, "visible_to") or DEFAULT_TURN_VISIBLE_TO),
|
||||
)
|
||||
|
||||
|
||||
async def _persist_turn_evaluation(conn: Any, turn_id: str, evaluation: dict[str, Any] | None) -> None:
|
||||
if not isinstance(evaluation, dict):
|
||||
return
|
||||
await conn.execute("SELECT set_config('app.ai_context', '1', true)")
|
||||
await conn.execute("SELECT set_config('app.current_ai_view', 'evaluator', true)")
|
||||
await conn.execute("SELECT set_config('app.current_sens_max', '2', true)")
|
||||
|
||||
for row in _evaluation_feedback_rows(evaluation):
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.feedback_scores (
|
||||
turn_id, dimension, score, rationale, top1_score, loop
|
||||
)
|
||||
VALUES ($1::uuid, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (turn_id, dimension) DO UPDATE SET
|
||||
score = EXCLUDED.score,
|
||||
rationale = EXCLUDED.rationale,
|
||||
top1_score = EXCLUDED.top1_score,
|
||||
loop = EXCLUDED.loop
|
||||
""",
|
||||
turn_id,
|
||||
row["dimension"],
|
||||
row["score"],
|
||||
row["rationale"],
|
||||
row["top1_score"],
|
||||
row["loop"],
|
||||
)
|
||||
|
||||
for row in _evaluation_technique_rows(evaluation):
|
||||
label_id = await conn.fetchval(
|
||||
"""
|
||||
INSERT INTO app.technique_label_def (code, display_name, category)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (code, version) DO UPDATE SET
|
||||
display_name = EXCLUDED.display_name,
|
||||
category = EXCLUDED.category,
|
||||
is_active = TRUE
|
||||
RETURNING label_id
|
||||
""",
|
||||
row["code"],
|
||||
row["label_ko"],
|
||||
row["category"],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.turn_technique (turn_id, label_id)
|
||||
VALUES ($1::uuid, $2)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
turn_id,
|
||||
label_id,
|
||||
)
|
||||
|
||||
for row in _evaluation_client_state_rows(evaluation):
|
||||
label_id = await conn.fetchval(
|
||||
"""
|
||||
INSERT INTO app.client_state_def (code, display_name)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (code, version) DO UPDATE SET
|
||||
display_name = EXCLUDED.display_name,
|
||||
is_active = TRUE
|
||||
RETURNING label_id
|
||||
""",
|
||||
row["code"],
|
||||
row["label_ko"],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.turn_client_state (turn_id, label_id)
|
||||
VALUES ($1::uuid, $2)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
turn_id,
|
||||
label_id,
|
||||
)
|
||||
|
||||
for row in _evaluation_comment_rows(evaluation):
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervisor_comment (
|
||||
turn_id, kind, text, intent_deviation
|
||||
)
|
||||
VALUES ($1::uuid, $2, $3, $4::jsonb)
|
||||
""",
|
||||
turn_id,
|
||||
row["kind"],
|
||||
row["text"],
|
||||
row["intent_deviation"],
|
||||
)
|
||||
|
||||
|
||||
async def _load_turn_evaluations(
|
||||
conn: Any,
|
||||
turn_refs: list[tuple[str, int, str]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
turn_ids = [turn_id for turn_id, _, _ in turn_refs]
|
||||
if not turn_ids:
|
||||
return {}
|
||||
feedback_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT turn_id::text AS turn_id, dimension, score, rationale, top1_score, loop
|
||||
FROM app.feedback_scores
|
||||
WHERE turn_id = ANY($1::uuid[])
|
||||
ORDER BY created_at, dimension
|
||||
""",
|
||||
turn_ids,
|
||||
)
|
||||
technique_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT
|
||||
tt.turn_id::text AS turn_id,
|
||||
d.code,
|
||||
d.display_name AS label_ko,
|
||||
d.category
|
||||
FROM app.turn_technique tt
|
||||
JOIN app.technique_label_def d ON d.label_id = tt.label_id
|
||||
WHERE tt.turn_id = ANY($1::uuid[])
|
||||
ORDER BY tt.turn_id, d.code
|
||||
""",
|
||||
turn_ids,
|
||||
)
|
||||
client_state_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT
|
||||
ts.turn_id::text AS turn_id,
|
||||
d.code,
|
||||
d.display_name AS label_ko
|
||||
FROM app.turn_client_state ts
|
||||
JOIN app.client_state_def d ON d.label_id = ts.label_id
|
||||
WHERE ts.turn_id = ANY($1::uuid[])
|
||||
ORDER BY ts.turn_id, d.code
|
||||
""",
|
||||
turn_ids,
|
||||
)
|
||||
comment_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT turn_id::text AS turn_id, intent_deviation
|
||||
FROM app.supervisor_comment
|
||||
WHERE turn_id = ANY($1::uuid[])
|
||||
AND intent_deviation IS NOT NULL
|
||||
ORDER BY created_at
|
||||
""",
|
||||
turn_ids,
|
||||
)
|
||||
return _rebuild_turn_evaluations(
|
||||
turn_refs,
|
||||
feedback_rows=feedback_rows,
|
||||
technique_rows=technique_rows,
|
||||
client_state_rows=client_state_rows,
|
||||
comment_rows=comment_rows,
|
||||
)
|
||||
|
||||
|
||||
async def _hydrate_session_turn_evaluations(sess: InProcSession) -> None:
|
||||
turn_refs = [
|
||||
(turn.turn_id, turn.turn_seq, turn.stage)
|
||||
for turn in sess.turns
|
||||
if turn.turn_id is not None
|
||||
]
|
||||
if not turn_refs:
|
||||
return
|
||||
async with acquire(ai_view="evaluator") as conn:
|
||||
evaluations = await _load_turn_evaluations(conn, turn_refs)
|
||||
for turn in sess.turns:
|
||||
if turn.turn_id and turn.turn_id in evaluations:
|
||||
turn.evaluation = evaluations[turn.turn_id]
|
||||
|
||||
|
||||
async def ensure_review_tables() -> None:
|
||||
"""Create runtime review/evaluation storage when the DB role allows it."""
|
||||
try:
|
||||
|
|
@ -441,6 +858,7 @@ async def load_session(
|
|||
principal: Principal,
|
||||
*,
|
||||
allow_ended: bool = False,
|
||||
include_turn_evaluation: bool = False,
|
||||
) -> InProcSession | None:
|
||||
try:
|
||||
get_pool()
|
||||
|
|
@ -496,7 +914,7 @@ async def load_session(
|
|||
)
|
||||
turn_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT seq, speaker, stage, text, text_masked, created_at,
|
||||
SELECT id, seq, speaker, stage, text, text_masked, created_at,
|
||||
llm_provider, model, tokens_in, tokens_out, cost_usd,
|
||||
audio_ref, silence_ms, speech_rate, barge_in, visible_to
|
||||
FROM app.turns
|
||||
|
|
@ -518,7 +936,9 @@ async def load_session(
|
|||
"learner_id": sess.learner_id,
|
||||
},
|
||||
)
|
||||
return sess
|
||||
if sess is not None and include_turn_evaluation:
|
||||
await _hydrate_session_turn_evaluations(sess)
|
||||
return sess
|
||||
except Exception:
|
||||
require_runtime_fallback_allowed("session load")
|
||||
return None
|
||||
|
|
@ -546,7 +966,7 @@ async def append_turn(
|
|||
)
|
||||
or 1
|
||||
)
|
||||
await conn.execute(
|
||||
inserted_turn_id = await conn.fetchval(
|
||||
"""
|
||||
INSERT INTO app.turns (
|
||||
session_id, seq, speaker, stage, text, text_masked, actor_kind,
|
||||
|
|
@ -559,6 +979,7 @@ async def append_turn(
|
|||
$13, $14, $15, $16, $17::text[]
|
||||
)
|
||||
ON CONFLICT (session_id, seq) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
session_id,
|
||||
seq,
|
||||
|
|
@ -578,6 +999,11 @@ async def append_turn(
|
|||
turn.barge_in,
|
||||
list(turn.visible_to or DEFAULT_TURN_VISIBLE_TO),
|
||||
)
|
||||
if inserted_turn_id is None:
|
||||
return False
|
||||
turn.turn_id = str(inserted_turn_id)
|
||||
# 원시 평가 row는 학습자 축어록이 아니라 evaluator 전용 데이터다.
|
||||
await _persist_turn_evaluation(conn, turn.turn_id, turn.evaluation)
|
||||
return True
|
||||
except Exception:
|
||||
require_runtime_fallback_allowed("session turn append")
|
||||
|
|
@ -703,7 +1129,7 @@ async def list_sessions(principal: Principal) -> tuple[list[InProcSession], bool
|
|||
)
|
||||
turn_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT seq, speaker, stage, text, text_masked, created_at,
|
||||
SELECT id, seq, speaker, stage, text, text_masked, created_at,
|
||||
llm_provider, model, tokens_in, tokens_out, cost_usd,
|
||||
audio_ref, silence_ms, speech_rate, barge_in, visible_to
|
||||
FROM app.turns
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue