yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

세션 평가·라이브코치·교수자 분석 라운드 마감 + 문서 정리 + 코드품질 리팩터

Browse source 
- 누적 작업트리 커밋: 회기 평가 복구·durable 저장, 라이브 코치 이력/근거, 교수자 학생분석, 음성 비언어 메타, PII 마스킹, 운영 티켓/헬스 등
- 문서: 완료 기록 docs/archive/ 냉동 보관, docs/ 단일 인덱스(docs/README.md)+통합 TODO(docs/TODO.md)로 정리
- 리팩터(행위 보존): Stage enum SSOT(taxonomy 소유·state_machine re-export), store recent/masked_turns 중복 제거, speaker_ko_label 단일 헬퍼, _list_sessions N+1 제거(state/turns 배치 + 턴평가 하이드레이션 배치)
- 검증: 백엔드 pytest 352 passed, _list_sessions E2E chromium-single-run 2 passed
This commit is contained in:
Yun Chan 2026-07-02 02:50:36 +09:00
parent 7c41c3ce79
commit 778e8526d4
108 changed files with 6457 additions and 455 deletions
Download patch file Download diff file Expand all files Collapse all files

287
apps/api/app/test_evaluation_persistence.py
View file

@ -3,13 +3,14 @@
from __future__ import annotations
from pathlib import Path
import json
import unittest
from unittest.mock import patch
from .deps import Principal, Role
from . import session_persistence
from .routes import sessions
from .services import evaluator, session_metrics
from .services import evaluator, guardrail, session_metrics
from .services import persona as persona_service
from .services import state_machine
from .store import InProcSession
@ -33,8 +34,26 @@ class FakeEvaluationConn:
raise AssertionError(f"unexpected fetchval query: {query}")
class FakeMissingEvaluationConn:
def __init__(self) -> None:
self.fetches: list[tuple[str, tuple[object, ...]]] = []
self.fetchrows: list[tuple[str, tuple[object, ...]]] = []
async def fetch(self, query: str, *args: object) -> list[dict[str, object]]:
self.fetches.append((query, args))
if "FROM app.sessions s" in query:
return [{"id": "11111111-1111-1111-1111-111111111111"}]
if "FROM app.turns" in query:
return []
raise AssertionError(f"unexpected fetch query: {query}")
async def fetchrow(self, query: str, *args: object) -> dict[str, object]:
self.fetchrows.append((query, args))
return {}
class FakeAcquire:
def __init__(self, conn: FakeEvaluationConn) -> None:
def __init__(self, conn) -> None:
self.conn = conn
async def __aenter__(self) -> FakeEvaluationConn:
@ -44,7 +63,7 @@ class FakeAcquire:
return None
class EvaluationPersistenceMappingTest(unittest.TestCase):
class EvaluationPersistenceMappingTest(unittest.IsolatedAsyncioTestCase):
def test_session_metrics_prefers_rehydrated_technique_label_ko(self) -> None:
ev = {
"techniques": [
@ -137,6 +156,91 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
],
)
def test_turn_evaluation_rows_mask_pii_before_db_persistence(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
spans = []
for entity_type, value in (
("NAME", "보라별"),
("ORG", "미래학교상담연구랩"),
):
start = text.find(value)
if start >= 0:
spans.append(guardrail.PiiEntitySpan(entity_type, start, start + len(value)))
return spans
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
evaluation = {
"appropriateness": "warn",
"appropriateness_note": "보라별에게 010-1234-5678을 되물었다.",
"techniques": [
{
"code": "reflection",
"rationale": "미래학교상담연구랩 이야기를 바로 조언했다.",
}
],
"client_state_read": [
{
"code": "avoidance",
"rationale": "보라별이 기관 미래학교상담연구랩을 피했다.",
}
],
"intent_deviation": {
"expected": "보라별의 감정을 확인",
"actual": "010-1234-5678 연락처를 재질문",
},
"alternative_utterances": [
{
"text": "보라별님, 미래학교상담연구랩 이야기는 잠시 미뤄도 괜찮아요.",
"rationale": "010-1234-5678 같은 연락처 재확인을 피함",
}
],
}
blob = json.dumps(
{
"feedback": session_persistence._evaluation_feedback_rows(evaluation),
"comments": session_persistence._evaluation_comment_rows(evaluation),
"alternatives": session_persistence._evaluation_alternative_rows(evaluation),
},
ensure_ascii=False,
)
for raw in ("보라별", "미래학교상담연구랩", "010-1234-5678"):
self.assertNotIn(raw, blob)
for masked in ("[NAME]", "[ORG]", "[PHONE]"):
self.assertIn(masked, blob)
def test_turn_from_legacy_row_remasks_raw_text_fallback(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
start = text.find("보라별")
if start < 0:
return []
return [guardrail.PiiEntitySpan("NAME", start, start + len("보라별"))]
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
turn = session_persistence._turn_from_row(
{
"id": "11111111-1111-1111-1111-111111111111",
"seq": 2,
"speaker": "client",
"stage": "explore",
"text": "보라별의 전화는 010-1234-5678입니다.",
"text_masked": "",
"created_at": None,
"visible_to": ("client", "evaluator"),
}
)
self.assertEqual(turn.text, turn.text_masked)
self.assertNotIn("보라별", turn.text)
self.assertNotIn("010-1234-5678", turn.text)
self.assertIn("[NAME]", turn.text)
self.assertIn("[PHONE]", turn.text)
def test_session_evaluation_write_from_result_preserves_payload_shape(self) -> None:
result = evaluator.SessionEvaluation(
session_id="session-1",
@ -159,6 +263,55 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
self.assertNotIn("error", write.payload)
self.assertIsNone(write.error)
def test_session_evaluation_write_masks_deep_payload_before_storage(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
spans = []
for entity_type, value in (
("NAME", "보라별"),
("ORG", "미래학교상담연구랩"),
):
start = text.find(value)
if start >= 0:
spans.append(guardrail.PiiEntitySpan(entity_type, start, start + len(value)))
return spans
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
result = evaluator.SessionEvaluation(
session_id="session-privacy",
stage="정리",
scope="session_end",
turns_evaluated=3,
strengths=["보라별의 감정을 반영했다."],
improvements=["010-1234-5678 같은 연락처 재확인은 피한다."],
intent_deviations=[
evaluator.IntentDeviation(
dimension="pacing",
expected="미래학교상담연구랩 이야기를 기다린다",
actual="보라별에게 바로 조언했다",
severity="moderate",
)
],
supervisor_rationale="보라별의 호소를 요약했다.",
supervisor_critique="미래학교상담연구랩과 010-1234-5678을 반복했다.",
alternative_utterances=["보라별님, 지금 감정부터 천천히 볼까요?"],
error="보라별 평가 경고",
)
write = session_persistence.SessionEvaluationWrite.from_result(
session_id="session-privacy",
learner_id="learner-privacy",
result=result,
)
blob = json.dumps({"payload": write.payload, "error": write.error}, ensure_ascii=False)
self.assertEqual(write.status, "error")
for raw in ("보라별", "미래학교상담연구랩", "010-1234-5678"):
self.assertNotIn(raw, blob)
for masked in ("[NAME]", "[ORG]", "[PHONE]"):
self.assertIn(masked, blob)
def test_session_evaluation_write_from_error_preserves_fallback_shape(self) -> None:
write = session_persistence.SessionEvaluationWrite.from_error(
session_id="session-1",
@ -186,6 +339,65 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
self.assertEqual(write.error, "TimeoutError")
def test_session_evaluation_write_from_error_masks_error_detail(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
start = text.find("보라별")
if start < 0:
return []
return [guardrail.PiiEntitySpan("NAME", start, start + len("보라별"))]
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
write = session_persistence.SessionEvaluationWrite.from_error(
session_id="session-1",
learner_id="learner-1",
scope="session_end",
stage="explore",
error="보라별 처리 중 010-1234-5678 오류",
)
self.assertNotIn("보라별", write.error or "")
self.assertNotIn("010-1234-5678", write.error or "")
self.assertIn("[NAME]", write.error or "")
self.assertIn("[PHONE]", write.error or "")
async def test_save_session_evaluation_remasks_direct_write_payload(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
start = text.find("보라별")
if start < 0:
return []
return [guardrail.PiiEntitySpan("NAME", start, start + len("보라별"))]
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
conn = FakeEvaluationConn()
write = session_persistence.SessionEvaluationWrite(
session_id="11111111-1111-1111-1111-111111111111",
learner_id="22222222-2222-2222-2222-222222222222",
status="ready",
source="engine",
scope="session_end",
stage="정리",
payload={"strengths": ["보라별의 전화 010-1234-5678을 반복했다."]},
error="보라별 direct write error",
)
with (
patch.object(session_persistence, "get_pool", return_value=object()),
patch.object(session_persistence, "acquire", return_value=FakeAcquire(conn)),
):
saved = await session_persistence.save_session_evaluation(write)
self.assertTrue(saved)
_, args = conn.executed[0]
blob = json.dumps({"payload": args[5], "error": args[6]}, ensure_ascii=False)
self.assertNotIn("보라별", blob)
self.assertNotIn("010-1234-5678", blob)
self.assertIn("[NAME]", blob)
self.assertIn("[PHONE]", blob)
def test_rebuild_turn_evaluation_restores_review_shape(self) -> None:
rebuilt = session_persistence._rebuild_turn_evaluations(
[("11111111-1111-1111-1111-111111111111", 2, "탐색")],
@ -270,6 +482,11 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
self.assertIn("ALTER TABLE app.turn_client_state ENABLE ROW LEVEL SECURITY", sql)
self.assertIn("ALTER TABLE app.supervisor_comment ENABLE ROW LEVEL SECURITY", sql)
self.assertIn("ALTER TABLE app.alternative_utterance ENABLE ROW LEVEL SECURITY", sql)
self.assertIn("CREATE POLICY p_feedback_delete", sql)
self.assertIn("CREATE POLICY p_turn_technique_delete", sql)
self.assertIn("CREATE POLICY p_turn_client_state_delete", sql)
self.assertIn("CREATE POLICY p_supervisor_comment_delete", sql)
self.assertIn("CREATE POLICY p_alternative_utterance_delete", sql)
feedback_insert = sql.split("CREATE POLICY p_feedback_insert", 1)[1].split(");", 1)[0]
self.assertNotIn("learner_id = app.current_uid()", feedback_insert)
@ -281,6 +498,32 @@ class EvaluationPersistenceMappingTest(unittest.TestCase):
class EvaluationPersistenceIOTest(unittest.IsolatedAsyncioTestCase):
async def test_missing_session_evaluation_finder_uses_stale_ai_context_query(self) -> None:
conn = FakeMissingEvaluationConn()
sentinel = object()
with (
patch.object(session_persistence, "get_pool", return_value=object()),
patch.object(session_persistence, "acquire", return_value=FakeAcquire(conn)) as acquire,
patch.object(session_persistence, "_session_from_rows", return_value=sentinel),
):
found, durable = await session_persistence.list_sessions_missing_session_evaluation(
older_than_seconds=150.0,
limit=3,
)
self.assertTrue(durable)
self.assertEqual(found, [sentinel])
acquire.assert_called_once_with(ai_context=True, ai_view="evaluator")
self.assertEqual(conn.fetches[0][1], (150.0, 3))
session_query = conn.fetches[0][0]
self.assertIn("s.ended_at IS NOT NULL", session_query)
self.assertIn("se.session_id IS NULL", session_query)
self.assertIn("'client' = ANY(t.visible_to)", session_query)
self.assertIn("ORDER BY s.ended_at ASC", session_query)
self.assertEqual(conn.fetchrows[0][1], ("11111111-1111-1111-1111-111111111111",))
self.assertIn("FROM app.turns", conn.fetches[1][0])
async def test_record_llm_call_audit_inserts_metadata_only(self) -> None:
conn = FakeEvaluationConn()
payload = {
@ -366,6 +609,44 @@ class EvaluationPersistenceIOTest(unittest.IsolatedAsyncioTestCase):
self.assertIn("DELETE FROM app.alternative_utterance", executed_sql)
self.assertIn("INSERT INTO app.alternative_utterance", executed_sql)
async def test_replace_turn_evaluation_clears_stale_normalized_rows_before_reinsert(self) -> None:
conn = FakeEvaluationConn()
evaluation = {
"loop": "fast",
"turn_seq": 3,
"stage": "탐색",
"appropriateness": "pos",
"techniques": [
{"code": "reflection", "label_ko": "반영", "category": "relational"}
],
"client_state_read": [{"code": "open", "label_ko": "개방"}],
"alternative_utterances": ["조금 더 머물러도 괜찮습니다."],
}
with (
patch.object(session_persistence, "get_pool", return_value=object()),
patch.object(session_persistence, "acquire", return_value=FakeAcquire(conn)) as acquire,
):
ok = await session_persistence.replace_turn_evaluation(
turn_id="11111111-1111-1111-1111-111111111111",
evaluation=evaluation,
)
self.assertTrue(ok)
acquire.assert_called_once_with(ai_context=True, ai_view="evaluator")
executed_sql = "\n".join(query for query, _ in conn.executed)
self.assertLess(
executed_sql.index("DELETE FROM app.feedback_scores"),
executed_sql.index("INSERT INTO app.feedback_scores"),
)
self.assertIn("DELETE FROM app.turn_technique", executed_sql)
self.assertIn("DELETE FROM app.turn_client_state", executed_sql)
self.assertIn("DELETE FROM app.supervisor_comment", executed_sql)
self.assertIn("DELETE FROM app.alternative_utterance", executed_sql)
self.assertIn("INSERT INTO app.turn_technique", executed_sql)
self.assertIn("INSERT INTO app.turn_client_state", executed_sql)
self.assertIn("INSERT INTO app.alternative_utterance", executed_sql)
async def test_route_loader_only_hydrates_when_requested(self) -> None:
principal = Principal(
user_id="00000000-0000-0000-0000-000000000101",