From 6b6241f46863d116664d16e27552427f1f09b188 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Fri, 3 Jul 2026 19:53:14 +0900 Subject: [PATCH] =?UTF-8?q?=ED=95=9C=EC=8B=A0=EB=8C=80=20=ED=94=BC?= =?UTF-8?q?=EB=93=9C=EB=B0=B1=20=EA=B0=9C=EC=84=A0=ED=8C=A9=20=EB=B0=98?= =?UTF-8?q?=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/app/routes/sessions.py | 6 +- apps/api/app/services/guardrail.py | 58 +++- apps/api/app/services/orchestrator.py | 133 +++++--- apps/api/app/services/persona.py | 2 +- apps/api/app/session_read_model.py | 16 +- apps/api/app/test_client_reply_quality.py | 73 ++++ apps/api/app/test_orchestrator_masking.py | 126 +++++++ apps/api/app/test_session_turn_persistence.py | 15 +- apps/api/engine_gateway/gateway.py | 43 ++- apps/api/engine_gateway/test_gateway_model.py | 33 ++ apps/web/e2e/session-review.spec.ts | 91 ++++- apps/web/e2e/support.ts | 11 +- apps/web/src/lib/api.gen.ts | 2 + apps/web/src/lib/api.ts | 2 + apps/web/src/pages/Session.tsx | 9 +- apps/web/src/pages/SessionReview.tsx | 63 +++- .../pages/session-review/session-review.css | 40 +++ apps/web/src/pages/session/session.css | 40 ++- docs/README.md | 3 + docs/TODO.md | 6 + docs/dev_dashboard.html | 2 + ...in-feedback-improvement-plan-2026-07-03.md | 318 ++++++++++++++++++ ...eedback-plan-counter-redteam-2026-07-03.md | 116 +++++++ ...anshin-feedback-plan-redteam-2026-07-03.md | 130 +++++++ scripts/watch-public-runtime-hidden.vbs | 3 + 25 files changed, 1247 insertions(+), 94 deletions(-) create mode 100644 apps/api/app/test_client_reply_quality.py create mode 100644 docs/ops/hanshin-feedback-improvement-plan-2026-07-03.md create mode 100644 docs/redteam/hanshin-feedback-plan-counter-redteam-2026-07-03.md create mode 100644 docs/redteam/hanshin-feedback-plan-redteam-2026-07-03.md create mode 100644 scripts/watch-public-runtime-hidden.vbs diff --git a/apps/api/app/routes/sessions.py b/apps/api/app/routes/sessions.py index 2273229..d447d5f 100644 --- a/apps/api/app/routes/sessions.py +++ b/apps/api/app/routes/sessions.py @@ -131,6 +131,7 @@ class TurnResponse(BaseModel): crisis_kind: str = "none" crisis_resource: Optional[CrisisResourceResponse] = None conversation_stopped: bool = False + output_error: Optional[str] = None class SessionEndResponse(BaseModel): @@ -678,11 +679,12 @@ def _stream_result_from_done( evaluation: Optional[dict], ) -> orchestrator.TurnResult: assert ctx.state_after is not None + output_error = str(data.get("output_error") or "") or None return orchestrator.TurnResult( turn_seq=ctx.state_after.turn_seq, stage=_stage_label(ctx.state_after.stage), effective_openness=ctx.state_after.effective_openness, - client_reply=final_reply or None, + client_reply=None if output_error else final_reply or None, safety_flagged=bool(data.get("safety_flagged")), state_after=ctx.state_after, evaluation=evaluation, @@ -694,6 +696,7 @@ def _stream_result_from_done( tokens_in=int(data.get("tokens_in") or 0), tokens_out=int(data.get("tokens_out") or 0), cost_usd=float(data.get("cost_usd") or 0.0), + output_error=output_error, ) @@ -1330,6 +1333,7 @@ async def submit_turn( crisis_kind=result.crisis_kind, crisis_resource=result.crisis_resource, conversation_stopped=result.conversation_stopped, + output_error=result.output_error, ) diff --git a/apps/api/app/services/guardrail.py b/apps/api/app/services/guardrail.py index e29bbc2..ed3a214 100644 --- a/apps/api/app/services/guardrail.py +++ b/apps/api/app/services/guardrail.py @@ -19,6 +19,7 @@ from __future__ import annotations import re from dataclasses import dataclass, field +from difflib import SequenceMatcher from enum import Enum from typing import Iterable, Protocol @@ -448,7 +449,50 @@ class OutputGuardResult: reasons: list[str] = field(default_factory=list) -def sanitize_client_reply(text: str, *, ideation_stage: int) -> OutputGuardResult: +_ROLE_META_PATTERNS = [ + re.compile(r"(?:내담자|상담자)\s*역할\s*로?\s*응답"), + re.compile(r"AI\s*로서"), + re.compile(r"상담자\s*입장\s*에서"), + re.compile(r"제\s*(?:핵심신념|자동적\s*사고|인지왜곡)\s*은"), +] +_OPENING_GREETING_RE = re.compile(r"^\s*(?:안녕하세요|처음\s*뵙겠습니다|반갑습니다)(?:[\s,.!?。！？]|$)") +_DUPLICATE_PUNCT_RE = re.compile(r"[\s\.,!?。！？…\"'“”‘’()\[\]{}:;·~\-]+") + + +def _normalize_duplicate_text(value: str) -> str: + return _DUPLICATE_PUNCT_RE.sub("", value.casefold()) + + +def _duplicate_tokens(value: str) -> list[str]: + cleaned = _DUPLICATE_PUNCT_RE.sub(" ", value.casefold()) + return [token for token in cleaned.split() if len(token) >= 2] + + +def _is_near_duplicate_client_reply(text: str, previous: str) -> bool: + current_norm = _normalize_duplicate_text(text) + previous_norm = _normalize_duplicate_text(previous) + if len(current_norm) < 12 or len(previous_norm) < 12: + return False + if current_norm == previous_norm or current_norm in previous_norm or previous_norm in current_norm: + return True + if SequenceMatcher(None, current_norm, previous_norm).ratio() >= 0.88: + return True + + current_tokens = set(_duplicate_tokens(text)) + previous_tokens = set(_duplicate_tokens(previous)) + if min(len(current_tokens), len(previous_tokens)) < 4: + return False + overlap = len(current_tokens & previous_tokens) / min(len(current_tokens), len(previous_tokens)) + return overlap >= 0.82 + + +def sanitize_client_reply( + text: str, + *, + ideation_stage: int, + turn_seq: int | None = None, + previous_client_reply: str | None = None, +) -> OutputGuardResult: """내담자 AI 응답 출력 가드레일. 1. 자살/자해 수단·방법 정보가 있으면 → needs_regeneration=True(차단, 재생성 신호). @@ -466,6 +510,18 @@ def sanitize_client_reply(text: str, *, ideation_stage: int) -> OutputGuardResul blocked = True needs_regen = True break + if any(pattern.search(text) for pattern in _ROLE_META_PATTERNS): + reasons.append("role_meta") + blocked = True + needs_regen = True + if turn_seq is not None and turn_seq > 1 and _OPENING_GREETING_RE.search(text): + reasons.append("repeat_greeting_after_opening") + blocked = True + needs_regen = True + if previous_client_reply and _is_near_duplicate_client_reply(text, previous_client_reply): + reasons.append("duplicate_client_reply") + blocked = True + needs_regen = True if ideation_stage > IDEATION_STAGE_CAP: reasons.append(f"ideation_over_cap:{ideation_stage}>{IDEATION_STAGE_CAP}") diff --git a/apps/api/app/services/orchestrator.py b/apps/api/app/services/orchestrator.py index c915374..03c2bf7 100644 --- a/apps/api/app/services/orchestrator.py +++ b/apps/api/app/services/orchestrator.py @@ -118,6 +118,7 @@ class TurnResult: tokens_in: int = 0 tokens_out: int = 0 cost_usd: float = 0.0 + output_error: Optional[str] = None # ════════════════════════════════════════════════════════════════════════════ @@ -211,6 +212,15 @@ def _mask_recent_turns(turns: Optional[list[dict[str, str]]]) -> list[dict[str, return masked +def _latest_client_reply(turns: list[dict[str, str]]) -> Optional[str]: + for turn in reversed(turns): + if turn.get("speaker") == "client": + text = str(turn.get("text", "")).strip() + if text: + return text + return None + + # ════════════════════════════════════════════════════════════════════════════ # 4~8단계 — 동기 생성 경로 (폴백/테스트) # ════════════════════════════════════════════════════════════════════════════ @@ -238,29 +248,57 @@ async def run_turn_generate( session_id=ctx.session_id, metadata={"stage": st.stage.value}, ) - started = time.perf_counter() - resp: GenerateResponse = await engine.generate(req) - latency_ms = int((time.perf_counter() - started) * 1000) - await _record_llm_audit( - audit_hook, - session_id=ctx.session_id, - provider=resp.provider, - model=resp.model, - tokens_in=resp.tokens_in, - tokens_out=resp.tokens_out, - cost_usd=resp.cost_usd, - inference_geo=resp.inference_geo, - latency_ms=latency_ms, - ) - reply = resp.text + previous_client_reply = _latest_client_reply(ctx.memory.recent_turns) + resp: GenerateResponse | None = None + reply = "" + safety_flagged = ctx.crisis is not None and ctx.crisis.escalate + for attempt in range(2): + started = time.perf_counter() + resp = await engine.generate(req) + latency_ms = int((time.perf_counter() - started) * 1000) + await _record_llm_audit( + audit_hook, + session_id=ctx.session_id, + provider=resp.provider, + model=resp.model, + tokens_in=resp.tokens_in, + tokens_out=resp.tokens_out, + cost_usd=resp.cost_usd, + inference_geo=resp.inference_geo, + latency_ms=latency_ms, + ) - # 5) 출력 가드레일 — 수단 차단 + ideation 상한 - guard = guardrail.sanitize_client_reply(reply, ideation_stage=st.ideation_stage) - reply = guard.text - safety_flagged = guard.blocked or (ctx.crisis is not None and ctx.crisis.escalate) - if guard.needs_regeneration: - # 수단정보 누출 → 안전 대체 응답으로 치환(1차). 재생성 루프는 후속. - reply = "…(말을 잇지 못하고 잠시 침묵한다)" + # 5) 출력 가드레일 — 수단 차단 + persona 품질 재생성 + guard = guardrail.sanitize_client_reply( + resp.text, + ideation_stage=st.ideation_stage, + turn_seq=st.turn_seq, + previous_client_reply=previous_client_reply, + ) + if guard.needs_regeneration: + if attempt == 0: + continue + return TurnResult( + turn_seq=st.turn_seq, + stage=st.stage.value, + effective_openness=st.effective_openness, + client_reply=None, + safety_flagged=True, + state_after=st, + evaluation=None, + crisis_kind=ctx.crisis.kind.value if ctx.crisis else "none", + llm_provider=resp.provider, + model=resp.model, + tokens_in=resp.tokens_in, + tokens_out=resp.tokens_out, + cost_usd=resp.cost_usd, + output_error="client_reply_quality_retryable", + ) + safety_flagged = safety_flagged or guard.blocked + reply = guard.text + break + + assert resp is not None # 6) 평가 훅(주입형) — 평가 AI 4차원 태깅 (Features 소유) evaluation: Optional[dict] = None @@ -340,8 +378,9 @@ async def run_turn_stream( accumulated = "" flagged = False + output_error: str | None = None stream_meta: dict[str, Any] = {} - display_sanitizer = guardrail.PiiPlaceholderStreamSanitizer() + previous_client_reply = _latest_client_reply(ctx.memory.recent_turns) if ctx.crisis is not None and ctx.crisis.escalate: flagged = True resource = guardrail.crisis_resource() @@ -375,9 +414,6 @@ async def run_turn_stream( if packet.event == ENGINE_GATEWAY_SSE_ERROR: payload = packet.payload detail = payload.detail if isinstance(payload, StreamErrorEvent) else "engine stream error" - display_tail = display_sanitizer.flush() - if display_tail: - yield StreamEvent("token", {"text": display_tail}) yield StreamEvent("error", {"detail": detail}) return if packet.event == ENGINE_GATEWAY_SSE_DONE: @@ -392,23 +428,27 @@ async def run_turn_stream( text_piece = payload.text accumulated += text_piece - # 출력 가드레일(누적 스캔) — 수단정보 발견 시 차단·재생성 신호 - guard = guardrail.sanitize_client_reply(accumulated, ideation_stage=st.ideation_stage) - if guard.needs_regeneration and not flagged: - flagged = True - yield StreamEvent("safety", {"reason": "means_info_blocked"}) - # 토큰은 더 내보내지 않고 안전 대체로 종결 - accumulated = "…(말을 잇지 못하고 잠시 침묵한다)" - break - - display_piece = display_sanitizer.feed(text_piece) - if display_piece: - yield StreamEvent("token", {"text": display_piece}) - - if not flagged: - display_tail = display_sanitizer.flush() - if display_tail: - yield StreamEvent("token", {"text": display_tail}) + guard = guardrail.sanitize_client_reply( + accumulated, + ideation_stage=st.ideation_stage, + turn_seq=st.turn_seq, + previous_client_reply=previous_client_reply, + ) + if guard.needs_regeneration: + flagged = True + output_error = "client_reply_quality_retryable" + yield StreamEvent( + "safety", + { + "reason": output_error, + "reasons": guard.reasons, + }, + ) + accumulated = "" + else: + flagged = flagged or guard.blocked + if guard.text: + yield StreamEvent("token", {"text": guard.text}) latency_ms = int((time.perf_counter() - started) * 1000) await _record_llm_audit( @@ -444,17 +484,12 @@ async def run_turn_stream( "tokens_in": _safe_int(stream_meta.get("tokens_in")), "tokens_out": _safe_int(stream_meta.get("tokens_out")), "cost_usd": _safe_float(stream_meta.get("cost_usd")), + "output_error": output_error, }, ) except EngineGatewaySseDecodeError as e: - display_tail = display_sanitizer.flush() - if display_tail: - yield StreamEvent("token", {"text": display_tail}) yield StreamEvent("error", {"detail": str(e)}) except EngineError as e: - display_tail = display_sanitizer.flush() - if display_tail: - yield StreamEvent("token", {"text": display_tail}) yield StreamEvent("error", {"detail": str(e)}) diff --git a/apps/api/app/services/persona.py b/apps/api/app/services/persona.py index e4d5364..d6fdc74 100644 --- a/apps/api/app/services/persona.py +++ b/apps/api/app/services/persona.py @@ -300,7 +300,7 @@ def build_turn_messages( # non-system history records 는 요청 계약상 보존하고, 별도 prompt 동작 변경에서 소비한다. if memory.recent_turns: for t in memory.recent_turns: - role = "assistant" if t.get("speaker") == "counselor" else "user" + role = "user" if t.get("speaker") == "counselor" else "assistant" # 내담자(자기) 과거 발화는 assistant, 상담자 발화는 user 로 매핑한다. messages.append(EngineMessage(role=role, content=t.get("text", ""), cache=False)) diff --git a/apps/api/app/session_read_model.py b/apps/api/app/session_read_model.py index 0c45d7d..b1f923c 100644 --- a/apps/api/app/session_read_model.py +++ b/apps/api/app/session_read_model.py @@ -807,10 +807,9 @@ def _review_summary_from_evaluation( return fallback status = str(evaluation_record.get("status") or "") if status != "ready": - error = _compact_text(str(evaluation_record.get("error") or payload.get("error") or "")) return ( - "저장된 축어록은 확인했지만 평가 AI 산출물이 아직 준비되지 않았습니다. " - + (f"사유: {error}" if error else "평가가 완료되면 코칭 항목이 갱신됩니다.") + "저장된 축어록은 확인했지만 deep-loop 평가 AI 산출물을 표시하지 못했습니다. " + "AI 평가 재시도가 필요합니다." ) rationale = _compact_text(str(payload.get("supervisor_rationale") or "")) critique = _compact_text(str(payload.get("supervisor_critique") or "")) @@ -1159,9 +1158,10 @@ def _review_note_from_turn_eval( return ReviewNote( author="평가 AI", tone="warn", - title="턴 평가 실패", + title="턴 직후 평가 실패", body=_review_note_body_markdown( - f"이 발화의 fast-loop 평가를 완료하지 못했습니다.\n\n사유: {error_text}" + "이 발화의 fast-loop(턴 직후) 평가를 완료하지 못했습니다.\n\n" + "AI 평가 재시도가 필요합니다." ), quote=quote, ) @@ -1178,7 +1178,7 @@ def _review_note_from_turn_eval( return ReviewNote( author="평가 AI", tone="warn", - title=f"의도와 다른 부분 · {dimension}".rstrip(" ·") or "의도와 다른 부분", + title=f"의도와 다른 부분 · {dimension} · 턴 직후".replace(" · · ", " · ").rstrip(" ·"), body=_review_note_body_markdown(body or "권장 반응과 실제 반응에 차이가 있었어요."), quote=quote, ) @@ -1188,7 +1188,7 @@ def _review_note_from_turn_eval( return ReviewNote( author="평가 AI", tone="good", - title="적절한 개입", + title="적절한 개입 · 턴 직후", body=_review_note_body_markdown( note_text or ( @@ -1203,7 +1203,7 @@ def _review_note_from_turn_eval( return ReviewNote( author="평가 AI", tone="warn", - title="점검해볼 지점", + title="점검해볼 지점 · 턴 직후", body=_review_note_body_markdown(note_text), quote=quote, ) diff --git a/apps/api/app/test_client_reply_quality.py b/apps/api/app/test_client_reply_quality.py new file mode 100644 index 0000000..7df0a9b --- /dev/null +++ b/apps/api/app/test_client_reply_quality.py @@ -0,0 +1,73 @@ +"""Strict client reply quality gate regressions.""" + +from __future__ import annotations + +import unittest + +from .services import guardrail + + +class ClientReplyQualityGateTest(unittest.TestCase): + def test_blocks_role_meta_speech_variants(self) -> None: + samples = [ + "내담자 역할로 응답하겠습니다. 엄마가 가보라고 해서요.", + "AI로서 답변드리면 저는 우울한 학생입니다.", + "상담자 입장에서 보면 제 핵심신념은 무가치감입니다.", + "제 핵심신념은 저는 쓸모없다는 것입니다.", + ] + + for sample in samples: + with self.subTest(sample=sample): + result = guardrail.sanitize_client_reply(sample, ideation_stage=1, turn_seq=2) + self.assertTrue(result.needs_regeneration) + self.assertIn("role_meta", result.reasons) + + def test_blocks_repeated_greeting_after_opening_turn_only_when_greeting_starts_reply(self) -> None: + blocked = guardrail.sanitize_client_reply( + "안녕하세요. 처음 뵙겠습니다. 저는 서연이에요.", + ideation_stage=1, + turn_seq=3, + ) + first_turn = guardrail.sanitize_client_reply( + "안녕하세요. 엄마가 가보라고 해서 왔어요.", + ideation_stage=1, + turn_seq=1, + ) + quoted = guardrail.sanitize_client_reply( + "방금 선생님이 안녕하세요라고 말해서 더 어색했어요.", + ideation_stage=1, + turn_seq=3, + ) + + self.assertTrue(blocked.needs_regeneration) + self.assertIn("repeat_greeting_after_opening", blocked.reasons) + self.assertFalse(first_turn.needs_regeneration) + self.assertFalse(quoted.needs_regeneration) + + def test_blocks_near_duplicate_previous_client_reply_but_allows_short_overlap(self) -> None: + previous = "몰라요. 엄마가 그냥 가보라고 해서 왔어요." + duplicate = guardrail.sanitize_client_reply( + "몰라요. 엄마가 그냥 가보라고 해서 왔어요.", + ideation_stage=1, + previous_client_reply=previous, + ) + near_duplicate = guardrail.sanitize_client_reply( + "엄마가 그냥 가보라고 해서 왔어요. 몰라요.", + ideation_stage=1, + previous_client_reply=previous, + ) + allowed = guardrail.sanitize_client_reply( + "엄마가 가보라고 한 건 맞는데, 지금은 좀 짜증나요.", + ideation_stage=1, + previous_client_reply=previous, + ) + + self.assertTrue(duplicate.needs_regeneration) + self.assertIn("duplicate_client_reply", duplicate.reasons) + self.assertTrue(near_duplicate.needs_regeneration) + self.assertIn("duplicate_client_reply", near_duplicate.reasons) + self.assertFalse(allowed.needs_regeneration) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/api/app/test_orchestrator_masking.py b/apps/api/app/test_orchestrator_masking.py index 807107f..97771eb 100644 --- a/apps/api/app/test_orchestrator_masking.py +++ b/apps/api/app/test_orchestrator_masking.py @@ -92,9 +92,11 @@ class CaptureGenerateEngine: self.payload: dict[str, Any] | None = None self._payload_builder = EngineClient(base_url="http://engine.test") self.text = text + self.requests: list[Any] = [] async def generate(self, req): self.request = req + self.requests.append(req) self.payload = self._payload_builder._payload(req) return GenerateResponse( text=self.text, @@ -283,6 +285,26 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase): self.assertIn("자동적 사고", blob) self.assertIn("행동활성화", blob) + def test_prepare_turn_maps_recent_turns_from_client_ai_perspective(self) -> None: + ctx = orchestrator.prepare_turn( + session_id="history-session", + case_id="history-case", + card=persona.P1, + state=_initial_state(), + learner_text="그 말을 듣고 어떤 생각이 들었나요?", + memory=orchestrator.TurnMemory( + recent_turns=[ + {"speaker": "counselor", "text": "왜 상담에 오게 됐나요?"}, + {"speaker": "client", "text": "엄마가 가보라고 해서요."}, + ], + ), + ) + + recent = ctx.messages[-3:-1] + self.assertEqual([message.role for message in recent], ["user", "assistant"]) + self.assertEqual(recent[0].content, "왜 상담에 오게 됐나요?") + self.assertEqual(recent[1].content, "엄마가 가보라고 해서요.") + async def test_run_turn_generate_sends_only_masked_engine_payload(self) -> None: ctx = _prepare_context() engine = CaptureGenerateEngine() @@ -395,6 +417,78 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase): self.assertNotIn("[NAME]", result.client_reply or "") self.assertIn("그 이름은 그대로고", result.client_reply or "") + async def test_run_turn_generate_retries_once_after_role_meta_reply(self) -> None: + class SequenceEngine(CaptureGenerateEngine): + def __init__(self) -> None: + super().__init__("") + self.responses = [ + "내담자 역할로 응답하겠습니다. 엄마가 가보라고 해서요.", + "엄마가 그냥 가보라고 해서 왔어요.", + ] + + async def generate(self, req): + self.request = req + self.requests.append(req) + self.payload = self._payload_builder._payload(req) + text = self.responses.pop(0) + return GenerateResponse( + text=text, + model="fake-model", + provider="fake-provider", + tokens_in=3, + tokens_out=4, + cost_usd=0.0, + ) + + ctx = orchestrator.prepare_turn( + session_id="quality-session", + case_id="quality-case", + card=persona.P1, + state=_initial_state(), + learner_text="어머니가 오라고 하셨군요. 지금은 어떤 마음인가요?", + ) + engine = SequenceEngine() + + result = await orchestrator.run_turn_generate(ctx, engine) # type: ignore[arg-type] + + self.assertEqual(len(engine.requests), 2) + self.assertEqual(result.client_reply, "엄마가 그냥 가보라고 해서 왔어요.") + self.assertFalse(result.safety_flagged) + + async def test_run_turn_generate_returns_retryable_error_without_saving_bad_fallback(self) -> None: + class AlwaysBadEngine(CaptureGenerateEngine): + def __init__(self) -> None: + super().__init__("") + + async def generate(self, req): + self.request = req + self.requests.append(req) + self.payload = self._payload_builder._payload(req) + return GenerateResponse( + text="AI로서 내담자 역할로 응답하겠습니다.", + model="fake-model", + provider="fake-provider", + tokens_in=3, + tokens_out=4, + cost_usd=0.0, + ) + + ctx = orchestrator.prepare_turn( + session_id="quality-session", + case_id="quality-case", + card=persona.P1, + state=_initial_state(), + learner_text="지금 이 자리에서 가장 말하기 어려운 게 뭔가요?", + ) + engine = AlwaysBadEngine() + + result = await orchestrator.run_turn_generate(ctx, engine) # type: ignore[arg-type] + + self.assertEqual(len(engine.requests), 2) + self.assertIsNone(result.client_reply) + self.assertTrue(result.safety_flagged) + self.assertEqual(getattr(result, "output_error", None), "client_reply_quality_retryable") + async def test_run_turn_stream_humanizes_split_masked_placeholder_reply(self) -> None: ctx = orchestrator.prepare_turn( session_id="masking-session", @@ -422,6 +516,38 @@ class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase): self.assertIn("그 이름은 그대로고요.", streamed) self.assertEqual(events[-1].event, "done") + async def test_run_turn_stream_buffers_role_meta_reply_without_token_leak(self) -> None: + ctx = orchestrator.prepare_turn( + session_id="quality-stream-session", + case_id="quality-stream-case", + card=persona.P1, + state=_initial_state(), + learner_text="어머니가 오라고 하셨군요. 지금은 어떤 마음인가요?", + ) + engine = CaptureStreamEngine( + chunks=[ + "내담자 ", + "역할로 응답하겠습니다. 엄마가 가보라고 해서요.", + ] + ) + + events = [ + event + async for event in orchestrator.run_turn_stream( + ctx, + engine, # type: ignore[arg-type] + ) + ] + + self.assertEqual([event.event for event in events], ["safety", "done"]) + self.assertEqual( + "".join(str(event.data.get("text", "")) for event in events if event.event == "token"), + "", + ) + self.assertEqual(events[0].data["reason"], "client_reply_quality_retryable") + self.assertTrue(events[-1].data["safety_flagged"]) + self.assertEqual(events[-1].data["output_error"], "client_reply_quality_retryable") + if __name__ == "__main__": unittest.main() diff --git a/apps/api/app/test_session_turn_persistence.py b/apps/api/app/test_session_turn_persistence.py index 53edc3f..c181e1e 100644 --- a/apps/api/app/test_session_turn_persistence.py +++ b/apps/api/app/test_session_turn_persistence.py @@ -621,9 +621,10 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase): review_turn = review.turns[0] self.assertIsNotNone(review_turn.note) assert review_turn.note is not None - self.assertEqual(review_turn.note.title, "턴 평가 실패") - self.assertIn("fast-loop 평가를 완료하지 못했습니다", review_turn.note.body) - self.assertIn("RuntimeError", review_turn.note.body) + self.assertEqual(review_turn.note.title, "턴 직후 평가 실패") + self.assertIn("fast-loop(턴 직후) 평가를 완료하지 못했습니다", review_turn.note.body) + self.assertIn("AI 평가 재시도가 필요합니다", review_turn.note.body) + self.assertNotIn("RuntimeError", review_turn.note.body) self.assertNotIn("김서연", review_turn.note.body) self.assertNotIn("010-1234-5678", review_turn.note.body) @@ -1117,8 +1118,8 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase): async for event in orchestrator.run_turn_stream(ctx, FakeStreamEngine()) # type: ignore[arg-type] ] - self.assertEqual([event.event for event in events], ["token", "error"]) - self.assertIn("engine unavailable", events[1].data["detail"]) + self.assertEqual([event.event for event in events], ["error"]) + self.assertIn("engine unavailable", events[0].data["detail"]) async def test_stream_turn_engine_error_event_does_not_append_partial_turns(self) -> None: principal = _principal() @@ -1667,7 +1668,9 @@ class SessionTurnPersistenceTest(unittest.IsolatedAsyncioTestCase): self.assertFalse(response.reviewReady) self.assertTrue(response.degraded) self.assertEqual(response.supervisorState, "평가 실패") - self.assertIn("session evaluation timeout after 45s", response.summary) + self.assertIn("deep-loop 평가 AI 산출물을 표시하지 못했습니다", response.summary) + self.assertIn("AI 평가 재시도가 필요합니다", response.summary) + self.assertNotIn("session evaluation timeout after 45s", response.summary) self.assertEqual(response.rubric, []) self.assertEqual(response.goodMoments, []) self.assertEqual(response.growthPoints, []) diff --git a/apps/api/engine_gateway/gateway.py b/apps/api/engine_gateway/gateway.py index 01d8d67..9239791 100644 --- a/apps/api/engine_gateway/gateway.py +++ b/apps/api/engine_gateway/gateway.py @@ -338,22 +338,39 @@ async def close_session(sid: str): # session_id 가 오면 풀을 재사용해 멀티턴 prompt caching 이점을 살린다. # ════════════════════════════════════════════════════════════════════════════ -def _split_messages(messages: list[GwMessage]) -> GatewayPromptParts: - """EngineMessage[] → named prompt parts for the current gateway turn. - - - system 들은 합쳐서 --system-prompt 로 주입할 텍스트로. - - 마지막 user 발화를 이번 턴 stdin content 로. - - 상주 세션 재사용 시에는 풀이 이미 컨텍스트를 들고 있으므로 마지막 user 만 보냄. - """ +def _split_messages(messages: list[GwMessage], *, ai_role: AIRole | None = None) -> GatewayPromptParts: + """EngineMessage[] → named prompt parts for the current gateway turn.""" system_parts: list[str] = [] - last_user = "" + non_system: list[GwMessage] = [] for m in messages: if m.role == "system": system_parts.append(m.content) - elif m.role == "user": - last_user = m.content + else: + non_system.append(m) + + last_user_index: int | None = None + for index, m in enumerate(non_system): + if m.role == "user": + last_user_index = index + + last_user = "" + if last_user_index is not None: + last_user = non_system[last_user_index].content + + user_payload = last_user + if ai_role == "client" and last_user_index is not None: + history_parts: list[str] = [] + for m in non_system[:last_user_index]: + content = m.content.strip() + if not content: + continue + speaker = "상담자" if m.role == "user" else "내담자" + history_parts.append(f"{speaker}: {content}") + if history_parts: + user_payload = "[직전 대화]\n" + "\n".join(history_parts) + "\n\n[이번 상담자 발화]\n" + last_user + system_prompt = "\n\n".join(p for p in system_parts if p.strip()) - return GatewayPromptParts(system_prompt=system_prompt, user_payload=last_user) + return GatewayPromptParts(system_prompt=system_prompt, user_payload=user_payload) def _inject_schema(system_prompt: str, schema: Optional[dict[str, Any]]) -> str: @@ -400,7 +417,7 @@ async def _resolve_session(req: GwGenerateReq, system_prompt: str) -> tuple[Engi @app.post("/v1/generate") async def v1_generate(req: GwGenerateReq): """단발 생성 (평가 deep-loop, 회기종료 압축 등). GenerateResponse 호환 dict 반환.""" - prompt_parts = _split_messages(req.messages) + prompt_parts = _split_messages(req.messages, ai_role=req.ai_role) system_prompt = _inject_schema(prompt_parts.system_prompt, req.structured_schema) if not prompt_parts.user_payload: raise HTTPException(400, "no user message in payload") @@ -436,7 +453,7 @@ async def v1_generate(req: GwGenerateReq): @app.post("/v1/stream") async def v1_stream(req: GwGenerateReq): """SSE 토큰 스트림. data: 라인으로 텍스트 델타를 흘리고 done/error 프레이밍.""" - prompt_parts = _split_messages(req.messages) + prompt_parts = _split_messages(req.messages, ai_role=req.ai_role) system_prompt = _inject_schema(prompt_parts.system_prompt, req.structured_schema) if not prompt_parts.user_payload: raise HTTPException(400, "no user message in payload") diff --git a/apps/api/engine_gateway/test_gateway_model.py b/apps/api/engine_gateway/test_gateway_model.py index f10ed42..15ba19c 100644 --- a/apps/api/engine_gateway/test_gateway_model.py +++ b/apps/api/engine_gateway/test_gateway_model.py @@ -210,6 +210,39 @@ class GatewayModelTest(unittest.TestCase): self.assertEqual(parts.system_prompt, "system one\n\nsystem two") self.assertEqual(parts.user_payload, "current client") + def test_split_messages_injects_client_history_before_current_counselor_turn(self): + parts = gateway._split_messages( + [ + contract.EngineMessage(role="system", content="client persona system"), + contract.EngineMessage(role="user", content="상담자 이전 질문"), + contract.EngineMessage(role="assistant", content="내담자 이전 답변"), + contract.EngineMessage(role="user", content="이번 상담자 발화"), + ], + ai_role="client", + ) + + self.assertEqual(parts.system_prompt, "client persona system") + self.assertIn("[직전 대화]", parts.user_payload) + self.assertIn("상담자: 상담자 이전 질문", parts.user_payload) + self.assertIn("내담자: 내담자 이전 답변", parts.user_payload) + self.assertIn("[이번 상담자 발화]", parts.user_payload) + self.assertTrue(parts.user_payload.rstrip().endswith("이번 상담자 발화")) + + def test_split_messages_does_not_inject_history_for_evaluator_requests(self): + parts = gateway._split_messages( + [ + contract.EngineMessage(role="system", content="eval system"), + contract.EngineMessage(role="user", content="이전 평가 입력"), + contract.EngineMessage(role="assistant", content="이전 평가 출력"), + contract.EngineMessage(role="user", content="이번 평가 입력"), + ], + ai_role="evaluator", + ) + + self.assertEqual(parts.system_prompt, "eval system") + self.assertEqual(parts.user_payload, "이번 평가 입력") + self.assertNotIn("[직전 대화]", parts.user_payload) + def test_split_messages_preserves_no_user_payload_boundary(self): parts = gateway._split_messages( [contract.EngineMessage(role="system", content="system only")] diff --git a/apps/web/e2e/session-review.spec.ts b/apps/web/e2e/session-review.spec.ts index fe7bbd9..18de1e5 100644 --- a/apps/web/e2e/session-review.spec.ts +++ b/apps/web/e2e/session-review.spec.ts @@ -46,6 +46,31 @@ async function routeSupervisorCapableLearner(page: Page) { }); } +async function routeLearnerReviewUser(page: Page) { + await page.route("**/api/auth/me", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + user_id: "00000000-0000-0000-0000-000000000101", + email: "learner@hs.ac.kr", + role: "learner", + display_name: "E2E Learner", + admin_access: false, + super_admin: false, + account_status: "approved", + approval_required: false, + cohort_ids: [], + consent_at: 1782820000, + onboarding_completed_at: 1782820001, + nickname: "E2E Learner", + self_introduction: "", + avatar_url: "", + }), + }); + }); +} + function isReviewResponse(sessionId: string) { return (response: Response) => { const url = new URL(response.url()); @@ -201,6 +226,19 @@ test.describe("session review", () => { expect(reviewAttempts).toBeGreaterThanOrEqual(2); }); + test("shows supervisor evaluation scope boundaries", async ({ page }) => { + await routeSupervisorCapableLearner(page); + const sessionId = "supervisor-evaluation-scope"; + await routeFilledSessionReview(page, sessionId); + + await page.goto(`/teach/session/${sessionId}/review`); + + await expect(page.getByText("AI 평가 범위", { exact: true })).toBeVisible(); + await expect(page.getByText("현재 구현", { exact: true })).toBeVisible(); + await expect(page.getByText(/fast-loop.*deep-loop/)).toBeVisible(); + await expect(page.getByText("임상팀 확정 필요", { exact: true })).toBeVisible(); + }); + test("refreshes an ended review from evaluation pending to ready", async ({ page }) => { await routeSupervisorCapableLearner(page); await routePrepostMeasures(page); @@ -376,6 +414,10 @@ test.describe("session review", () => { await page.goto(`/teach/session/${sessionId}/review`); await expect(page.getByText("평가 실패")).toBeVisible(); + await expect( + page.getByText("저장된 축어록은 확인했지만 deep-loop 평가 AI 산출물을 표시하지 못했습니다. AI 평가 재시도가 필요합니다."), + ).toBeVisible(); + await expect(page.getByText("engine_error: evaluator timeout")).toHaveCount(0); await expect(page.getByRole("button", { name: "AI 평가 재시도" })).toBeVisible(); await expect(page.getByRole("button", { name: "검토 완료" })).toBeDisabled(); @@ -438,7 +480,10 @@ test.describe("session review", () => { await page.getByRole("button", { name: "AI 평가 재시도" }).click(); - await expect(page.getByText("engine_error: retry timeout")).toBeVisible(); + await expect( + page.getByText("AI 평가 재시도를 완료하지 못했습니다. 잠시 뒤 다시 실행해 주세요."), + ).toBeVisible(); + await expect(page.getByText("engine_error: retry timeout")).toHaveCount(0); await expect(page.getByText("평가 실패")).toBeVisible(); await expect(page.getByRole("button", { name: "AI 평가 재시도" })).toBeEnabled(); expect(reevaluateRequests).toBe(1); @@ -538,6 +583,50 @@ test.describe("session review", () => { await expectNoHorizontalOverflow(page); }); + test("keeps masked API transcript while naturalizing transcript placeholders", async ({ + page, + }) => { + await routeLearnerReviewUser(page); + await routePrepostMeasures(page); + const sessionId = "review-placeholder-display"; + const response = filledReviewResponse(sessionId); + response.turns[0] = { + ...response.turns[0], + text: "저는 [NAME]이고 [ORG]에서 상담받고 있어요.", + }; + if (!response.caseWorksheet) { + throw new Error("filled review fixture must include a worksheet"); + } + response.caseWorksheet.sections[0].items[0].evidence = [ + { + turnId: "t1", + speaker: "client", + quote: "저는 [NAME]이고 [ORG]에서 상담받고 있어요.", + }, + ]; + await page.route(`**/api/sessions/${sessionId}/review`, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(response), + }); + }); + + const reviewResponsePromise = page.waitForResponse(isReviewResponse(sessionId)); + await page.goto(`/learn/session/${sessionId}/review`); + const reviewResponse = await reviewResponsePromise; + await expectResponseOk(reviewResponse); + const apiReview = await reviewResponse.json(); + expect(apiReview.turns[0].text).toContain("[NAME]"); + + const transcript = page.locator(".sr-card--transcript"); + await expect(transcript).toContainText( + "저는 익명 내담자이고 소속 기관에서 상담받고 있어요.", + ); + await expect(transcript).not.toContainText("[NAME]"); + await expect(page.locator(".sr-ws-evidence").filter({ hasText: "[NAME]" })).toBeVisible(); + }); + test("stores learner pre/post pilot evidence without effect claims", async ({ page }) => { await signInAsLearner(page); await routeFilledSessionReview(page); diff --git a/apps/web/e2e/support.ts b/apps/web/e2e/support.ts index 3b61551..4090c87 100644 --- a/apps/web/e2e/support.ts +++ b/apps/web/e2e/support.ts @@ -21,6 +21,13 @@ function sleep(ms: number) { }); } +let e2eEmailCounter = 0; + +function uniqueE2EEmail(prefix: string, domain: string) { + e2eEmailCounter += 1; + return `${prefix}.${Date.now()}.${e2eEmailCounter}@${domain}`; +} + export async function withGlobalEngineConfigLock( label: string, action: () => Promise, @@ -64,7 +71,7 @@ export async function withGlobalEngineConfigLock( export async function signInAsLearner(page: Page) { const res = await page.request.post("/api/auth/dev-login", { data: { - email: "learner@hs.ac.kr", + email: uniqueE2EEmail("learner", "hs.ac.kr"), role: "learner", display_name: "E2E Learner", }, @@ -83,7 +90,7 @@ export async function signInAsLearner(page: Page) { export async function signInAsTeacher(page: Page) { const res = await page.request.post("/api/auth/dev-login", { data: { - email: "teacher@hs.ac.kr", + email: uniqueE2EEmail("teacher", "hs.ac.kr"), role: "teacher", display_name: "E2E Teacher", }, diff --git a/apps/web/src/lib/api.gen.ts b/apps/web/src/lib/api.gen.ts index c9f1b97..0c6d3cf 100644 --- a/apps/web/src/lib/api.gen.ts +++ b/apps/web/src/lib/api.gen.ts @@ -4006,6 +4006,8 @@ export interface components { crisis_resource?: components["schemas"]["CrisisResourceResponse"] | null; /** Effective Openness */ effective_openness: number; + /** Output Error */ + output_error?: string | null; /** * Safety Flagged * @default false diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 10a47fb..172dd9b 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -278,6 +278,7 @@ export interface SessionStreamDone { crisis_kind?: string; crisis_resource?: CrisisResource | null; conversation_stopped?: boolean; + output_error?: string | null; } function safeParse(data: string): unknown { @@ -346,6 +347,7 @@ export async function openSessionStream( crisis_kind: parsed.crisis_kind, crisis_resource: parsed.crisis_resource, conversation_stopped: parsed.conversation_stopped, + output_error: parsed.output_error, }; handlers.onDone?.(donePayload); return; diff --git a/apps/web/src/pages/Session.tsx b/apps/web/src/pages/Session.tsx index 2ce9597..c72c815 100644 --- a/apps/web/src/pages/Session.tsx +++ b/apps/web/src/pages/Session.tsx @@ -1503,6 +1503,11 @@ export default function Session() { if (done.conversation_stopped || done.crisis_resource) { conversationStopped = true; } + const qualityRetryable = done.output_error === "client_reply_quality_retryable"; + if (qualityRetryable) { + setTurnError("내담자 응답 품질을 확인하지 못했습니다. 발화를 조금 다듬어 다시 시도해 주세요."); + pushSignal("warn", "내담자 응답 재시도 필요"); + } if (clientId != null) { const id = clientId; @@ -1528,12 +1533,12 @@ export default function Session() { setUtterances((prev) => prev.map((u) => (u.id === learnerId ? { ...u, turnSeq: done.turn_seq } : u)), ); - if (!conversationStopped) { + if (!conversationStopped && !qualityRetryable) { pushSignal("warn", "내담자 응답 없음"); } } setAvatarState("listening"); - if (!conversationStopped) { + if (!conversationStopped && !qualityRetryable) { void requestLiveCoach({ learnerText: text, clientReply, diff --git a/apps/web/src/pages/SessionReview.tsx b/apps/web/src/pages/SessionReview.tsx index 60e6540..9c224b5 100644 --- a/apps/web/src/pages/SessionReview.tsx +++ b/apps/web/src/pages/SessionReview.tsx @@ -115,6 +115,35 @@ function renderSummary(summary: string) { }); } +function displayReviewSummary(summary: string) { + if (/(?:사유:\s*)?(?:engine_error|session evaluation timeout|timeout|RuntimeError)/i.test(summary)) { + return "저장된 축어록은 확인했지만 deep-loop 평가 AI 산출물을 표시하지 못했습니다. AI 평가 재시도가 필요합니다."; + } + return summary; +} + +function displayEvaluationRetryError(message: string) { + if (!message.trim()) return "AI 평가를 다시 실행하지 못했습니다."; + return "AI 평가 재시도를 완료하지 못했습니다. 잠시 뒤 다시 실행해 주세요."; +} + +const REVIEW_DISPLAY_PLACEHOLDERS: Record = { + "[NAME]": "익명 내담자", + "[ORG]": "소속 기관", + "[PHONE]": "연락처", + "[EMAIL]": "이메일", + "[RRN]": "주민등록번호", + "[ADDRESS]": "주소", +}; + +function displayTranscriptText(text: string) { + let next = text; + for (const [placeholder, label] of Object.entries(REVIEW_DISPLAY_PLACEHOLDERS)) { + next = next.split(placeholder).join(label); + } + return next; +} + function shouldPollReviewReady(data: SessionReviewResponse) { return ( !data.reviewReady && @@ -298,6 +327,27 @@ function SupervisorCallout({ note }: { note: ReviewNote }) { ); } +function EvaluationScopeNotice() { + return ( +
+
+ + AI 평가 범위 +
+
+
+
현재 구현
+
fast-loop는 턴 직후 발화를 점검하고, deep-loop는 종료 후 회기 전체를 재분석합니다.
+
+
+
임상팀 확정 필요
+
점수 해석 기준, 교수자 최종 판정 문구, 기관별 루브릭 가중치는 아직 확정 전입니다.
+
+
+
+ ); +} + function JumpablePoint({ point, turns, @@ -838,13 +888,13 @@ export default function SessionReview() { } setReviewReloadSeq((seq) => seq + 1); } catch (err) { - setEvaluationRetryError( + const message = err instanceof ApiError ? err.detail : err instanceof Error ? err.message - : "AI 평가를 다시 실행하지 못했습니다.", - ); + : ""; + setEvaluationRetryError(displayEvaluationRetryError(message)); } finally { setEvaluationRetrying(false); } @@ -1042,7 +1092,7 @@ export default function SessionReview() { 리뷰 요약

- {renderSummary(data.summary)} + {renderSummary(displayReviewSummary(data.summary))}

@@ -1250,7 +1300,7 @@ export default function SessionReview() { (turn.speaker === "client" ? " sr-turn__said--client" : "") } > - {turn.text} + {displayTranscriptText(turn.text)}

{turn.note ? : null}
@@ -1279,6 +1329,7 @@ export default function SessionReview() { {teacherReviewStatusLabel} +