vignette/apps/api/app/test_orchestrator_masking.py
Yun Chan 403a36f9a6 스트리밍 턴의 감사 기록 저장을 done 뒤로 옮긴다
감사 기록은 실패해도 응답을 막지 않는 최선노력 저장인데, 커밋이 끝날 때까지
done을 붙잡아 음성 합성 시작도 늦췄다(운영 NAS에서 커밋당 0.3~0.9초). 지연
측정 위치와 인자는 그대로 두고 done을 먼저 내보낸다. 유일한 운영 소비자는 done
뒤에도 생성기를 끝까지 읽으므로 감사 행은 같은 요청 안에서 그대로 남는다.
2026-10-27 11:20:44 +09:00

716 lines
27 KiB
Python

"""Regression tests for P1 PII masking before engine requests."""
from __future__ import annotations
import json
import unittest
from typing import Any
from unittest.mock import patch
from .contracts.engine_gateway import EngineGatewaySseLineDecoder
from .engine_client import EngineClient, GenerateResponse
from .services import guardrail, orchestrator, persona, state_machine
RAW_PHONE = "010-1234-5678"
RAW_EMAIL = "test@example.com"
RAW_RRN = "990101-1234567"
RAW_TEXT = f"My phone is {RAW_PHONE}, email {RAW_EMAIL}, and RRN {RAW_RRN}."
RAW_VALUES = (RAW_PHONE, RAW_EMAIL, RAW_RRN)
MASK_VALUES = ("[PHONE]", "[EMAIL]", "[RRN]")
RAW_KO_NAME = "김서연"
RAW_KO_ORG = "한신대학교"
RAW_KO_DEPT = "상담심리학과"
RAW_KO_TEXT = (
f"내담자 {RAW_KO_NAME}은 {RAW_KO_ORG} {RAW_KO_DEPT} 학생이고 "
"연락은 하지 말아 주세요."
)
RAW_KO_VALUES = (RAW_KO_NAME, RAW_KO_ORG, RAW_KO_DEPT)
MASK_KO_VALUES = ("[NAME]", "[ORG]")
def _json_blob(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
def _message_blob(messages: object) -> str:
return "\n".join(message.content for message in messages) # type: ignore[attr-defined]
def _initial_state() -> state_machine.SessionState:
card = persona.P1
return state_machine.init_state(
params=card.openness_params(),
)
def _prepare_context() -> orchestrator.TurnContext:
return orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text=RAW_TEXT,
memory=orchestrator.TurnMemory(
recent_turns=[
{
"speaker": "counselor",
"text": "Previous learner contact was already masked: [PHONE] [EMAIL] [RRN].",
}
],
),
)
def _assert_no_raw_pii(test: unittest.TestCase, value: object) -> None:
blob = _json_blob(value)
for raw in RAW_VALUES:
test.assertNotIn(raw, blob)
def _assert_masked_pii_present(test: unittest.TestCase, value: object) -> None:
blob = _json_blob(value)
for masked in MASK_VALUES:
test.assertIn(masked, blob)
def _assert_no_raw_ko_pii(test: unittest.TestCase, value: object) -> None:
blob = _json_blob(value)
for raw in RAW_KO_VALUES:
test.assertNotIn(raw, blob)
def _assert_masked_ko_pii_present(test: unittest.TestCase, value: object) -> None:
blob = _json_blob(value)
for masked in MASK_KO_VALUES:
test.assertIn(masked, blob)
class CaptureGenerateEngine:
def __init__(self, text: str = "Masked engine reply.") -> None:
self.request = None
self.payload: dict[str, Any] | None = None
self._payload_builder = EngineClient(base_url="http://engine.test")
self.text = text
self.requests: list[Any] = []
async def generate(self, req):
self.request = req
self.requests.append(req)
self.payload = self._payload_builder._payload(req)
return GenerateResponse(
text=self.text,
model="fake-model",
provider="fake-provider",
tokens_in=3,
tokens_out=4,
cost_usd=0.0,
)
class CaptureStreamEngine:
engine_mode = "fake-provider"
default_model = "fake-model"
def __init__(self, chunks: list[str] | None = None) -> None:
self.request = None
self.payload: dict[str, Any] | None = None
self._payload_builder = EngineClient(base_url="http://engine.test")
self.chunks = chunks or ["Masked stream reply."]
async def stream(self, req):
self.request = req
self.payload = self._payload_builder._payload(req)
for chunk in self.chunks:
yield "event: token"
yield "data: " + json.dumps({"text": chunk}, ensure_ascii=False)
yield "event: done"
yield (
'data: {"provider":"fake-provider","model":"fake-model",'
'"tokens_in":5,"tokens_out":6,"cost_usd":0.0}'
)
async def stream_packets(self, req):
decoder = EngineGatewaySseLineDecoder()
async for raw in self.stream(req):
packet = decoder.feed_line(raw)
if packet is not None:
yield packet
class OrchestratorMaskingGateTest(unittest.IsolatedAsyncioTestCase):
def setUp(self) -> None:
self.client_affect_provider_patch = patch.object(
orchestrator.settings,
"client_affect_provider",
"legacy",
)
self.client_affect_provider_patch.start()
self.addCleanup(self.client_affect_provider_patch.stop)
self.presidio_patch = patch.object(
guardrail,
"_try_load_presidio",
return_value=(None, None),
)
self.presidio_patch.start()
guardrail.set_ko_pii_recognizer(None)
self.addCleanup(self.presidio_patch.stop)
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
def test_prepare_turn_keeps_raw_text_but_builds_masked_engine_messages(self) -> None:
ctx = _prepare_context()
self.assertEqual(ctx.learner_text_raw, RAW_TEXT)
for raw in RAW_VALUES:
self.assertIn(raw, ctx.learner_text_raw)
self.assertNotIn(raw, ctx.learner_text_masked)
self.assertNotIn(raw, _message_blob(ctx.messages))
for masked in MASK_VALUES:
self.assertIn(masked, ctx.learner_text_masked)
self.assertIn(masked, _message_blob(ctx.messages))
def test_prepare_turn_masks_raw_pii_from_context_inputs(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text="Current text has no identifiers.",
memory=orchestrator.TurnMemory(
recall_summary=f"Recall mentioned {RAW_PHONE}.",
pinned_facts=[f"Pinned email {RAW_EMAIL}."],
recent_turns=[
{"speaker": "counselor", "text": f"Previous raw RRN {RAW_RRN}."},
],
),
)
blob = _message_blob(ctx.messages)
for raw in RAW_VALUES:
self.assertNotIn(raw, blob)
for masked in MASK_VALUES:
self.assertIn(masked, blob)
def test_mask_pii_masks_korean_name_and_institution_context(self) -> None:
masked = guardrail.mask_pii(
f"이름: {RAW_KO_NAME}, 소속은 {RAW_KO_ORG} {RAW_KO_DEPT}입니다."
)
self.assertFalse(masked.used_presidio)
self.assertIn("NAME", masked.entities)
self.assertIn("ORG", masked.entities)
for raw in RAW_KO_VALUES:
self.assertNotIn(raw, masked.text_masked)
self.assertIn("[NAME]", masked.text_masked)
self.assertGreaterEqual(masked.text_masked.count("[ORG]"), 2)
def test_mask_pii_does_not_mask_common_korean_context_words_as_names(self) -> None:
masked = guardrail.mask_pii("학교 가는 게 힘들고 엄마랑 친구 이야기를 하면 불안해요.")
self.assertEqual(masked.text_masked, "학교 가는 게 힘들고 엄마랑 친구 이야기를 하면 불안해요.")
self.assertNotIn("NAME", masked.entities)
self.assertNotIn("ORG", masked.entities)
def test_mask_pii_does_not_mask_career_topic_as_name(self) -> None:
masked = guardrail.mask_pii("아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요.")
self.assertEqual(
masked.text_masked,
"아직 정해진 건 없어요. 진로는 그대로고 엄마한테 말도 못 했어요.",
)
self.assertNotIn("NAME", masked.entities)
def test_mask_pii_does_not_mask_evaluator_abstract_words_as_names(self) -> None:
# 실제 교수자 리뷰 화면에서 관측된 오탐 문형: 성씨 문자로 시작하는 일반
# 명사/용언(유의점·방치되는·마무리했을·연결감·표현함·어지러움)이
# 조사와 결합하면 문맥-없는 인명 패턴에 걸려 평가 문장이 훼손된다.
text = (
"유의점은 세 가지다. 정서 신호가 방치되는 것을 피하고, 회기를 "
"마무리했을 때 연결감과 작업동맹을 지키며, 감정을 표현함을 존중한다. "
"어지러움은 신체 신호로 다룬다. 뭘 할 수 있게 되는 건지 잘 "
"모르겠는데요."
)
masked = guardrail.mask_pii(text)
self.assertEqual(masked.text_masked, text)
self.assertNotIn("NAME", masked.entities)
def test_prepare_turn_masks_korean_pii_from_engine_messages(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text=RAW_KO_TEXT,
memory=orchestrator.TurnMemory(
recall_summary=f"지난 회기 요약에 {RAW_KO_NAME}과 {RAW_KO_ORG}가 남아 있었다.",
pinned_facts=[f"소속 {RAW_KO_DEPT}"],
recent_turns=[
{"speaker": "counselor", "text": f"{RAW_KO_NAME} 씨가 상담실에 왔다."},
],
),
)
blob = _message_blob(ctx.messages)
_assert_no_raw_ko_pii(self, blob)
_assert_masked_ko_pii_present(self, blob)
for raw in RAW_KO_VALUES:
self.assertIn(raw, ctx.learner_text_raw)
self.assertNotIn(raw, ctx.learner_text_masked)
def test_prepare_turn_applies_optional_ko_recognizer_before_regex_fallback(self) -> None:
class FakeKoRecognizer:
def analyze(self, text: str):
spans = []
for entity_type, value in (
("NAME", "보라별"),
("ORG", "미래학교상담연구랩"),
):
start = text.index(value)
spans.append(guardrail.PiiEntitySpan(entity_type, start, start + len(value)))
return spans
guardrail.set_ko_pii_recognizer(FakeKoRecognizer())
raw_text = "별명은 보라별이고 기관은 미래학교상담연구랩입니다. 전화는 010-1234-5678입니다."
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text=raw_text,
)
blob = _message_blob(ctx.messages)
self.assertIn("보라별", ctx.learner_text_raw)
self.assertIn("미래학교상담연구랩", ctx.learner_text_raw)
for raw in ("보라별", "미래학교상담연구랩", "010-1234-5678"):
self.assertNotIn(raw, ctx.learner_text_masked)
self.assertNotIn(raw, blob)
for masked in ("[NAME]", "[ORG]", "[PHONE]"):
self.assertIn(masked, ctx.learner_text_masked)
self.assertIn(masked, blob)
def test_prepare_turn_threads_theory_mode_into_engine_messages(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="theory-session",
case_id="theory-case",
card=persona.P2,
state=_initial_state(),
learner_text="그냥 아무것도 하기 싫어요.",
theory_mode="cbt",
)
blob = _message_blob(ctx.messages)
self.assertIn("[L3-T 이론모드: CBT]", blob)
self.assertIn("자동적 사고", blob)
self.assertIn("행동활성화", blob)
def test_prepare_turn_maps_recent_turns_from_client_ai_perspective(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="history-session",
case_id="history-case",
card=persona.P1,
state=_initial_state(),
learner_text="그 말을 듣고 어떤 생각이 들었나요?",
memory=orchestrator.TurnMemory(
recent_turns=[
{"speaker": "counselor", "text": "왜 상담에 오게 됐나요?"},
{"speaker": "client", "text": "엄마가 가보라고 해서요."},
],
),
)
recent = ctx.messages[-3:-1]
self.assertEqual([message.role for message in recent], ["user", "assistant"])
self.assertEqual(recent[0].content, "왜 상담에 오게 됐나요?")
self.assertEqual(recent[1].content, "엄마가 가보라고 해서요.")
async def test_run_turn_generate_sends_only_masked_engine_payload(self) -> None:
ctx = _prepare_context()
engine = CaptureGenerateEngine()
audit_payloads: list[dict[str, Any]] = []
async def audit_hook(payload: dict[str, Any]) -> None:
audit_payloads.append(payload)
await orchestrator.run_turn_generate(
ctx,
engine, # type: ignore[arg-type]
audit_hook=audit_hook,
)
self.assertIsNotNone(engine.request)
self.assertIsNotNone(engine.payload)
_assert_no_raw_pii(self, engine.request.messages)
_assert_no_raw_pii(self, engine.payload)
_assert_masked_pii_present(self, engine.request.messages)
_assert_masked_pii_present(self, engine.payload)
self.assertEqual(ctx.learner_text_raw, RAW_TEXT)
self.assertEqual(len(audit_payloads), 1)
self.assertEqual(audit_payloads[0]["provider"], "fake-provider")
self.assertEqual(audit_payloads[0]["model"], "fake-model")
_assert_no_raw_pii(self, audit_payloads)
for key in ("messages", "prompt", "text"):
self.assertNotIn(key, audit_payloads[0])
async def test_run_turn_stream_sends_only_masked_engine_payload(self) -> None:
ctx = _prepare_context()
engine = CaptureStreamEngine()
audit_payloads: list[dict[str, Any]] = []
async def audit_hook(payload: dict[str, Any]) -> None:
audit_payloads.append(payload)
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
audit_hook=audit_hook,
)
]
self.assertEqual(events[-1].event, "done")
self.assertEqual(
"".join(str(event.data.get("text", "")) for event in events if event.event == "token"),
"Masked stream reply.",
)
self.assertIsNotNone(engine.request)
self.assertIsNotNone(engine.payload)
_assert_no_raw_pii(self, engine.request.messages)
_assert_no_raw_pii(self, engine.payload)
_assert_masked_pii_present(self, engine.request.messages)
_assert_masked_pii_present(self, engine.payload)
self.assertEqual(ctx.learner_text_raw, RAW_TEXT)
self.assertEqual(len(audit_payloads), 1)
self.assertEqual(audit_payloads[0]["tokens_in"], 5)
self.assertEqual(audit_payloads[0]["tokens_out"], 6)
_assert_no_raw_pii(self, audit_payloads)
for key in ("messages", "prompt", "text"):
self.assertNotIn(key, audit_payloads[0])
async def test_private_affect_trace_is_absent_from_public_result_and_done_event(self) -> None:
ctx = _prepare_context()
engine = CaptureStreamEngine()
async def apply_private_trace(
context: orchestrator.TurnContext,
*,
audit_hook=None,
) -> None:
setattr(context, "client_affect_trace", {
"probabilities": [0.0, 0.0, 0.4, 0.6, 0.0],
})
with patch.object(orchestrator, "_apply_client_affect", apply_private_trace):
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
)
]
done = events[-1]
self.assertEqual(done.event, "done")
self.assertNotIn("client_affect_trace", done.data)
self.assertNotIn("probabilities", _json_blob(done.data))
generated = orchestrator.TurnResult(
turn_seq=ctx.state_after.turn_seq,
stage=ctx.state_after.stage.value,
effective_openness=ctx.state_after.effective_openness,
client_reply="Masked stream reply.",
safety_flagged=False,
state_after=ctx.state_after,
)
self.assertFalse(hasattr(generated, "client_affect_trace"))
async def test_run_turn_generate_sends_only_masked_korean_pii(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text=RAW_KO_TEXT,
)
engine = CaptureGenerateEngine()
audit_payloads: list[dict[str, Any]] = []
async def audit_hook(payload: dict[str, Any]) -> None:
audit_payloads.append(payload)
await orchestrator.run_turn_generate(
ctx,
engine, # type: ignore[arg-type]
audit_hook=audit_hook,
)
self.assertIsNotNone(engine.request)
self.assertIsNotNone(engine.payload)
_assert_no_raw_ko_pii(self, engine.request.messages)
_assert_no_raw_ko_pii(self, engine.payload)
_assert_masked_ko_pii_present(self, engine.request.messages)
_assert_masked_ko_pii_present(self, engine.payload)
_assert_no_raw_ko_pii(self, audit_payloads)
async def test_run_turn_generate_humanizes_masked_placeholder_reply(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text="진로 이야기를 이어가고 싶어요.",
)
engine = CaptureGenerateEngine(
text="아직 정해진 건 하나도 없어요. [NAME]는 그대로고, 엄마한테 뭐라고 말할지도 모르겠고요."
)
result = await orchestrator.run_turn_generate(
ctx,
engine, # type: ignore[arg-type]
)
self.assertNotIn("[NAME]", result.client_reply or "")
self.assertIn("그 이름은 그대로고", result.client_reply or "")
async def test_run_turn_generate_retries_once_after_role_meta_reply(self) -> None:
class SequenceEngine(CaptureGenerateEngine):
def __init__(self) -> None:
super().__init__("")
self.responses = [
"내담자 역할로 응답하겠습니다. 엄마가 가보라고 해서요.",
"엄마가 그냥 가보라고 해서 왔어요.",
]
async def generate(self, req):
self.request = req
self.requests.append(req)
self.payload = self._payload_builder._payload(req)
text = self.responses.pop(0)
return GenerateResponse(
text=text,
model="fake-model",
provider="fake-provider",
tokens_in=3,
tokens_out=4,
cost_usd=0.0,
)
ctx = orchestrator.prepare_turn(
session_id="quality-session",
case_id="quality-case",
card=persona.P1,
state=_initial_state(),
learner_text="어머니가 오라고 하셨군요. 지금은 어떤 마음인가요?",
)
engine = SequenceEngine()
result = await orchestrator.run_turn_generate(ctx, engine) # type: ignore[arg-type]
self.assertEqual(len(engine.requests), 2)
self.assertEqual(result.client_reply, "엄마가 그냥 가보라고 해서 왔어요.")
self.assertFalse(result.safety_flagged)
async def test_run_turn_generate_returns_retryable_error_without_saving_bad_fallback(self) -> None:
class AlwaysBadEngine(CaptureGenerateEngine):
def __init__(self) -> None:
super().__init__("")
async def generate(self, req):
self.request = req
self.requests.append(req)
self.payload = self._payload_builder._payload(req)
return GenerateResponse(
text="AI로서 내담자 역할로 응답하겠습니다.",
model="fake-model",
provider="fake-provider",
tokens_in=3,
tokens_out=4,
cost_usd=0.0,
)
ctx = orchestrator.prepare_turn(
session_id="quality-session",
case_id="quality-case",
card=persona.P1,
state=_initial_state(),
learner_text="지금 이 자리에서 가장 말하기 어려운 게 뭔가요?",
)
engine = AlwaysBadEngine()
result = await orchestrator.run_turn_generate(ctx, engine) # type: ignore[arg-type]
self.assertEqual(len(engine.requests), 2)
self.assertIsNone(result.client_reply)
self.assertTrue(result.safety_flagged)
self.assertEqual(getattr(result, "output_error", None), "client_reply_quality_retryable")
async def test_run_turn_stream_humanizes_split_masked_placeholder_reply(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="masking-session",
case_id="masking-case",
card=persona.P1,
state=_initial_state(),
learner_text="진로 이야기를 이어가고 싶어요.",
)
engine = CaptureStreamEngine(
chunks=["아직 정해진 건 하나도 없어요. ", "[NA", "ME]는", " 그대로고요."]
)
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
)
]
streamed = "".join(
str(event.data.get("text", "")) for event in events if event.event == "token"
)
self.assertNotIn("[NAME]", streamed)
self.assertIn("그 이름은 그대로고요.", streamed)
self.assertEqual(events[-1].event, "done")
async def test_run_turn_stream_buffers_role_meta_reply_without_token_leak(self) -> None:
ctx = orchestrator.prepare_turn(
session_id="quality-stream-session",
case_id="quality-stream-case",
card=persona.P1,
state=_initial_state(),
learner_text="어머니가 오라고 하셨군요. 지금은 어떤 마음인가요?",
)
engine = CaptureStreamEngine(
chunks=[
"내담자 ",
"역할로 응답하겠습니다. 엄마가 가보라고 해서요.",
]
)
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
)
]
self.assertEqual([event.event for event in events], ["safety", "done"])
self.assertEqual(
"".join(str(event.data.get("text", "")) for event in events if event.event == "token"),
"",
)
self.assertEqual(events[0].data["reason"], "client_reply_quality_retryable")
self.assertTrue(events[-1].data["safety_flagged"])
self.assertEqual(events[-1].data["output_error"], "client_reply_quality_retryable")
class StreamAuditAfterDoneTest(unittest.IsolatedAsyncioTestCase):
"""USER-LATENCY: done 이벤트가 감사 저장보다 먼저 사용자에게 전달돼야 한다."""
def setUp(self) -> None:
self.client_affect_provider_patch = patch.object(
orchestrator.settings,
"client_affect_provider",
"legacy",
)
self.client_affect_provider_patch.start()
self.addCleanup(self.client_affect_provider_patch.stop)
self.presidio_patch = patch.object(
guardrail,
"_try_load_presidio",
return_value=(None, None),
)
self.presidio_patch.start()
guardrail.set_ko_pii_recognizer(None)
self.addCleanup(self.presidio_patch.stop)
self.addCleanup(guardrail.set_ko_pii_recognizer, None)
async def test_done_event_is_observed_before_audit_hook_is_called(self) -> None:
ctx = _prepare_context()
engine = CaptureStreamEngine()
audit_payloads: list[dict[str, Any]] = []
async def audit_hook(payload: dict[str, Any]) -> None:
audit_payloads.append(payload)
done_seen_hook_call_count: int | None = None
done_event: Any = None
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
audit_hook=audit_hook,
):
if event.event == "done":
# 소비자가 done을 받는 바로 이 시점에 감사 hook은 아직 호출되지
# 않아야 한다(지연의 핵심: done이 사용자에게 먼저 전달된다).
done_seen_hook_call_count = len(audit_payloads)
done_event = event
self.assertIsNotNone(done_event)
self.assertEqual(done_seen_hook_call_count, 0)
self.assertEqual(len(audit_payloads), 1)
async def test_audit_hook_called_once_with_same_payload_as_before(self) -> None:
ctx = _prepare_context()
engine = CaptureStreamEngine()
audit_payloads: list[dict[str, Any]] = []
async def audit_hook(payload: dict[str, Any]) -> None:
audit_payloads.append(payload)
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
audit_hook=audit_hook,
)
]
done_event = events[-1]
self.assertEqual(done_event.event, "done")
self.assertEqual(len(audit_payloads), 1)
payload = audit_payloads[0]
self.assertEqual(payload["session_id"], ctx.session_id)
self.assertEqual(payload["provider"], "fake-provider")
self.assertEqual(payload["model"], "fake-model")
self.assertEqual(payload["tokens_in"], 5)
self.assertEqual(payload["tokens_out"], 6)
self.assertEqual(payload["cost_usd"], 0.0)
self.assertIsInstance(payload["latency_ms"], int)
self.assertGreaterEqual(payload["latency_ms"], 0)
self.assertEqual(payload["provider"], done_event.data["llm_provider"])
self.assertEqual(payload["model"], done_event.data["model"])
self.assertEqual(payload["tokens_in"], done_event.data["tokens_in"])
self.assertEqual(payload["tokens_out"], done_event.data["tokens_out"])
self.assertEqual(payload["cost_usd"], done_event.data["cost_usd"])
async def test_audit_hook_exception_does_not_break_the_stream(self) -> None:
ctx = _prepare_context()
engine = CaptureStreamEngine()
hook_calls: list[dict[str, Any]] = []
async def failing_audit_hook(payload: dict[str, Any]) -> None:
hook_calls.append(payload)
raise RuntimeError("audit sink unavailable")
events = [
event
async for event in orchestrator.run_turn_stream(
ctx,
engine, # type: ignore[arg-type]
audit_hook=failing_audit_hook,
)
]
self.assertEqual([event.event for event in events], ["token", "done"])
self.assertEqual(len(hook_calls), 1)
if __name__ == "__main__":
unittest.main()