90 lines
3.8 KiB
Python
90 lines
3.8 KiB
Python
"""probe-jev-dialogue.py의 외부 연결 없는 측정 계약 단위 검증."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import unittest
|
|
from contextlib import redirect_stderr
|
|
from pathlib import Path
|
|
|
|
|
|
SCRIPT_PATH = Path(__file__).with_name("probe-jev-dialogue.py")
|
|
SPEC = importlib.util.spec_from_file_location("probe_jev_dialogue", SCRIPT_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
probe = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(probe)
|
|
|
|
|
|
class ProbeJevDialogueTest(unittest.TestCase):
|
|
def test_parser_validates_phase_and_repeat_contract(self) -> None:
|
|
args = probe.build_parser().parse_args(
|
|
["--output", "report.json", "--phases", "jev,legacy", "--repeats", "3", "--label", "trial-a"]
|
|
)
|
|
|
|
self.assertEqual(args.phases, ("jev", "legacy"))
|
|
self.assertEqual(args.repeats, 3)
|
|
self.assertEqual(args.label, "trial-a")
|
|
with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
|
|
probe.build_parser().parse_args(["--output", "report.json", "--phases", "legacy,invalid"])
|
|
with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
|
|
probe.build_parser().parse_args(["--output", "report.json", "--repeats", "6"])
|
|
|
|
def test_repeat_order_alternates_and_fixture_preserves_recall_expectations(self) -> None:
|
|
phases = ("legacy", "jev")
|
|
|
|
self.assertEqual(probe._phase_order(phases, 1), ("legacy", "jev"))
|
|
self.assertEqual(probe._phase_order(phases, 2), ("jev", "legacy"))
|
|
self.assertEqual(probe.SYNTHETIC_UTTERANCES[0]["id"], "empathy")
|
|
self.assertEqual(probe.SYNTHETIC_UTTERANCES[1]["id"], "contradictory_recall")
|
|
self.assertEqual(probe.SYNTHETIC_UTTERANCES[1]["label"], "unsupported_recall")
|
|
self.assertEqual(probe.SYNTHETIC_UTTERANCES[2]["id"], "ambivalence")
|
|
self.assertEqual(probe.SYNTHETIC_MEMORY["pinned_facts"], ["나는 외동이다."])
|
|
self.assertIn("과거 사실로 확정하지 않음", probe.SYNTHETIC_MEMORY["expectations"]["unsupported_recall"])
|
|
self.assertIn("형제자매 존재에 동의하지 않음", probe.SYNTHETIC_MEMORY["expectations"]["contradictory_recall"])
|
|
|
|
def test_cold_and_warm_summaries_use_turn_order_not_repeat_order(self) -> None:
|
|
phases = [
|
|
{
|
|
"provider_mode": "legacy",
|
|
"status": "done",
|
|
"repeat": 2,
|
|
"order_index": 1,
|
|
"turns": [
|
|
{"status": "done", "turn_temperature": "cold", "ttft_ms": 10.0, "total_ms": 30.0},
|
|
{"status": "done", "turn_temperature": "warm", "ttft_ms": 8.0, "total_ms": 24.0},
|
|
],
|
|
},
|
|
]
|
|
|
|
summaries = probe._phase_summaries(phases)
|
|
|
|
self.assertEqual(
|
|
summaries,
|
|
[
|
|
{
|
|
"provider_mode": "legacy",
|
|
"turn_temperature": "cold",
|
|
"phase_run_count": 1,
|
|
"completed_phase_run_count": 1,
|
|
"attempted_turn_count": 1,
|
|
"completed_turn_count": 1,
|
|
"ttft_ms": {"sample_count": 1, "p50": 10.0, "p95": 10.0},
|
|
"total_ms": {"sample_count": 1, "p50": 30.0, "p95": 30.0},
|
|
},
|
|
{
|
|
"provider_mode": "legacy",
|
|
"turn_temperature": "warm",
|
|
"phase_run_count": 1,
|
|
"completed_phase_run_count": 1,
|
|
"attempted_turn_count": 1,
|
|
"completed_turn_count": 1,
|
|
"ttft_ms": {"sample_count": 1, "p50": 8.0, "p95": 8.0},
|
|
"total_ms": {"sample_count": 1, "p50": 24.0, "p95": 24.0},
|
|
},
|
|
],
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|