70 lines
2.3 KiB
Python
70 lines
2.3 KiB
Python
import json
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from app.services import guardrail
|
|
from app.services.pii_masking_eval import evaluate_fixture, load_cases
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
FIXTURE_PATH = REPO_ROOT / "data" / "privacy" / "pii-masking-ko-fixtures.json"
|
|
SCRIPT_PATH = REPO_ROOT / "scripts" / "evaluate-pii-masking.py"
|
|
|
|
|
|
class PiiMaskingEvalTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.presidio_patch = patch.object(
|
|
guardrail,
|
|
"_try_load_presidio",
|
|
return_value=(None, None),
|
|
)
|
|
self.presidio_patch.start()
|
|
self.addCleanup(self.presidio_patch.stop)
|
|
|
|
def test_fixture_cases_are_valid_json_list(self) -> None:
|
|
cases = load_cases(FIXTURE_PATH)
|
|
|
|
self.assertGreaterEqual(len(cases), 5)
|
|
self.assertTrue(all(case.get("id") for case in cases))
|
|
self.assertTrue(all(case.get("text") for case in cases))
|
|
|
|
def test_ko_name_org_fixture_passes_without_raw_identifier_leak(self) -> None:
|
|
report = evaluate_fixture(FIXTURE_PATH)
|
|
|
|
self.assertTrue(report["passed"], report)
|
|
self.assertEqual(report["cases_failed"], 0)
|
|
self.assertEqual(report["expected_entity_recall"], 1.0)
|
|
self.assertEqual(report["forbidden_substring_removal"], 1.0)
|
|
self.assertEqual(report["unexpected_entity_violations"], 0)
|
|
blob = json.dumps(report, ensure_ascii=False)
|
|
for raw in ("김서연", "박민수", "한신대학교", "상담심리학과", "마음봄상담센터"):
|
|
self.assertNotIn(raw, blob)
|
|
|
|
def test_cli_reports_json_and_nonzero_gate_shape(self) -> None:
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-X",
|
|
"utf8",
|
|
str(SCRIPT_PATH),
|
|
"--fixtures",
|
|
str(FIXTURE_PATH),
|
|
"--json",
|
|
],
|
|
cwd=str(REPO_ROOT),
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
)
|
|
|
|
report = json.loads(completed.stdout)
|
|
self.assertTrue(report["passed"])
|
|
self.assertEqual(report["cases_total"], 5)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|