평가 및 화면 구조 정리
This commit is contained in:
Yun Chan
parent
1248ae8ca4
commit
391639c1de
44 changed files with 5816 additions and 4501 deletions
140
apps/api/app/services/case_worksheet_rubric.py
Normal file
140
apps/api/app/services/case_worksheet_rubric.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
"""Validation helpers for externally owned case worksheet rubrics.
|
||||
|
||||
The clinical team owns scoring criteria. This module only validates the
|
||||
machine-readable scaffold that lets those criteria live outside application
|
||||
code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
SCHEMA_VERSION = "vignette.case_worksheet_rubric.v1"
|
||||
VALID_STATUSES = {"scaffold_only", "draft", "approved"}
|
||||
EXPECTED_CONTENT_OWNER = "clinical_team"
|
||||
|
||||
|
||||
def load_rubric(path: Path) -> dict[str, Any]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("case worksheet rubric must be a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def validate_rubric(
|
||||
rubric: Mapping[str, Any],
|
||||
*,
|
||||
expected_item_keys: Mapping[str, set[str]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
schema_version = str(rubric.get("schema_version") or "")
|
||||
status = str(rubric.get("status") or "")
|
||||
scoring_enabled = bool(rubric.get("scoring_enabled"))
|
||||
sections = rubric.get("sections")
|
||||
|
||||
if schema_version != SCHEMA_VERSION:
|
||||
errors.append("schema_version must be vignette.case_worksheet_rubric.v1")
|
||||
if status not in VALID_STATUSES:
|
||||
errors.append("status must be one of scaffold_only, draft, approved")
|
||||
if str(rubric.get("content_owner") or "") != EXPECTED_CONTENT_OWNER:
|
||||
errors.append("content_owner must be clinical_team")
|
||||
if scoring_enabled and status != "approved":
|
||||
errors.append("scoring_enabled requires status=approved")
|
||||
if status == "approved":
|
||||
approval = rubric.get("approval")
|
||||
if not isinstance(approval, Mapping):
|
||||
errors.append("approved rubric requires approval metadata")
|
||||
else:
|
||||
if not str(approval.get("clinical_reviewer") or ""):
|
||||
errors.append("approved rubric requires approval.clinical_reviewer")
|
||||
if not str(approval.get("approved_at") or ""):
|
||||
errors.append("approved rubric requires approval.approved_at")
|
||||
|
||||
section_count = 0
|
||||
item_count = 0
|
||||
section_item_keys: dict[str, set[str]] = {}
|
||||
if not isinstance(sections, list) or not sections:
|
||||
errors.append("sections must be a non-empty list")
|
||||
else:
|
||||
seen_sections: set[str] = set()
|
||||
for section in sections:
|
||||
if not isinstance(section, Mapping):
|
||||
errors.append("each section must be an object")
|
||||
continue
|
||||
section_key = str(section.get("key") or "")
|
||||
if not section_key:
|
||||
errors.append("section.key is required")
|
||||
continue
|
||||
if section_key in seen_sections:
|
||||
errors.append(f"duplicate section key: {section_key}")
|
||||
seen_sections.add(section_key)
|
||||
section_count += 1
|
||||
items = section.get("items")
|
||||
if not isinstance(items, list) or not items:
|
||||
errors.append(f"{section_key}: items must be a non-empty list")
|
||||
continue
|
||||
seen_items: set[str] = set()
|
||||
for item in items:
|
||||
if not isinstance(item, Mapping):
|
||||
errors.append(f"{section_key}: each item must be an object")
|
||||
continue
|
||||
item_key = str(item.get("key") or "")
|
||||
if not item_key:
|
||||
errors.append(f"{section_key}: item.key is required")
|
||||
continue
|
||||
if item_key in seen_items:
|
||||
errors.append(f"{section_key}: duplicate item key: {item_key}")
|
||||
seen_items.add(item_key)
|
||||
item_count += 1
|
||||
criteria = item.get("criteria")
|
||||
score_scale = item.get("score_scale")
|
||||
if scoring_enabled:
|
||||
if not isinstance(criteria, list) or not criteria:
|
||||
errors.append(f"{section_key}.{item_key}: scoring requires non-empty criteria")
|
||||
if not _valid_score_scale(score_scale):
|
||||
errors.append(f"{section_key}.{item_key}: scoring requires a valid score_scale")
|
||||
elif not criteria:
|
||||
warnings.append(f"{section_key}.{item_key}: criteria pending clinical team input")
|
||||
section_item_keys[section_key] = seen_items
|
||||
|
||||
if expected_item_keys is not None:
|
||||
expected_sections = set(expected_item_keys)
|
||||
actual_sections = set(section_item_keys)
|
||||
for missing_section in sorted(expected_sections - actual_sections):
|
||||
errors.append(f"missing worksheet section: {missing_section}")
|
||||
for extra_section in sorted(actual_sections - expected_sections):
|
||||
errors.append(f"unexpected worksheet section: {extra_section}")
|
||||
for section_key in sorted(expected_sections & actual_sections):
|
||||
missing_items = expected_item_keys[section_key] - section_item_keys[section_key]
|
||||
extra_items = section_item_keys[section_key] - expected_item_keys[section_key]
|
||||
for item_key in sorted(missing_items):
|
||||
errors.append(f"{section_key}: missing worksheet item: {item_key}")
|
||||
for item_key in sorted(extra_items):
|
||||
errors.append(f"{section_key}: unexpected worksheet item: {item_key}")
|
||||
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"rubric_id": str(rubric.get("rubric_id") or ""),
|
||||
"status": status,
|
||||
"scoring_enabled": scoring_enabled,
|
||||
"sections_total": section_count,
|
||||
"items_total": item_count,
|
||||
"passed": not errors,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
def _valid_score_scale(value: object) -> bool:
|
||||
if not isinstance(value, Mapping):
|
||||
return False
|
||||
minimum = value.get("min")
|
||||
maximum = value.get("max")
|
||||
anchors = value.get("anchors")
|
||||
if not isinstance(minimum, int) or not isinstance(maximum, int) or minimum >= maximum:
|
||||
return False
|
||||
return isinstance(anchors, list) and len(anchors) >= 2
|
||||
|
|
@ -42,13 +42,21 @@ _KOREAN_SURNAME_CHARS = (
|
|||
"명기반왕금옥육인맹제모탁국어은편용예봉경"
|
||||
)
|
||||
_KOREAN_FULL_NAME = rf"[{_KOREAN_SURNAME_CHARS}][가-힣]{{1,3}}"
|
||||
_KOREAN_FULL_NAME_BEFORE_SUFFIX = rf"[{_KOREAN_SURNAME_CHARS}][가-힣]{{1,3}}?"
|
||||
_KOREAN_NAME_STOPWORDS = {
|
||||
"연락",
|
||||
"연락처",
|
||||
"이메일",
|
||||
"주민번호",
|
||||
"번호",
|
||||
"이름",
|
||||
"이야기",
|
||||
"생각",
|
||||
"마음",
|
||||
"기분",
|
||||
"상담",
|
||||
"기록",
|
||||
"진료",
|
||||
"학교",
|
||||
"엄마",
|
||||
"아빠",
|
||||
|
|
@ -82,12 +90,33 @@ _PII_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||
r"(?=$|[\s,.;!?。])"
|
||||
),
|
||||
),
|
||||
# 한국어 이름: "제 이름은 김서연입니다", "보호자 이름은 박민수입니다" 같은 자연 발화형 라벨.
|
||||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
r"(?P<prefix>(?:(?:제|저의|내|나의|보호자|학생|내담자|상담자|친구|엄마|아빠|어머니|아버지)\s+)?"
|
||||
r"(?:이름|성명|실명|본명)\s*(?:은|는|이|가)?\s*)"
|
||||
rf"(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
r"(?P<suffix>\s*(?:입니다|이에요|예요|이고|이고요|이라고|라고)?)"
|
||||
r"(?=$|[\s,.;!?。])"
|
||||
),
|
||||
),
|
||||
# 한국어 이름: "저는 김서연입니다", "제가 박민수예요", "김서연입니다" 같은 자기소개형 문장.
|
||||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
r"(?P<prefix>(?:(?:저는|나는|제가|내가)\s*)?)"
|
||||
rf"(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
r"(?P<suffix>\s*(?:입니다|이에요|예요|이고|이고요))"
|
||||
r"(?=$|[\s,.;!?。])"
|
||||
),
|
||||
),
|
||||
# 한국어 이름: 역할/관계 명사 뒤에 붙은 인명 + 조사/호칭.
|
||||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
r"(?P<prefix>(?:내담자|상담자|학생|보호자|담임|교수|선생님|친구|엄마|아빠|어머니|아버지|동생|언니|오빠|형|누나)\s+)"
|
||||
rf"(?P<value>{_KOREAN_FULL_NAME})"
|
||||
rf"(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
r"(?P<suffix>\s*(?:님|씨|학생|상담자|내담자)?"
|
||||
r"(?:은|는|이|가|을|를|와|과|에게|한테|라고|이라는|입니다|이에요|예요|이고|이고요))"
|
||||
),
|
||||
|
|
@ -96,7 +125,7 @@ _PII_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_FULL_NAME})"
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
r"(?P<suffix>(?:은|는|이|가|을|를|와|과|에게|한테|라고|이라는))"
|
||||
),
|
||||
),
|
||||
|
|
@ -104,21 +133,21 @@ _PII_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||
(
|
||||
"NAME",
|
||||
re.compile(
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_FULL_NAME})"
|
||||
rf"(?<![가-힣])(?P<value>{_KOREAN_FULL_NAME_BEFORE_SUFFIX})"
|
||||
r"(?P<suffix>\s?(?:씨|님)(?:은|는|이|가|을|를|와|과|에게|한테|고|이고|인데)?)"
|
||||
r"(?=$|[\s,.;!?。])"
|
||||
),
|
||||
),
|
||||
# 주민등록번호 (6자리-7자리)
|
||||
("RRN", re.compile(r"\b\d{6}[-\s]?\d{7}\b")),
|
||||
("RRN", re.compile(r"(?<!\d)\d{6}[-\s]?\d{7}(?!\d)")),
|
||||
# 휴대폰 (010-1234-5678 등)
|
||||
("PHONE", re.compile(r"\b01[016789][-\s]?\d{3,4}[-\s]?\d{4}\b")),
|
||||
("PHONE", re.compile(r"(?<!\d)01[016789][-\s]?\d{3,4}[-\s]?\d{4}(?!\d)")),
|
||||
# 일반 전화
|
||||
("PHONE", re.compile(r"\b0\d{1,2}[-\s]?\d{3,4}[-\s]?\d{4}\b")),
|
||||
("PHONE", re.compile(r"(?<!\d)0\d{1,2}[-\s]?\d{3,4}[-\s]?\d{4}(?!\d)")),
|
||||
# 이메일
|
||||
("EMAIL", re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b")),
|
||||
# 카드/계좌 유사 긴 숫자열 (12자리 이상)
|
||||
("NUMID", re.compile(r"\b\d{12,}\b")),
|
||||
("NUMID", re.compile(r"(?<!\d)\d{12,}(?!\d)")),
|
||||
# 구체적 날짜(생년월일 등): 2001.4.18 / 2001-04-18 / 2001년 4월 18일
|
||||
("DATE", re.compile(r"(?:19|20)\d{2}\s?[.\-/년]\s?\d{1,2}\s?[.\-/월]\s?\d{1,2}\s?일?")),
|
||||
# 금액(원): 1,200원 / 1200원 (3자리+ 또는 콤마구분) — 식별 맥락 보호
|
||||
|
|
|
|||
|
|
@ -10,6 +10,16 @@ from . import guardrail
|
|||
|
||||
MaskFunc = Callable[[str], guardrail.MaskResult]
|
||||
|
||||
REPORT_SCHEMA_VERSION = "vignette.pii_masking_eval_report.v1"
|
||||
INPUT_SCHEMA_VERSION = "vignette.pii_masking_eval_input.v1"
|
||||
|
||||
DEFAULT_CASE_META = {
|
||||
"locale": "ko-KR",
|
||||
"source": "synthetic",
|
||||
"category": "unspecified",
|
||||
"severity": "medium",
|
||||
}
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[dict[str, Any]]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
|
@ -18,9 +28,18 @@ def load_cases(path: Path) -> list[dict[str, Any]]:
|
|||
return [dict(item) for item in data]
|
||||
|
||||
|
||||
def evaluate_case(case: Mapping[str, Any], *, mask_func: MaskFunc = guardrail.mask_pii) -> dict[str, Any]:
|
||||
def evaluate_case(
|
||||
case: Mapping[str, Any],
|
||||
*,
|
||||
mask_func: MaskFunc = guardrail.mask_pii,
|
||||
include_evidence_text: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
case_id = str(case.get("id") or "")
|
||||
text = str(case.get("text") or "")
|
||||
locale = str(case.get("locale") or DEFAULT_CASE_META["locale"])
|
||||
source = str(case.get("source") or DEFAULT_CASE_META["source"])
|
||||
category = str(case.get("category") or DEFAULT_CASE_META["category"])
|
||||
severity = str(case.get("severity") or DEFAULT_CASE_META["severity"])
|
||||
result = mask_func(text)
|
||||
entities = set(result.entities)
|
||||
expected_entities = {str(item) for item in case.get("expected_entities") or []}
|
||||
|
|
@ -34,25 +53,36 @@ def evaluate_case(case: Mapping[str, Any], *, mask_func: MaskFunc = guardrail.ma
|
|||
required_missing = [item for item in required_substrings if item and item not in result.text_masked]
|
||||
passed = not (missing_entities or unexpected_detected or forbidden_remaining or required_missing)
|
||||
|
||||
return {
|
||||
report = {
|
||||
"id": case_id,
|
||||
"locale": locale,
|
||||
"source": source,
|
||||
"category": category,
|
||||
"severity": severity,
|
||||
"passed": passed,
|
||||
"entities": sorted(entities),
|
||||
"masked_text": result.text_masked,
|
||||
"missing_entities": missing_entities,
|
||||
"unexpected_entities": unexpected_detected,
|
||||
"forbidden_remaining": forbidden_remaining,
|
||||
"forbidden_remaining_count": len(forbidden_remaining),
|
||||
"required_missing": required_missing,
|
||||
}
|
||||
if include_evidence_text:
|
||||
report["masked_text"] = result.text_masked
|
||||
report["forbidden_remaining"] = forbidden_remaining
|
||||
return report
|
||||
|
||||
|
||||
def evaluate_cases(
|
||||
cases: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
mask_func: MaskFunc = guardrail.mask_pii,
|
||||
include_evidence_text: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
case_list = list(cases)
|
||||
results = [evaluate_case(case, mask_func=mask_func) for case in case_list]
|
||||
results = [
|
||||
evaluate_case(case, mask_func=mask_func, include_evidence_text=include_evidence_text)
|
||||
for case in case_list
|
||||
]
|
||||
total_expected_entities = 0
|
||||
matched_expected_entities = 0
|
||||
total_forbidden = 0
|
||||
|
|
@ -64,11 +94,17 @@ def evaluate_cases(
|
|||
total_expected_entities += len(expected_entities)
|
||||
matched_expected_entities += len(expected_entities) - len(result["missing_entities"])
|
||||
total_forbidden += len(forbidden)
|
||||
removed_forbidden += len(forbidden) - len(result["forbidden_remaining"])
|
||||
remaining_forbidden = int(result.get("forbidden_remaining_count", len(result.get("forbidden_remaining", []))))
|
||||
removed_forbidden += len(forbidden) - remaining_forbidden
|
||||
unexpected_violations += len(result["unexpected_entities"])
|
||||
|
||||
passed_cases = sum(1 for result in results if result["passed"])
|
||||
return {
|
||||
"schema_version": REPORT_SCHEMA_VERSION,
|
||||
"input_schema_version": INPUT_SCHEMA_VERSION,
|
||||
"run_mode": "technical_dry_run",
|
||||
"data_source": "local_fixture",
|
||||
"evidence_text_included": include_evidence_text,
|
||||
"passed": passed_cases == len(results),
|
||||
"cases_total": len(results),
|
||||
"cases_passed": passed_cases,
|
||||
|
|
@ -76,15 +112,45 @@ def evaluate_cases(
|
|||
"expected_entity_recall": _ratio(matched_expected_entities, total_expected_entities),
|
||||
"forbidden_substring_removal": _ratio(removed_forbidden, total_forbidden),
|
||||
"unexpected_entity_violations": unexpected_violations,
|
||||
"by_source": _breakdown(case_list, results, "source"),
|
||||
"by_category": _breakdown(case_list, results, "category"),
|
||||
"by_severity": _breakdown(case_list, results, "severity"),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_fixture(path: Path, *, mask_func: MaskFunc = guardrail.mask_pii) -> dict[str, Any]:
|
||||
return evaluate_cases(load_cases(path), mask_func=mask_func)
|
||||
def evaluate_fixture(
|
||||
path: Path,
|
||||
*,
|
||||
mask_func: MaskFunc = guardrail.mask_pii,
|
||||
include_evidence_text: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
return evaluate_cases(load_cases(path), mask_func=mask_func, include_evidence_text=include_evidence_text)
|
||||
|
||||
|
||||
def _ratio(numerator: int, denominator: int) -> float:
|
||||
if denominator <= 0:
|
||||
return 1.0
|
||||
return round(numerator / denominator, 4)
|
||||
|
||||
|
||||
def _case_meta(case: Mapping[str, Any], key: str) -> str:
|
||||
fallback = DEFAULT_CASE_META.get(key, "unspecified")
|
||||
return str(case.get(key) or fallback)
|
||||
|
||||
|
||||
def _breakdown(
|
||||
cases: list[Mapping[str, Any]],
|
||||
results: list[Mapping[str, Any]],
|
||||
key: str,
|
||||
) -> dict[str, dict[str, int]]:
|
||||
grouped: dict[str, dict[str, int]] = {}
|
||||
for case, result in zip(cases, results):
|
||||
value = _case_meta(case, key)
|
||||
bucket = grouped.setdefault(value, {"cases_total": 0, "cases_passed": 0, "cases_failed": 0})
|
||||
bucket["cases_total"] += 1
|
||||
if result.get("passed"):
|
||||
bucket["cases_passed"] += 1
|
||||
else:
|
||||
bucket["cases_failed"] += 1
|
||||
return dict(sorted(grouped.items()))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue