대시보드 폴드아웃/드릴다운 정리 + 페르소나 역린·misconduct 반응 + 게이트웨이 격리·RAG 비차단 수정
SSOT 대시보드:
- 한신대 기술분석 PDF(19쪽) 정합성 분석 + 이번 세션 발견 섹션 추가
- 섹션 폴드아웃(접기)·상단 목차(드릴다운)·모두 펼치기/접기 — 내용 보존, 레이아웃만 정리
페르소나 반응 강화('저항·반응 조절' 핵심 차별):
- PersonaCard.triggers(역린) 필드 + CCD 핵심상처 파생 역린 블록
- L0에 무례·모욕·조롱 시 현실적 동맹 균열 반응 지침
버그·성능 수정(라이브/E2E로 포착):
- 게이트웨이 페르소나 격리: --append-system-prompt를 --system-prompt(교체)로 + --exclude-dynamic-system-prompt-sections (내담자 캐릭터 붕괴·개발맥락 누출 차단)
- RAG: 임베더 동기 로드(약 7-13초)를 _warm_rag_caches 백그라운드 warm으로(세션 생성 블로킹 회귀 수정)
- voice TTS RMS 데드힌트 제거, init_state OpennessParams 파라미터객체화
- 한국어 PII(날짜·금액·주소) 마스킹 보강
- 레이아웃 시각 게이트: 폼 컨트롤 값 스크롤 오탐 제외(7/7)
검증: 백엔드 84/84, E2E 42(데스크톱 27·모바일 11·아바타 4), 시각 게이트 7/7
This commit is contained in:
Yun Chan
parent
cb2aebd76c
commit
085460b5e0
327 changed files with 31226 additions and 1829 deletions
396
scripts/check-phase3-artifacts.py
Normal file
396
scripts/check-phase3-artifacts.py
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Validate Phase 3 pilot evidence artifacts without modifying source data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
FORBIDDEN_HEADER_TERMS = {
|
||||
"address",
|
||||
"api_key",
|
||||
"birthdate",
|
||||
"cookie",
|
||||
"date_of_birth",
|
||||
"dob",
|
||||
"email",
|
||||
"full_name",
|
||||
"guardian_name",
|
||||
"identity_map",
|
||||
"national_id",
|
||||
"participant_name",
|
||||
"phone",
|
||||
"raw_audio",
|
||||
"raw_source_case",
|
||||
"raw_voice",
|
||||
"secret",
|
||||
"ssn",
|
||||
"student_id",
|
||||
"student_name",
|
||||
"token",
|
||||
}
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE)
|
||||
PHONE_RE = re.compile(r"(?:\+?\d[\d .()-]{7,}\d)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CsvSpec:
|
||||
path: str
|
||||
headers: tuple[str, ...]
|
||||
|
||||
|
||||
CSV_SPECS = (
|
||||
CsvSpec(
|
||||
"00-intake/pilot_roster.csv",
|
||||
(
|
||||
"participant_id",
|
||||
"cohort_id",
|
||||
"consent_version",
|
||||
"consent_signed_at",
|
||||
"withdrawal_state",
|
||||
"enrolled_at",
|
||||
),
|
||||
),
|
||||
CsvSpec(
|
||||
"00-intake/consent_receipts.csv",
|
||||
("participant_id", "consent_version", "signed_at", "signer_role", "receipt_id"),
|
||||
),
|
||||
CsvSpec(
|
||||
"01-sessions/session_completion.csv",
|
||||
(
|
||||
"participant_id",
|
||||
"session_id",
|
||||
"persona_id",
|
||||
"started_at",
|
||||
"ended_at",
|
||||
"completion_state",
|
||||
"turns_count",
|
||||
"supervisor_reviewed_at",
|
||||
),
|
||||
),
|
||||
CsvSpec(
|
||||
"02-measures/prepost_measures.csv",
|
||||
("participant_id", "measure_name", "timepoint", "score", "collected_at"),
|
||||
),
|
||||
CsvSpec(
|
||||
"02-measures/sus_responses.csv",
|
||||
("participant_id", "item", "response", "collected_at"),
|
||||
),
|
||||
CsvSpec(
|
||||
"04-privacy/withdrawal_log.csv",
|
||||
("participant_id", "requested_at", "effective_at", "scope", "status", "attestation_path"),
|
||||
),
|
||||
)
|
||||
|
||||
REQUIRED_MARKDOWN = ("04-privacy/privacy_audit.md",)
|
||||
|
||||
KPI_METRICS = {
|
||||
"embedding_consistency",
|
||||
"hallucination_rate",
|
||||
"icc",
|
||||
"inter_rater_kappa",
|
||||
"pilot_completion",
|
||||
"self_efficacy_prepost",
|
||||
"session_completion",
|
||||
"sus",
|
||||
"top1",
|
||||
}
|
||||
|
||||
MANIFEST_KEYS = {
|
||||
"agreement",
|
||||
"anonymization",
|
||||
"approvals",
|
||||
"consent_scope",
|
||||
"created_at",
|
||||
"dataset_name",
|
||||
"export_id",
|
||||
"export_status",
|
||||
"files",
|
||||
"pii_scan",
|
||||
"purpose",
|
||||
}
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self) -> None:
|
||||
self.errors: list[str] = []
|
||||
self.warnings: list[str] = []
|
||||
self.info: list[str] = []
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
self.errors.append(message)
|
||||
|
||||
def warn(self, message: str) -> None:
|
||||
self.warnings.append(message)
|
||||
|
||||
def note(self, message: str) -> None:
|
||||
self.info.append(message)
|
||||
|
||||
def to_dict(self, evidence_root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"evidence_root": str(evidence_root),
|
||||
"status": "pass" if not self.errors else "fail",
|
||||
"errors": self.errors,
|
||||
"warnings": self.warnings,
|
||||
"info": self.info,
|
||||
}
|
||||
|
||||
|
||||
def read_csv_header(path: Path, report: Report) -> list[str] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.reader(handle)
|
||||
header = next(reader, None)
|
||||
if not header:
|
||||
report.error(f"{path}: empty CSV or missing header")
|
||||
return None
|
||||
return [cell.strip() for cell in header]
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{path}: not valid UTF-8/UTF-8-SIG")
|
||||
except OSError as exc:
|
||||
report.error(f"{path}: cannot read CSV: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def scan_csv_values(path: Path, report: Report, max_rows: int) -> None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for row_number, row in enumerate(reader, start=2):
|
||||
if row_number > max_rows + 1:
|
||||
report.note(f"{path}: scanned first {max_rows} data rows for identifier patterns")
|
||||
return
|
||||
for column, value in row.items():
|
||||
if not value:
|
||||
continue
|
||||
if EMAIL_RE.search(value):
|
||||
report.error(f"{path}:{row_number}: possible email in column '{column}'")
|
||||
elif column and column.lower() not in {"turns_count", "score", "response", "item"}:
|
||||
if PHONE_RE.fullmatch(value.strip()):
|
||||
report.warn(f"{path}:{row_number}: possible phone-like value in column '{column}'")
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{path}: not valid UTF-8/UTF-8-SIG")
|
||||
except OSError as exc:
|
||||
report.error(f"{path}: cannot scan CSV: {exc}")
|
||||
|
||||
|
||||
def validate_csv(root: Path, spec: CsvSpec, report: Report, max_scan_rows: int) -> None:
|
||||
path = root / spec.path
|
||||
if not path.exists():
|
||||
report.error(f"missing required file: {spec.path}")
|
||||
return
|
||||
header = read_csv_header(path, report)
|
||||
if header is None:
|
||||
return
|
||||
|
||||
missing = [name for name in spec.headers if name not in header]
|
||||
if missing:
|
||||
report.error(f"{spec.path}: missing required headers: {', '.join(missing)}")
|
||||
|
||||
forbidden = sorted(
|
||||
column for column in header for term in FORBIDDEN_HEADER_TERMS if term in column.lower()
|
||||
)
|
||||
if forbidden:
|
||||
report.error(f"{spec.path}: forbidden identifier/secret-like headers: {', '.join(forbidden)}")
|
||||
|
||||
scan_csv_values(path, report, max_rows=max_scan_rows)
|
||||
|
||||
|
||||
def read_json(path: Path, report: Report) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except json.JSONDecodeError as exc:
|
||||
report.error(f"{path}: invalid JSON: {exc}")
|
||||
return None
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{path}: not valid UTF-8")
|
||||
return None
|
||||
except OSError as exc:
|
||||
report.error(f"{path}: cannot read JSON: {exc}")
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
report.error(f"{path}: top-level JSON must be an object")
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def validate_kpi_report(root: Path, report: Report) -> None:
|
||||
rel_path = "02-measures/kpi_report.json"
|
||||
path = root / rel_path
|
||||
if not path.exists():
|
||||
report.error(f"missing required file: {rel_path}")
|
||||
return
|
||||
data = read_json(path, report)
|
||||
if data is None:
|
||||
return
|
||||
|
||||
metrics = data.get("metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
report.error(f"{rel_path}: missing object key 'metrics'")
|
||||
return
|
||||
missing = sorted(KPI_METRICS - set(metrics))
|
||||
if missing:
|
||||
report.error(f"{rel_path}: missing metric keys: {', '.join(missing)}")
|
||||
|
||||
for metric_name, metric in metrics.items():
|
||||
if not isinstance(metric, dict):
|
||||
report.error(f"{rel_path}: metric '{metric_name}' must be an object")
|
||||
continue
|
||||
for key in ("value", "threshold", "pass", "method", "source_files"):
|
||||
if key not in metric:
|
||||
report.warn(f"{rel_path}: metric '{metric_name}' missing '{key}'")
|
||||
|
||||
|
||||
def validate_manifest(root: Path, report: Report) -> None:
|
||||
rel_path = "03-export/export_manifest.json"
|
||||
path = root / rel_path
|
||||
if not path.exists():
|
||||
report.error(f"missing required file: {rel_path}")
|
||||
return
|
||||
data = read_json(path, report)
|
||||
if data is None:
|
||||
return
|
||||
|
||||
missing = sorted(MANIFEST_KEYS - set(data))
|
||||
if missing:
|
||||
report.error(f"{rel_path}: missing keys: {', '.join(missing)}")
|
||||
|
||||
status = data.get("export_status")
|
||||
if status == "approved_for_recursive_learning_seed":
|
||||
approvals = data.get("approvals")
|
||||
if not isinstance(approvals, dict):
|
||||
report.error(f"{rel_path}: approved export requires approvals object")
|
||||
else:
|
||||
for key in ("data_steward", "legal_or_privacy_reviewer", "technical_operator", "approved_at"):
|
||||
if not approvals.get(key):
|
||||
report.error(f"{rel_path}: approved export missing approval '{key}'")
|
||||
|
||||
files = data.get("files")
|
||||
if isinstance(files, list):
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
report.error(f"{rel_path}: files[] entries must be objects")
|
||||
continue
|
||||
for key in ("path", "rows", "sha256", "schema"):
|
||||
if key not in item:
|
||||
report.warn(f"{rel_path}: file entry missing '{key}'")
|
||||
elif files is not None:
|
||||
report.error(f"{rel_path}: 'files' must be a list")
|
||||
|
||||
|
||||
def validate_markdown(root: Path, report: Report) -> None:
|
||||
for rel_path in REQUIRED_MARKDOWN:
|
||||
path = root / rel_path
|
||||
if not path.exists():
|
||||
report.error(f"missing required file: {rel_path}")
|
||||
continue
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{rel_path}: not valid UTF-8")
|
||||
continue
|
||||
except OSError as exc:
|
||||
report.error(f"{rel_path}: cannot read Markdown: {exc}")
|
||||
continue
|
||||
if not content.strip():
|
||||
report.error(f"{rel_path}: file is empty")
|
||||
if "Legal/privacy reviewer:" not in content:
|
||||
report.warn(f"{rel_path}: expected legal/privacy reviewer field")
|
||||
|
||||
|
||||
def validate(root: Path, max_scan_rows: int) -> Report:
|
||||
report = Report()
|
||||
if not root.exists():
|
||||
report.error(f"evidence root does not exist: {root}")
|
||||
return report
|
||||
if not root.is_dir():
|
||||
report.error(f"evidence root is not a directory: {root}")
|
||||
return report
|
||||
|
||||
for spec in CSV_SPECS:
|
||||
validate_csv(root, spec, report, max_scan_rows=max_scan_rows)
|
||||
validate_kpi_report(root, report)
|
||||
validate_manifest(root, report)
|
||||
validate_markdown(root, report)
|
||||
return report
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Read-only Phase 3 evidence checker. Writes only when --output is provided."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="Run validation checks. This is also the default behavior.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Alias for read-only validation; kept for operator clarity.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evidence-root",
|
||||
type=Path,
|
||||
default=Path("evidence/phase3"),
|
||||
help="Directory containing Phase 3 evidence files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
help="Optional JSON report path. No files are written unless this is set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Print the validation report as JSON instead of text.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-scan-rows",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Maximum data rows per CSV to scan for obvious identifier patterns.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def print_text_report(data: dict[str, Any]) -> None:
|
||||
print(f"Phase 3 artifact check: {data['status']}")
|
||||
print(f"Evidence root: {data['evidence_root']}")
|
||||
for label in ("errors", "warnings", "info"):
|
||||
items = data[label]
|
||||
if not items:
|
||||
continue
|
||||
print(f"\n{label.upper()}:")
|
||||
for item in items:
|
||||
print(f"- {item}")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
report = validate(args.evidence_root, max_scan_rows=max(0, args.max_scan_rows))
|
||||
data = report.to_dict(args.evidence_root)
|
||||
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(data, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(data, indent=2, ensure_ascii=True))
|
||||
else:
|
||||
print_text_report(data)
|
||||
|
||||
return 0 if not report.errors else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
Loading…
Add table
Add a link
Reference in a new issue