대시보드 이슈 정리 1차
This commit is contained in:
Yun Chan
parent
94cc56592f
commit
f472883c31
13 changed files with 592 additions and 311 deletions
|
|
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
|
@ -39,6 +40,44 @@ FORBIDDEN_HEADER_TERMS = {
|
|||
|
||||
EMAIL_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE)
|
||||
PHONE_RE = re.compile(r"(?:\+?\d[\d .()-]{7,}\d)")
|
||||
APPROVED_EXPORT_STATUS = "approved_for_recursive_learning_seed"
|
||||
|
||||
CSV_ALLOWED_VALUES = {
|
||||
"00-intake/pilot_roster.csv": {
|
||||
"withdrawal_state": {
|
||||
"active",
|
||||
"withdrawn_before_data_use",
|
||||
"withdrawn_after_data_use",
|
||||
"excluded_by_operator",
|
||||
},
|
||||
},
|
||||
"01-sessions/session_completion.csv": {
|
||||
"completion_state": {
|
||||
"completed",
|
||||
"abandoned",
|
||||
"operator_cancelled",
|
||||
"excluded_from_analysis",
|
||||
},
|
||||
},
|
||||
"02-measures/prepost_measures.csv": {
|
||||
"timepoint": {"pre", "post"},
|
||||
},
|
||||
"04-privacy/withdrawal_log.csv": {
|
||||
"scope": {
|
||||
"future_sessions_only",
|
||||
"exclude_from_analysis",
|
||||
"exclude_from_export",
|
||||
"delete_where_policy_allows",
|
||||
},
|
||||
"status": {
|
||||
"received",
|
||||
"in_progress",
|
||||
"completed",
|
||||
"rejected_by_policy",
|
||||
"needs_legal_review",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -104,6 +143,27 @@ KPI_METRICS = {
|
|||
"top1",
|
||||
}
|
||||
|
||||
KPI_REPORT_KEYS = {
|
||||
"cohort_size",
|
||||
"exclusions",
|
||||
"generated_at",
|
||||
"metrics",
|
||||
"open_schema_gaps",
|
||||
"pilot_id",
|
||||
"review",
|
||||
"source_window",
|
||||
}
|
||||
|
||||
KPI_METRIC_REQUIRED_KEYS = {
|
||||
"denominator",
|
||||
"method",
|
||||
"numerator",
|
||||
"pass",
|
||||
"source_files",
|
||||
"threshold",
|
||||
"value",
|
||||
}
|
||||
|
||||
MANIFEST_KEYS = {
|
||||
"agreement",
|
||||
"anonymization",
|
||||
|
|
@ -114,8 +174,12 @@ MANIFEST_KEYS = {
|
|||
"export_id",
|
||||
"export_status",
|
||||
"files",
|
||||
"known_limitations",
|
||||
"pii_scan",
|
||||
"purpose",
|
||||
"selection_criteria",
|
||||
"source_tables",
|
||||
"source_window",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -182,6 +246,29 @@ def scan_csv_values(path: Path, report: Report, max_rows: int) -> None:
|
|||
report.error(f"{path}: cannot scan CSV: {exc}")
|
||||
|
||||
|
||||
def validate_csv_allowed_values(root: Path, spec: CsvSpec, report: Report) -> None:
|
||||
allowed_by_column = CSV_ALLOWED_VALUES.get(spec.path)
|
||||
if not allowed_by_column:
|
||||
return
|
||||
path = root / spec.path
|
||||
try:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for row_number, row in enumerate(reader, start=2):
|
||||
for column, allowed_values in allowed_by_column.items():
|
||||
value = (row.get(column) or "").strip()
|
||||
if value and value not in allowed_values:
|
||||
expected = ", ".join(sorted(allowed_values))
|
||||
report.error(
|
||||
f"{spec.path}:{row_number}: invalid {column} '{value}' "
|
||||
f"(expected one of: {expected})"
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{path}: not valid UTF-8/UTF-8-SIG")
|
||||
except OSError as exc:
|
||||
report.error(f"{path}: cannot validate CSV values: {exc}")
|
||||
|
||||
|
||||
def validate_csv(root: Path, spec: CsvSpec, report: Report, max_scan_rows: int) -> None:
|
||||
path = root / spec.path
|
||||
if not path.exists():
|
||||
|
|
@ -202,6 +289,7 @@ def validate_csv(root: Path, spec: CsvSpec, report: Report, max_scan_rows: int)
|
|||
report.error(f"{spec.path}: forbidden identifier/secret-like headers: {', '.join(forbidden)}")
|
||||
|
||||
scan_csv_values(path, report, max_rows=max_scan_rows)
|
||||
validate_csv_allowed_values(root, spec, report)
|
||||
|
||||
|
||||
def read_json(path: Path, report: Report) -> dict[str, Any] | None:
|
||||
|
|
@ -233,6 +321,10 @@ def validate_kpi_report(root: Path, report: Report) -> None:
|
|||
if data is None:
|
||||
return
|
||||
|
||||
missing_top_level = sorted(KPI_REPORT_KEYS - set(data))
|
||||
if missing_top_level:
|
||||
report.error(f"{rel_path}: missing keys: {', '.join(missing_top_level)}")
|
||||
|
||||
metrics = data.get("metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
report.error(f"{rel_path}: missing object key 'metrics'")
|
||||
|
|
@ -245,9 +337,70 @@ def validate_kpi_report(root: Path, report: Report) -> None:
|
|||
if not isinstance(metric, dict):
|
||||
report.error(f"{rel_path}: metric '{metric_name}' must be an object")
|
||||
continue
|
||||
for key in ("value", "threshold", "pass", "method", "source_files"):
|
||||
for key in sorted(KPI_METRIC_REQUIRED_KEYS):
|
||||
if key not in metric:
|
||||
report.warn(f"{rel_path}: metric '{metric_name}' missing '{key}'")
|
||||
report.error(f"{rel_path}: metric '{metric_name}' missing '{key}'")
|
||||
if "source_files" in metric and not isinstance(metric["source_files"], list):
|
||||
report.error(f"{rel_path}: metric '{metric_name}' source_files must be a list")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def numeric_value(value: Any) -> float | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def validate_manifest_file_entry(
|
||||
root: Path,
|
||||
rel_path: str,
|
||||
item: dict[str, Any],
|
||||
report: Report,
|
||||
*,
|
||||
approved: bool,
|
||||
) -> None:
|
||||
required_keys = ("path", "rows", "sha256", "schema")
|
||||
for key in required_keys:
|
||||
if key not in item:
|
||||
message = f"{rel_path}: file entry missing '{key}'"
|
||||
if approved:
|
||||
report.error(message)
|
||||
else:
|
||||
report.warn(message)
|
||||
file_path = item.get("path")
|
||||
expected_sha = item.get("sha256")
|
||||
if not isinstance(file_path, str) or not file_path.strip():
|
||||
if approved:
|
||||
report.error(f"{rel_path}: approved file entry requires non-empty path")
|
||||
return
|
||||
try:
|
||||
root_resolved = root.resolve()
|
||||
candidate = (root / file_path).resolve()
|
||||
candidate.relative_to(root_resolved)
|
||||
except ValueError:
|
||||
report.error(f"{rel_path}: file path escapes evidence root: {file_path}")
|
||||
return
|
||||
if not approved:
|
||||
return
|
||||
if not candidate.exists():
|
||||
report.error(f"{rel_path}: approved file missing: {file_path}")
|
||||
return
|
||||
if not candidate.is_file():
|
||||
report.error(f"{rel_path}: approved file path is not a file: {file_path}")
|
||||
return
|
||||
if isinstance(expected_sha, str) and expected_sha:
|
||||
actual_sha = sha256_file(candidate)
|
||||
if actual_sha != expected_sha:
|
||||
report.error(f"{rel_path}: sha256 mismatch for {file_path}")
|
||||
|
||||
|
||||
def validate_manifest(root: Path, report: Report) -> None:
|
||||
|
|
@ -265,7 +418,8 @@ def validate_manifest(root: Path, report: Report) -> None:
|
|||
report.error(f"{rel_path}: missing keys: {', '.join(missing)}")
|
||||
|
||||
status = data.get("export_status")
|
||||
if status == "approved_for_recursive_learning_seed":
|
||||
approved = status == APPROVED_EXPORT_STATUS
|
||||
if approved:
|
||||
approvals = data.get("approvals")
|
||||
if not isinstance(approvals, dict):
|
||||
report.error(f"{rel_path}: approved export requires approvals object")
|
||||
|
|
@ -273,16 +427,40 @@ def validate_manifest(root: Path, report: Report) -> None:
|
|||
for key in ("data_steward", "legal_or_privacy_reviewer", "technical_operator", "approved_at"):
|
||||
if not approvals.get(key):
|
||||
report.error(f"{rel_path}: approved export missing approval '{key}'")
|
||||
pii_scan = data.get("pii_scan")
|
||||
if not isinstance(pii_scan, dict):
|
||||
report.error(f"{rel_path}: approved export requires pii_scan object")
|
||||
elif pii_scan.get("status") != "pass":
|
||||
report.error(f"{rel_path}: approved export requires pii_scan.status='pass'")
|
||||
agreement = data.get("agreement")
|
||||
if not isinstance(agreement, dict):
|
||||
report.error(f"{rel_path}: approved export requires agreement object")
|
||||
else:
|
||||
kappa = numeric_value(agreement.get("kappa"))
|
||||
icc = numeric_value(agreement.get("icc"))
|
||||
if kappa is None or kappa < 0.60:
|
||||
report.error(f"{rel_path}: approved export requires agreement.kappa >= 0.60")
|
||||
if icc is None or icc < 0.75:
|
||||
report.error(f"{rel_path}: approved export requires agreement.icc >= 0.75")
|
||||
selection_criteria = data.get("selection_criteria")
|
||||
if not isinstance(selection_criteria, dict):
|
||||
report.error(f"{rel_path}: approved export requires selection_criteria object")
|
||||
elif selection_criteria.get("include_withdrawn") is not False:
|
||||
report.error(f"{rel_path}: approved export requires selection_criteria.include_withdrawn=false")
|
||||
consent_scope = data.get("consent_scope")
|
||||
allowed_uses = consent_scope.get("allowed_uses") if isinstance(consent_scope, dict) else None
|
||||
if not isinstance(allowed_uses, list) or "recursive_learning_seed" not in allowed_uses:
|
||||
report.error(f"{rel_path}: approved export requires recursive_learning_seed consent scope")
|
||||
|
||||
files = data.get("files")
|
||||
if isinstance(files, list):
|
||||
if approved and not files:
|
||||
report.error(f"{rel_path}: approved export requires at least one file entry")
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
report.error(f"{rel_path}: files[] entries must be objects")
|
||||
continue
|
||||
for key in ("path", "rows", "sha256", "schema"):
|
||||
if key not in item:
|
||||
report.warn(f"{rel_path}: file entry missing '{key}'")
|
||||
validate_manifest_file_entry(root, rel_path, item, report, approved=approved)
|
||||
elif files is not None:
|
||||
report.error(f"{rel_path}: 'files' must be a list")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue