yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

전 저장소 리팩터링과 SSOT 정비

Browse source
This commit is contained in:
Yun Chan 2026-07-15 21:31:30 +09:00
parent 14ecbd4e7d
commit 3dfddcac6f
173 changed files with 19679 additions and 6952 deletions
Download patch file Download diff file Expand all files Collapse all files

184
apps/api/app/services/dataset_export.py
View file

@ -52,7 +52,9 @@ PII_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
("email", re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.I)),
(
"phone",
re.compile(r"\b(?:\+?82[-. ]?)?(?:0?1[016789]|0[2-9]\d?)[-. ]?\d{3,4}[-. ]?\d{4}\b"),
re.compile(
r"\b(?:\+?82[-. ]?)?(?:0?1[016789]|0[2-9]\d?)[-. ]?\d{3,4}[-. ]?\d{4}\b"
),
),
("national_id", re.compile(r"\b\d{6}[- ]?[1-4]\d{6}\b")),
("student_id", re.compile(r"\b20\d{2}[- ]?\d{4,8}\b")),
@ -62,7 +64,12 @@ PII_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
r"(?i)\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|session[_-]?secret|cookie)\b\s*[:=]\s*\S+"
),
),
("secret", re.compile(r"\b(?:sk-[A-Za-z0-9_-]{12,}|AIza[0-9A-Za-z_-]{20,}|xox[baprs]-[A-Za-z0-9-]+)\b")),
(
"secret",
re.compile(
r"\b(?:sk-[A-Za-z0-9_-]{12,}|AIza[0-9A-Za-z_-]{20,}|xox[baprs]-[A-Za-z0-9-]+)\b"
),
),
)
@ -155,7 +162,13 @@ def scan_for_pii(value: Any, path: str = "$") -> list[dict[str, Any]]:
key = str(raw_key)
next_path = f"{path}.{key}"
if key.lower() in BLOCKED_FIELD_NAMES:
findings.append({"kind": "blocked_field", "path": next_path, "sample": f"<{key.lower()}>"})
findings.append(
{
"kind": "blocked_field",
"path": next_path,
"sample": f"<{key.lower()}>",
}
)
findings.extend(scan_for_pii(item, next_path))
return findings
if isinstance(value, list):
@ -195,11 +208,19 @@ def build_dataset_record(
"speaker": row.get("speaker") or "",
"text_masked": text_masked,
"techniques": json_safe(normalize_json_value(row.get("techniques") or [])),
"client_states": json_safe(normalize_json_value(row.get("client_states") or [])),
"feedback_scores": json_safe(normalize_json_value(row.get("feedback_scores") or [])),
"supervisor_comments": export_safe_supervisor_comments(row.get("supervisor_comments") or []),
"client_states": json_safe(
normalize_json_value(row.get("client_states") or [])
),
"feedback_scores": json_safe(
normalize_json_value(row.get("feedback_scores") or [])
),
"supervisor_comments": export_safe_supervisor_comments(
row.get("supervisor_comments") or []
),
"source_refs": {
"session_started_at": json_safe(row.get("session_started_at") or row.get("started_at")),
"session_started_at": json_safe(
row.get("session_started_at") or row.get("started_at")
),
"export_manifest_id": export_manifest_id,
},
"privacy": {
@ -214,7 +235,14 @@ def write_jsonl(records: Sequence[Mapping[str, Any]], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8", newline="\n") as handle:
for record in records:
handle.write(json.dumps(json_safe(record), ensure_ascii=False, sort_keys=True, separators=(",", ":")))
handle.write(
json.dumps(
json_safe(record),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
)
handle.write("\n")
@ -226,7 +254,9 @@ def sha256_file(path: Path) -> str:
return digest.hexdigest()
def cohen_kappa(annotations: Iterable[Mapping[str, Any]], label_key: str) -> float | None:
def cohen_kappa(
annotations: Iterable[Mapping[str, Any]], label_key: str
) -> float | None:
pairs: list[tuple[Any, Any]] = []
by_item: dict[Any, list[Any]] = defaultdict(list)
for annotation in annotations:
@ -244,13 +274,18 @@ def cohen_kappa(annotations: Iterable[Mapping[str, Any]], label_key: str) -> flo
observed = sum(1 for left, right in pairs if left == right) / total
left_counts = Counter(left for left, _ in pairs)
right_counts = Counter(right for _, right in pairs)
expected = sum((left_counts[label] / total) * (right_counts[label] / total) for label in set(left_counts) | set(right_counts))
expected = sum(
(left_counts[label] / total) * (right_counts[label] / total)
for label in set(left_counts) | set(right_counts)
)
if math.isclose(1.0, expected):
return 1.0 if math.isclose(1.0, observed) else None
return round((observed - expected) / (1.0 - expected), 4)
def intraclass_correlation(annotations: Iterable[Mapping[str, Any]], score_key: str) -> float | None:
def intraclass_correlation(
annotations: Iterable[Mapping[str, Any]], score_key: str
) -> float | None:
by_item: dict[Any, list[float]] = defaultdict(list)
for annotation in annotations:
labels = normalize_json_value(annotation.get("labels") or {})
@ -275,7 +310,9 @@ def intraclass_correlation(annotations: Iterable[Mapping[str, Any]], score_key:
residual = 0.0
for row_index, row in enumerate(matrix):
for col_index, value in enumerate(row):
residual += (value - row_means[row_index] - col_means[col_index] + grand_mean) ** 2
residual += (
value - row_means[row_index] - col_means[col_index] + grand_mean
) ** 2
mse = residual / ((n - 1) * (k - 1))
denominator = msr + (k - 1) * mse + (k * (msc - mse) / n)
if math.isclose(denominator, 0.0):
@ -284,7 +321,9 @@ def intraclass_correlation(annotations: Iterable[Mapping[str, Any]], score_key:
def infer_source_window(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
starts = [record.get("source_refs", {}).get("session_started_at") for record in records]
starts = [
record.get("source_refs", {}).get("session_started_at") for record in records
]
starts = [value for value in starts if value]
return {
"started_at": min(starts) if starts else "",
@ -292,35 +331,39 @@ def infer_source_window(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
}
def build_manifest(
*,
export_id: str,
dataset_name: str,
export_status: str,
purpose: str,
records: Sequence[Mapping[str, Any]],
jsonl_path: str,
jsonl_sha256: str,
pii_findings: Sequence[Mapping[str, Any]],
participants_included: int,
participants_excluded: int = 0,
cohort_id: str = "phase3",
consent_version: str = "",
agreement: Mapping[str, Any] | None = None,
approvals: Mapping[str, str] | None = None,
known_limitations: Sequence[str] | None = None,
created_at: datetime | None = None,
) -> dict[str, Any]:
if export_status not in EXPORT_STATUSES:
raise ValueError(f"unsupported export_status: {export_status}")
@dataclass(frozen=True, slots=True)
class DatasetManifestInput:
"""Approved/dry-run dataset artifact metadata crossing the export boundary."""
export_id: str
dataset_name: str
export_status: str
purpose: str
records: Sequence[Mapping[str, Any]]
jsonl_path: str
jsonl_sha256: str
pii_findings: Sequence[Mapping[str, Any]]
participants_included: int
participants_excluded: int = 0
cohort_id: str = "phase3"
consent_version: str = ""
agreement: Mapping[str, Any] | None = None
approvals: Mapping[str, str] | None = None
known_limitations: Sequence[str] | None = None
created_at: datetime | None = None
def build_manifest(spec: DatasetManifestInput) -> dict[str, Any]:
if spec.export_status not in EXPORT_STATUSES:
raise ValueError(f"unsupported export_status: {spec.export_status}")
agreement_payload = {
"kappa": None,
"icc": None,
"gold_status": "not_gold",
}
if agreement:
agreement_payload.update(dict(agreement))
if spec.agreement:
agreement_payload.update(dict(spec.agreement))
approvals_payload = {
"data_steward": "",
@ -328,23 +371,27 @@ def build_manifest(
"technical_operator": "",
"approved_at": "",
}
if approvals:
approvals_payload.update({key: value for key, value in approvals.items() if value is not None})
if spec.approvals:
approvals_payload.update(
{key: value for key, value in spec.approvals.items() if value is not None}
)
pii_status = "pass" if not pii_findings else "fail"
limitations = list(known_limitations or [])
if export_status != APPROVED_EXPORT_STATUS:
limitations.append("technical dry-run only; data-steward/legal approval is not complete")
if pii_findings:
pii_status = "pass" if not spec.pii_findings else "fail"
limitations = list(spec.known_limitations or [])
if spec.export_status != APPROVED_EXPORT_STATUS:
limitations.append(
"technical dry-run only; data-steward/legal approval is not complete"
)
if spec.pii_findings:
limitations.append("PII scan found records requiring reviewer disposition")
manifest = {
"export_id": export_id,
"dataset_name": dataset_name,
"export_status": export_status,
"created_at": json_safe(created_at or datetime.now(UTC)),
"purpose": purpose,
"source_window": infer_source_window(records),
"export_id": spec.export_id,
"dataset_name": spec.dataset_name,
"export_status": spec.export_status,
"created_at": json_safe(spec.created_at or datetime.now(UTC)),
"purpose": spec.purpose,
"source_window": infer_source_window(spec.records),
"source_tables": [
"app.sessions",
"app.turns",
@ -356,17 +403,17 @@ def build_manifest(
"ds.export_manifest",
],
"selection_criteria": {
"cohort_id": cohort_id,
"cohort_id": spec.cohort_id,
"min_completed_sessions": 0,
"include_withdrawn": False,
"excluded_safety_scope": ["self_harm_scenario_primary"],
},
"consent_scope": {
"consent_version": consent_version,
"consent_version": spec.consent_version,
"allowed_uses": ["education_quality_review", "recursive_learning_seed"],
"withdrawal_cutoff_applied_at": "",
"participants_included": participants_included,
"participants_excluded": participants_excluded,
"participants_included": spec.participants_included,
"participants_excluded": spec.participants_excluded,
},
"anonymization": {
"participant_key": "pseudonymous export key; no identity map included",
@ -379,14 +426,14 @@ def build_manifest(
"version": "1",
"ran_at": json_safe(datetime.now(UTC)),
"status": pii_status,
"findings": [json_safe(finding) for finding in pii_findings],
"findings": [json_safe(finding) for finding in spec.pii_findings],
},
"agreement": agreement_payload,
"files": [
{
"path": jsonl_path,
"rows": len(records),
"sha256": jsonl_sha256,
"path": spec.jsonl_path,
"rows": len(spec.records),
"sha256": spec.jsonl_sha256,
"schema": DATASET_ITEM_SCHEMA,
}
],
@ -411,13 +458,28 @@ def validate_manifest_gate(manifest: Mapping[str, Any]) -> None:
if (agreement.get("icc") or 0) < 0.75:
errors.append("ICC must be >= 0.75")
selection_criteria = manifest.get("selection_criteria") or {}
if not isinstance(selection_criteria, Mapping) or selection_criteria.get("include_withdrawn") is not False:
if (
not isinstance(selection_criteria, Mapping)
or selection_criteria.get("include_withdrawn") is not False
):
errors.append("include_withdrawn must be false")
consent_scope = manifest.get("consent_scope") or {}
allowed_uses = consent_scope.get("allowed_uses") if isinstance(consent_scope, Mapping) else None
if not isinstance(allowed_uses, list) or "recursive_learning_seed" not in allowed_uses:
allowed_uses = (
consent_scope.get("allowed_uses")
if isinstance(consent_scope, Mapping)
else None
)
if (
not isinstance(allowed_uses, list)
or "recursive_learning_seed" not in allowed_uses
):
errors.append("recursive_learning_seed consent scope is required")
for key in ("data_steward", "legal_or_privacy_reviewer", "technical_operator", "approved_at"):
for key in (
"data_steward",
"legal_or_privacy_reviewer",
"technical_operator",
"approved_at",
):
if not str(approvals.get(key) or "").strip():
errors.append(f"approval missing: {key}")
if errors: