G7 독립 평가 입력 검증 도구 추가
This commit is contained in:
Yun Chan
parent
fcc45e083e
commit
3067b72526
8 changed files with 881 additions and 5 deletions
578
scripts/prepare-g7-human-voice-gain-intake.py
Normal file
578
scripts/prepare-g7-human-voice-gain-intake.py
Normal file
|
|
@ -0,0 +1,578 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Prepare or compile a deidentified G7 human voice-gain intake.
|
||||
|
||||
Template mode creates empty, explicitly non-evidence input files. Compile mode
|
||||
accepts only deidentified metrics, recomputes reliability and production gates,
|
||||
and writes a final pack only when every production requirement passes. Console
|
||||
output never includes input paths, participant/labeler keys, labels, or values
|
||||
from invalid cells.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps/api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.contracts.g7_external_evidence import ( # noqa: E402
|
||||
G7HumanVoiceGainEvidencePack,
|
||||
)
|
||||
from app.services.g7_voice_gain_evidence import ( # noqa: E402
|
||||
evaluate_human_voice_gain,
|
||||
)
|
||||
|
||||
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
PARTICIPANTS_NAME = "participants.csv"
|
||||
LABELERS_NAME = "labelers.csv"
|
||||
OBSERVATIONS_NAME = "observations.csv"
|
||||
README_NAME = "README.md"
|
||||
INTAKE_SCHEMA_VERSION = "g7_human_voice_gain_intake_v1"
|
||||
|
||||
MANIFEST_KEYS = {
|
||||
"intake_schema_version",
|
||||
"template_only",
|
||||
"provenance",
|
||||
"text_only_model",
|
||||
"voice_enabled_model",
|
||||
"power_plan",
|
||||
"reported_icc",
|
||||
"reported_categorical_kappa",
|
||||
"reliability_report_sha256",
|
||||
}
|
||||
PARTICIPANT_FIELDS = (
|
||||
"participant_key",
|
||||
"split",
|
||||
"consent_receipt_sha256",
|
||||
)
|
||||
LABELER_FIELDS = (
|
||||
"labeler_key",
|
||||
"blinded_to_model_condition",
|
||||
"blinded_to_other_labelers",
|
||||
"labeled_independently",
|
||||
"attestation_sha256",
|
||||
)
|
||||
OBSERVATION_FIELDS = (
|
||||
"observation_id",
|
||||
"participant_key",
|
||||
"session_key",
|
||||
"axis",
|
||||
"text_only_status",
|
||||
"text_only_score",
|
||||
"voice_enabled_status",
|
||||
"voice_enabled_score",
|
||||
"labeler_key",
|
||||
"label_score",
|
||||
"label_category",
|
||||
)
|
||||
|
||||
DEIDENTIFIED_PARTICIPANT_RE = re.compile(
|
||||
r"^(?:calibration|held)-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$"
|
||||
)
|
||||
DEIDENTIFIED_LABELER_RE = re.compile(r"^labeler-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$")
|
||||
DEIDENTIFIED_SESSION_RE = re.compile(r"^session-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IntakeError(Exception):
|
||||
code: str
|
||||
source: str = "intake"
|
||||
row: int | None = None
|
||||
field: str | None = None
|
||||
|
||||
|
||||
def _base_report() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "vignette.g7-human-voice-gain-intake.v1",
|
||||
"passed": False,
|
||||
"pack_written": False,
|
||||
"clinical_claim_allowed": False,
|
||||
"privacy_boundary": {
|
||||
"input_paths_logged": False,
|
||||
"participant_keys_logged": False,
|
||||
"labeler_keys_logged": False,
|
||||
"labels_logged": False,
|
||||
"raw_cell_values_logged": False,
|
||||
"raw_audio_or_transcript_accepted": False,
|
||||
},
|
||||
"errors": [],
|
||||
"result": {},
|
||||
}
|
||||
|
||||
|
||||
def _safe_error(error: IntakeError) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"code": error.code, "source": error.source}
|
||||
if error.row is not None:
|
||||
result["row"] = error.row
|
||||
if error.field is not None:
|
||||
result["field"] = error.field
|
||||
return result
|
||||
|
||||
|
||||
def _json_pointer(location: tuple[int | str, ...]) -> str:
|
||||
if not location:
|
||||
return "/"
|
||||
return "/" + "/".join(
|
||||
str(item).replace("~", "~0").replace("/", "~1") for item in location
|
||||
)
|
||||
|
||||
|
||||
def _validation_errors(error: ValidationError) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"code": f"contract:{item['type']}",
|
||||
"source": "compiled_pack",
|
||||
"pointer": _json_pointer(tuple(item["loc"])),
|
||||
}
|
||||
for item in error.errors(
|
||||
include_url=False,
|
||||
include_context=False,
|
||||
include_input=False,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _require_exact_keys(payload: dict[str, Any]) -> None:
|
||||
if set(payload) != MANIFEST_KEYS:
|
||||
raise IntakeError("manifest_fields_invalid", source=MANIFEST_NAME)
|
||||
|
||||
|
||||
def _read_manifest(directory: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads((directory / MANIFEST_NAME).read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise IntakeError(
|
||||
f"manifest_read:{type(exc).__name__}", source=MANIFEST_NAME
|
||||
) from None
|
||||
if not isinstance(value, dict):
|
||||
raise IntakeError("manifest_object_required", source=MANIFEST_NAME)
|
||||
_require_exact_keys(value)
|
||||
if value.get("intake_schema_version") != INTAKE_SCHEMA_VERSION:
|
||||
raise IntakeError("intake_schema_version_invalid", source=MANIFEST_NAME)
|
||||
if value.get("template_only") is not False:
|
||||
raise IntakeError("template_cannot_compile", source=MANIFEST_NAME)
|
||||
return value
|
||||
|
||||
|
||||
def _read_csv(
|
||||
directory: Path, filename: str, fields: tuple[str, ...]
|
||||
) -> list[dict[str, str]]:
|
||||
try:
|
||||
with (directory / filename).open(
|
||||
"r", encoding="utf-8-sig", newline=""
|
||||
) as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if reader.fieldnames != list(fields):
|
||||
raise IntakeError("csv_headers_invalid", source=filename)
|
||||
rows = []
|
||||
for row_number, row in enumerate(reader, start=2):
|
||||
if None in row or any(value is None for value in row.values()):
|
||||
raise IntakeError(
|
||||
"csv_shape_invalid", source=filename, row=row_number
|
||||
)
|
||||
if any(len(value) > 256 for value in row.values()):
|
||||
raise IntakeError(
|
||||
"csv_cell_too_long", source=filename, row=row_number
|
||||
)
|
||||
rows.append(dict(row))
|
||||
except IntakeError:
|
||||
raise
|
||||
except (OSError, UnicodeError, csv.Error) as exc:
|
||||
raise IntakeError(f"csv_read:{type(exc).__name__}", source=filename) from None
|
||||
if not rows:
|
||||
raise IntakeError("csv_rows_required", source=filename)
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_true(value: str, *, source: str, row: int, field: str) -> bool:
|
||||
if value != "true":
|
||||
raise IntakeError("literal_true_required", source, row, field)
|
||||
return True
|
||||
|
||||
|
||||
def _parse_score(
|
||||
value: str,
|
||||
*,
|
||||
nullable: bool,
|
||||
source: str,
|
||||
row: int,
|
||||
field: str,
|
||||
) -> float | None:
|
||||
if value == "" and nullable:
|
||||
return None
|
||||
try:
|
||||
result = float(value)
|
||||
except ValueError:
|
||||
raise IntakeError("score_invalid", source, row, field) from None
|
||||
if not 0.0 <= result <= 1.0:
|
||||
raise IntakeError("score_out_of_range", source, row, field)
|
||||
return result
|
||||
|
||||
|
||||
def _parse_participants(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
|
||||
participants = []
|
||||
for row_number, row in enumerate(rows, start=2):
|
||||
key = row["participant_key"]
|
||||
split = row["split"]
|
||||
if DEIDENTIFIED_PARTICIPANT_RE.fullmatch(key) is None:
|
||||
raise IntakeError(
|
||||
"deidentified_participant_key_required",
|
||||
PARTICIPANTS_NAME,
|
||||
row_number,
|
||||
"participant_key",
|
||||
)
|
||||
expected_prefix = "calibration-" if split == "calibration" else "held-"
|
||||
if split not in {"calibration", "held_out"} or not key.startswith(
|
||||
expected_prefix
|
||||
):
|
||||
raise IntakeError(
|
||||
"participant_split_key_mismatch",
|
||||
PARTICIPANTS_NAME,
|
||||
row_number,
|
||||
"split",
|
||||
)
|
||||
participants.append(row)
|
||||
return participants
|
||||
|
||||
|
||||
def _parse_labelers(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
|
||||
labelers = []
|
||||
for row_number, row in enumerate(rows, start=2):
|
||||
key = row["labeler_key"]
|
||||
if DEIDENTIFIED_LABELER_RE.fullmatch(key) is None:
|
||||
raise IntakeError(
|
||||
"deidentified_labeler_key_required",
|
||||
LABELERS_NAME,
|
||||
row_number,
|
||||
"labeler_key",
|
||||
)
|
||||
labelers.append(
|
||||
{
|
||||
"labeler_key": key,
|
||||
"blinded_to_model_condition": _parse_true(
|
||||
row["blinded_to_model_condition"],
|
||||
source=LABELERS_NAME,
|
||||
row=row_number,
|
||||
field="blinded_to_model_condition",
|
||||
),
|
||||
"blinded_to_other_labelers": _parse_true(
|
||||
row["blinded_to_other_labelers"],
|
||||
source=LABELERS_NAME,
|
||||
row=row_number,
|
||||
field="blinded_to_other_labelers",
|
||||
),
|
||||
"labeled_independently": _parse_true(
|
||||
row["labeled_independently"],
|
||||
source=LABELERS_NAME,
|
||||
row=row_number,
|
||||
field="labeled_independently",
|
||||
),
|
||||
"attestation_sha256": row["attestation_sha256"],
|
||||
}
|
||||
)
|
||||
return labelers
|
||||
|
||||
|
||||
def _parse_observations(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
|
||||
grouped: dict[tuple[str, str, str, str], dict[str, Any]] = {}
|
||||
order: list[tuple[str, str, str, str]] = []
|
||||
for row_number, row in enumerate(rows, start=2):
|
||||
participant_key = row["participant_key"]
|
||||
session_key = row["session_key"]
|
||||
labeler_key = row["labeler_key"]
|
||||
if DEIDENTIFIED_PARTICIPANT_RE.fullmatch(participant_key) is None:
|
||||
raise IntakeError(
|
||||
"deidentified_participant_key_required",
|
||||
OBSERVATIONS_NAME,
|
||||
row_number,
|
||||
"participant_key",
|
||||
)
|
||||
if DEIDENTIFIED_SESSION_RE.fullmatch(session_key) is None:
|
||||
raise IntakeError(
|
||||
"deidentified_session_key_required",
|
||||
OBSERVATIONS_NAME,
|
||||
row_number,
|
||||
"session_key",
|
||||
)
|
||||
if DEIDENTIFIED_LABELER_RE.fullmatch(labeler_key) is None:
|
||||
raise IntakeError(
|
||||
"deidentified_labeler_key_required",
|
||||
OBSERVATIONS_NAME,
|
||||
row_number,
|
||||
"labeler_key",
|
||||
)
|
||||
key = (
|
||||
row["observation_id"],
|
||||
participant_key,
|
||||
session_key,
|
||||
row["axis"],
|
||||
)
|
||||
core = {
|
||||
"observation_id": row["observation_id"],
|
||||
"participant_key": participant_key,
|
||||
"session_key": session_key,
|
||||
"axis": row["axis"],
|
||||
"text_only_status": row["text_only_status"],
|
||||
"text_only_score": _parse_score(
|
||||
row["text_only_score"],
|
||||
nullable=True,
|
||||
source=OBSERVATIONS_NAME,
|
||||
row=row_number,
|
||||
field="text_only_score",
|
||||
),
|
||||
"voice_enabled_status": row["voice_enabled_status"],
|
||||
"voice_enabled_score": _parse_score(
|
||||
row["voice_enabled_score"],
|
||||
nullable=True,
|
||||
source=OBSERVATIONS_NAME,
|
||||
row=row_number,
|
||||
field="voice_enabled_score",
|
||||
),
|
||||
"raw_audio_included": False,
|
||||
"transcript_included": False,
|
||||
}
|
||||
if key not in grouped:
|
||||
grouped[key] = {**core, "labels": []}
|
||||
order.append(key)
|
||||
elif any(grouped[key][name] != value for name, value in core.items()):
|
||||
raise IntakeError(
|
||||
"observation_core_mismatch", OBSERVATIONS_NAME, row_number
|
||||
)
|
||||
grouped[key]["labels"].append(
|
||||
{
|
||||
"labeler_key": labeler_key,
|
||||
"score": _parse_score(
|
||||
row["label_score"],
|
||||
nullable=False,
|
||||
source=OBSERVATIONS_NAME,
|
||||
row=row_number,
|
||||
field="label_score",
|
||||
),
|
||||
"category": row["label_category"],
|
||||
}
|
||||
)
|
||||
return [grouped[key] for key in order]
|
||||
|
||||
|
||||
def _draft_pack(directory: Path) -> dict[str, Any]:
|
||||
manifest = _read_manifest(directory)
|
||||
participants = _parse_participants(
|
||||
_read_csv(directory, PARTICIPANTS_NAME, PARTICIPANT_FIELDS)
|
||||
)
|
||||
labelers = _parse_labelers(_read_csv(directory, LABELERS_NAME, LABELER_FIELDS))
|
||||
observations = _parse_observations(
|
||||
_read_csv(directory, OBSERVATIONS_NAME, OBSERVATION_FIELDS)
|
||||
)
|
||||
labeler_keys = [item["labeler_key"] for item in labelers]
|
||||
return {
|
||||
"provenance": manifest["provenance"],
|
||||
"text_only_model": manifest["text_only_model"],
|
||||
"voice_enabled_model": manifest["voice_enabled_model"],
|
||||
"power_plan": manifest["power_plan"],
|
||||
"participants": participants,
|
||||
"labeler_attestations": labelers,
|
||||
"reliability": {
|
||||
"labeler_keys": labeler_keys,
|
||||
"reported_icc": manifest["reported_icc"],
|
||||
"reported_categorical_kappa": manifest["reported_categorical_kappa"],
|
||||
"report_sha256": manifest["reliability_report_sha256"],
|
||||
},
|
||||
"observations": observations,
|
||||
}
|
||||
|
||||
|
||||
def compile_intake(directory: Path) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
||||
report = _base_report()
|
||||
try:
|
||||
payload = _draft_pack(directory)
|
||||
pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
|
||||
result = evaluate_human_voice_gain(pack)
|
||||
except IntakeError as exc:
|
||||
report["errors"] = [_safe_error(exc)]
|
||||
return report, None
|
||||
except ValidationError as exc:
|
||||
report["errors"] = _validation_errors(exc)
|
||||
return report, None
|
||||
except Exception as exc:
|
||||
report["errors"] = [
|
||||
{"code": f"evaluation:{type(exc).__name__}", "source": "compiled_pack"}
|
||||
]
|
||||
return report, None
|
||||
|
||||
report["passed"] = result.passed
|
||||
report["result"] = result.model_dump(mode="json")
|
||||
if not result.passed:
|
||||
return report, None
|
||||
return report, pack.model_dump(mode="json")
|
||||
|
||||
|
||||
def _atomic_create_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
if path.exists():
|
||||
raise IntakeError("output_exists", source="output")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
dir=path.parent,
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
delete=False,
|
||||
) as handle:
|
||||
temporary = Path(handle.name)
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
try:
|
||||
os.link(temporary, path)
|
||||
except FileExistsError:
|
||||
raise IntakeError("output_exists", source="output") from None
|
||||
except OSError as exc:
|
||||
raise IntakeError(
|
||||
f"output_link:{type(exc).__name__}", source="output"
|
||||
) from None
|
||||
try:
|
||||
temporary.unlink()
|
||||
except OSError as exc:
|
||||
path.unlink(missing_ok=True)
|
||||
raise IntakeError(
|
||||
f"output_cleanup:{type(exc).__name__}", source="output"
|
||||
) from None
|
||||
temporary = None
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def create_template(directory: Path) -> None:
|
||||
if directory.exists():
|
||||
raise IntakeError("template_directory_exists", source="template")
|
||||
directory.mkdir(parents=True)
|
||||
manifest = {
|
||||
"intake_schema_version": INTAKE_SCHEMA_VERSION,
|
||||
"template_only": True,
|
||||
"provenance": {
|
||||
"protocol_sha256": "",
|
||||
"consent_protocol_sha256": "",
|
||||
"dataset_manifest_sha256": "",
|
||||
"split_manifest_sha256": "",
|
||||
"labeling_protocol_sha256": "",
|
||||
"analysis_plan_sha256": "",
|
||||
"registered_at": "",
|
||||
"held_out_labels_opened_at": "",
|
||||
},
|
||||
"text_only_model": {
|
||||
"role": "text_only_baseline",
|
||||
"provider": "",
|
||||
"model_id": "",
|
||||
"model_version": "",
|
||||
"artifact_sha256": "",
|
||||
"configuration_sha256": "",
|
||||
},
|
||||
"voice_enabled_model": {
|
||||
"role": "voice_enabled_candidate",
|
||||
"provider": "",
|
||||
"model_id": "",
|
||||
"model_version": "",
|
||||
"artifact_sha256": "",
|
||||
"configuration_sha256": "",
|
||||
},
|
||||
"power_plan": {
|
||||
"primary_metric": "paired_one_minus_mae_gain",
|
||||
"clustering_unit": "participant",
|
||||
"required_held_out_participants": 30,
|
||||
"required_held_out_sessions": 50,
|
||||
"required_paired_axis_observations": 150,
|
||||
"alpha": 0.05,
|
||||
"target_power": 0.8,
|
||||
"minimally_detectable_gain": 0.01,
|
||||
"planned_bootstrap_samples": 10000,
|
||||
},
|
||||
"reported_icc": None,
|
||||
"reported_categorical_kappa": None,
|
||||
"reliability_report_sha256": "",
|
||||
}
|
||||
(directory / MANIFEST_NAME).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
for filename, fields in (
|
||||
(PARTICIPANTS_NAME, PARTICIPANT_FIELDS),
|
||||
(LABELERS_NAME, LABELER_FIELDS),
|
||||
(OBSERVATIONS_NAME, OBSERVATION_FIELDS),
|
||||
):
|
||||
with (directory / filename).open("w", encoding="utf-8", newline="") as handle:
|
||||
csv.writer(handle, lineterminator="\n").writerow(fields)
|
||||
(directory / README_NAME).write_text(
|
||||
"# G7 human voice-gain intake\n\n"
|
||||
"이 디렉터리는 빈 입력 틀이지 증거가 아니다. `manifest.json`의 모든 provenance를 "
|
||||
"실제 외부 연구 산출물 SHA-256으로 채우고 `template_only`를 `false`로 바꿔라. "
|
||||
"CSV에는 비식별 키와 수치만 넣고 이름, 이메일, 원음, 축어록, 자유서술을 넣지 마라. "
|
||||
"각 held-out 회기는 goal/task/bond 3축과 동일한 blind labeler panel을 가져야 한다.\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser(description=__doc__)
|
||||
mode = result.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--create-template", type=Path, metavar="DIR")
|
||||
mode.add_argument("--compile", type=Path, metavar="DIR")
|
||||
result.add_argument("--out", type=Path, help="final pack JSON; compile only")
|
||||
return result
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
args = parser().parse_args(list(argv) if argv is not None else None)
|
||||
report = _base_report()
|
||||
if args.create_template is not None:
|
||||
if args.out is not None:
|
||||
parser().error("--out is compile-only")
|
||||
try:
|
||||
create_template(args.create_template)
|
||||
except IntakeError as exc:
|
||||
report["errors"] = [_safe_error(exc)]
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 1
|
||||
report["template_created"] = True
|
||||
report["template_is_evidence"] = False
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if args.out is None:
|
||||
parser().error("--compile requires --out")
|
||||
report, pack_payload = compile_intake(args.compile)
|
||||
if pack_payload is not None:
|
||||
try:
|
||||
_atomic_create_json(args.out, pack_payload)
|
||||
except IntakeError as exc:
|
||||
report["passed"] = False
|
||||
report["errors"] = [_safe_error(exc)]
|
||||
else:
|
||||
report["pack_written"] = True
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if report["passed"] is True and report["pack_written"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue