G7 독립 평가 입력 검증 도구 추가

scripts/prepare-g7-human-voice-gain-intake.py

@@ -0,0 +1,578 @@
+#!/usr/bin/env python3
+"""Prepare or compile a deidentified G7 human voice-gain intake.
+
+Template mode creates empty, explicitly non-evidence input files. Compile mode
+accepts only deidentified metrics, recomputes reliability and production gates,
+and writes a final pack only when every production requirement passes. Console
+output never includes input paths, participant/labeler keys, labels, or values
+from invalid cells.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+import re
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+from pydantic import ValidationError
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+API_ROOT = REPO_ROOT / "apps/api"
+if str(API_ROOT) not in sys.path:
+    sys.path.insert(0, str(API_ROOT))
+
+from app.contracts.g7_external_evidence import (  # noqa: E402
+    G7HumanVoiceGainEvidencePack,
+)
+from app.services.g7_voice_gain_evidence import (  # noqa: E402
+    evaluate_human_voice_gain,
+)
+
+
+MANIFEST_NAME = "manifest.json"
+PARTICIPANTS_NAME = "participants.csv"
+LABELERS_NAME = "labelers.csv"
+OBSERVATIONS_NAME = "observations.csv"
+README_NAME = "README.md"
+INTAKE_SCHEMA_VERSION = "g7_human_voice_gain_intake_v1"
+
+MANIFEST_KEYS = {
+    "intake_schema_version",
+    "template_only",
+    "provenance",
+    "text_only_model",
+    "voice_enabled_model",
+    "power_plan",
+    "reported_icc",
+    "reported_categorical_kappa",
+    "reliability_report_sha256",
+}
+PARTICIPANT_FIELDS = (
+    "participant_key",
+    "split",
+    "consent_receipt_sha256",
+)
+LABELER_FIELDS = (
+    "labeler_key",
+    "blinded_to_model_condition",
+    "blinded_to_other_labelers",
+    "labeled_independently",
+    "attestation_sha256",
+)
+OBSERVATION_FIELDS = (
+    "observation_id",
+    "participant_key",
+    "session_key",
+    "axis",
+    "text_only_status",
+    "text_only_score",
+    "voice_enabled_status",
+    "voice_enabled_score",
+    "labeler_key",
+    "label_score",
+    "label_category",
+)
+
+DEIDENTIFIED_PARTICIPANT_RE = re.compile(
+    r"^(?:calibration|held)-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$"
+)
+DEIDENTIFIED_LABELER_RE = re.compile(r"^labeler-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$")
+DEIDENTIFIED_SESSION_RE = re.compile(r"^session-[A-Za-z0-9][A-Za-z0-9._:-]{2,95}$")
+
+
+@dataclass(frozen=True, slots=True)
+class IntakeError(Exception):
+    code: str
+    source: str = "intake"
+    row: int | None = None
+    field: str | None = None
+
+
+def _base_report() -> dict[str, Any]:
+    return {
+        "schema_version": "vignette.g7-human-voice-gain-intake.v1",
+        "passed": False,
+        "pack_written": False,
+        "clinical_claim_allowed": False,
+        "privacy_boundary": {
+            "input_paths_logged": False,
+            "participant_keys_logged": False,
+            "labeler_keys_logged": False,
+            "labels_logged": False,
+            "raw_cell_values_logged": False,
+            "raw_audio_or_transcript_accepted": False,
+        },
+        "errors": [],
+        "result": {},
+    }
+
+
+def _safe_error(error: IntakeError) -> dict[str, Any]:
+    result: dict[str, Any] = {"code": error.code, "source": error.source}
+    if error.row is not None:
+        result["row"] = error.row
+    if error.field is not None:
+        result["field"] = error.field
+    return result
+
+
+def _json_pointer(location: tuple[int | str, ...]) -> str:
+    if not location:
+        return "/"
+    return "/" + "/".join(
+        str(item).replace("~", "~0").replace("/", "~1") for item in location
+    )
+
+
+def _validation_errors(error: ValidationError) -> list[dict[str, Any]]:
+    return [
+        {
+            "code": f"contract:{item['type']}",
+            "source": "compiled_pack",
+            "pointer": _json_pointer(tuple(item["loc"])),
+        }
+        for item in error.errors(
+            include_url=False,
+            include_context=False,
+            include_input=False,
+        )
+    ]
+
+
+def _require_exact_keys(payload: dict[str, Any]) -> None:
+    if set(payload) != MANIFEST_KEYS:
+        raise IntakeError("manifest_fields_invalid", source=MANIFEST_NAME)
+
+
+def _read_manifest(directory: Path) -> dict[str, Any]:
+    try:
+        value = json.loads((directory / MANIFEST_NAME).read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise IntakeError(
+            f"manifest_read:{type(exc).__name__}", source=MANIFEST_NAME
+        ) from None
+    if not isinstance(value, dict):
+        raise IntakeError("manifest_object_required", source=MANIFEST_NAME)
+    _require_exact_keys(value)
+    if value.get("intake_schema_version") != INTAKE_SCHEMA_VERSION:
+        raise IntakeError("intake_schema_version_invalid", source=MANIFEST_NAME)
+    if value.get("template_only") is not False:
+        raise IntakeError("template_cannot_compile", source=MANIFEST_NAME)
+    return value
+
+
+def _read_csv(
+    directory: Path, filename: str, fields: tuple[str, ...]
+) -> list[dict[str, str]]:
+    try:
+        with (directory / filename).open(
+            "r", encoding="utf-8-sig", newline=""
+        ) as handle:
+            reader = csv.DictReader(handle)
+            if reader.fieldnames != list(fields):
+                raise IntakeError("csv_headers_invalid", source=filename)
+            rows = []
+            for row_number, row in enumerate(reader, start=2):
+                if None in row or any(value is None for value in row.values()):
+                    raise IntakeError(
+                        "csv_shape_invalid", source=filename, row=row_number
+                    )
+                if any(len(value) > 256 for value in row.values()):
+                    raise IntakeError(
+                        "csv_cell_too_long", source=filename, row=row_number
+                    )
+                rows.append(dict(row))
+    except IntakeError:
+        raise
+    except (OSError, UnicodeError, csv.Error) as exc:
+        raise IntakeError(f"csv_read:{type(exc).__name__}", source=filename) from None
+    if not rows:
+        raise IntakeError("csv_rows_required", source=filename)
+    return rows
+
+
+def _parse_true(value: str, *, source: str, row: int, field: str) -> bool:
+    if value != "true":
+        raise IntakeError("literal_true_required", source, row, field)
+    return True
+
+
+def _parse_score(
+    value: str,
+    *,
+    nullable: bool,
+    source: str,
+    row: int,
+    field: str,
+) -> float | None:
+    if value == "" and nullable:
+        return None
+    try:
+        result = float(value)
+    except ValueError:
+        raise IntakeError("score_invalid", source, row, field) from None
+    if not 0.0 <= result <= 1.0:
+        raise IntakeError("score_out_of_range", source, row, field)
+    return result
+
+
+def _parse_participants(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
+    participants = []
+    for row_number, row in enumerate(rows, start=2):
+        key = row["participant_key"]
+        split = row["split"]
+        if DEIDENTIFIED_PARTICIPANT_RE.fullmatch(key) is None:
+            raise IntakeError(
+                "deidentified_participant_key_required",
+                PARTICIPANTS_NAME,
+                row_number,
+                "participant_key",
+            )
+        expected_prefix = "calibration-" if split == "calibration" else "held-"
+        if split not in {"calibration", "held_out"} or not key.startswith(
+            expected_prefix
+        ):
+            raise IntakeError(
+                "participant_split_key_mismatch",
+                PARTICIPANTS_NAME,
+                row_number,
+                "split",
+            )
+        participants.append(row)
+    return participants
+
+
+def _parse_labelers(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
+    labelers = []
+    for row_number, row in enumerate(rows, start=2):
+        key = row["labeler_key"]
+        if DEIDENTIFIED_LABELER_RE.fullmatch(key) is None:
+            raise IntakeError(
+                "deidentified_labeler_key_required",
+                LABELERS_NAME,
+                row_number,
+                "labeler_key",
+            )
+        labelers.append(
+            {
+                "labeler_key": key,
+                "blinded_to_model_condition": _parse_true(
+                    row["blinded_to_model_condition"],
+                    source=LABELERS_NAME,
+                    row=row_number,
+                    field="blinded_to_model_condition",
+                ),
+                "blinded_to_other_labelers": _parse_true(
+                    row["blinded_to_other_labelers"],
+                    source=LABELERS_NAME,
+                    row=row_number,
+                    field="blinded_to_other_labelers",
+                ),
+                "labeled_independently": _parse_true(
+                    row["labeled_independently"],
+                    source=LABELERS_NAME,
+                    row=row_number,
+                    field="labeled_independently",
+                ),
+                "attestation_sha256": row["attestation_sha256"],
+            }
+        )
+    return labelers
+
+
+def _parse_observations(rows: list[dict[str, str]]) -> list[dict[str, Any]]:
+    grouped: dict[tuple[str, str, str, str], dict[str, Any]] = {}
+    order: list[tuple[str, str, str, str]] = []
+    for row_number, row in enumerate(rows, start=2):
+        participant_key = row["participant_key"]
+        session_key = row["session_key"]
+        labeler_key = row["labeler_key"]
+        if DEIDENTIFIED_PARTICIPANT_RE.fullmatch(participant_key) is None:
+            raise IntakeError(
+                "deidentified_participant_key_required",
+                OBSERVATIONS_NAME,
+                row_number,
+                "participant_key",
+            )
+        if DEIDENTIFIED_SESSION_RE.fullmatch(session_key) is None:
+            raise IntakeError(
+                "deidentified_session_key_required",
+                OBSERVATIONS_NAME,
+                row_number,
+                "session_key",
+            )
+        if DEIDENTIFIED_LABELER_RE.fullmatch(labeler_key) is None:
+            raise IntakeError(
+                "deidentified_labeler_key_required",
+                OBSERVATIONS_NAME,
+                row_number,
+                "labeler_key",
+            )
+        key = (
+            row["observation_id"],
+            participant_key,
+            session_key,
+            row["axis"],
+        )
+        core = {
+            "observation_id": row["observation_id"],
+            "participant_key": participant_key,
+            "session_key": session_key,
+            "axis": row["axis"],
+            "text_only_status": row["text_only_status"],
+            "text_only_score": _parse_score(
+                row["text_only_score"],
+                nullable=True,
+                source=OBSERVATIONS_NAME,
+                row=row_number,
+                field="text_only_score",
+            ),
+            "voice_enabled_status": row["voice_enabled_status"],
+            "voice_enabled_score": _parse_score(
+                row["voice_enabled_score"],
+                nullable=True,
+                source=OBSERVATIONS_NAME,
+                row=row_number,
+                field="voice_enabled_score",
+            ),
+            "raw_audio_included": False,
+            "transcript_included": False,
+        }
+        if key not in grouped:
+            grouped[key] = {**core, "labels": []}
+            order.append(key)
+        elif any(grouped[key][name] != value for name, value in core.items()):
+            raise IntakeError(
+                "observation_core_mismatch", OBSERVATIONS_NAME, row_number
+            )
+        grouped[key]["labels"].append(
+            {
+                "labeler_key": labeler_key,
+                "score": _parse_score(
+                    row["label_score"],
+                    nullable=False,
+                    source=OBSERVATIONS_NAME,
+                    row=row_number,
+                    field="label_score",
+                ),
+                "category": row["label_category"],
+            }
+        )
+    return [grouped[key] for key in order]
+
+
+def _draft_pack(directory: Path) -> dict[str, Any]:
+    manifest = _read_manifest(directory)
+    participants = _parse_participants(
+        _read_csv(directory, PARTICIPANTS_NAME, PARTICIPANT_FIELDS)
+    )
+    labelers = _parse_labelers(_read_csv(directory, LABELERS_NAME, LABELER_FIELDS))
+    observations = _parse_observations(
+        _read_csv(directory, OBSERVATIONS_NAME, OBSERVATION_FIELDS)
+    )
+    labeler_keys = [item["labeler_key"] for item in labelers]
+    return {
+        "provenance": manifest["provenance"],
+        "text_only_model": manifest["text_only_model"],
+        "voice_enabled_model": manifest["voice_enabled_model"],
+        "power_plan": manifest["power_plan"],
+        "participants": participants,
+        "labeler_attestations": labelers,
+        "reliability": {
+            "labeler_keys": labeler_keys,
+            "reported_icc": manifest["reported_icc"],
+            "reported_categorical_kappa": manifest["reported_categorical_kappa"],
+            "report_sha256": manifest["reliability_report_sha256"],
+        },
+        "observations": observations,
+    }
+
+
+def compile_intake(directory: Path) -> tuple[dict[str, Any], dict[str, Any] | None]:
+    report = _base_report()
+    try:
+        payload = _draft_pack(directory)
+        pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
+        result = evaluate_human_voice_gain(pack)
+    except IntakeError as exc:
+        report["errors"] = [_safe_error(exc)]
+        return report, None
+    except ValidationError as exc:
+        report["errors"] = _validation_errors(exc)
+        return report, None
+    except Exception as exc:
+        report["errors"] = [
+            {"code": f"evaluation:{type(exc).__name__}", "source": "compiled_pack"}
+        ]
+        return report, None
+
+    report["passed"] = result.passed
+    report["result"] = result.model_dump(mode="json")
+    if not result.passed:
+        return report, None
+    return report, pack.model_dump(mode="json")
+
+
+def _atomic_create_json(path: Path, payload: dict[str, Any]) -> None:
+    if path.exists():
+        raise IntakeError("output_exists", source="output")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            "w",
+            encoding="utf-8",
+            newline="\n",
+            dir=path.parent,
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as handle:
+            temporary = Path(handle.name)
+            json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        try:
+            os.link(temporary, path)
+        except FileExistsError:
+            raise IntakeError("output_exists", source="output") from None
+        except OSError as exc:
+            raise IntakeError(
+                f"output_link:{type(exc).__name__}", source="output"
+            ) from None
+        try:
+            temporary.unlink()
+        except OSError as exc:
+            path.unlink(missing_ok=True)
+            raise IntakeError(
+                f"output_cleanup:{type(exc).__name__}", source="output"
+            ) from None
+        temporary = None
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def create_template(directory: Path) -> None:
+    if directory.exists():
+        raise IntakeError("template_directory_exists", source="template")
+    directory.mkdir(parents=True)
+    manifest = {
+        "intake_schema_version": INTAKE_SCHEMA_VERSION,
+        "template_only": True,
+        "provenance": {
+            "protocol_sha256": "",
+            "consent_protocol_sha256": "",
+            "dataset_manifest_sha256": "",
+            "split_manifest_sha256": "",
+            "labeling_protocol_sha256": "",
+            "analysis_plan_sha256": "",
+            "registered_at": "",
+            "held_out_labels_opened_at": "",
+        },
+        "text_only_model": {
+            "role": "text_only_baseline",
+            "provider": "",
+            "model_id": "",
+            "model_version": "",
+            "artifact_sha256": "",
+            "configuration_sha256": "",
+        },
+        "voice_enabled_model": {
+            "role": "voice_enabled_candidate",
+            "provider": "",
+            "model_id": "",
+            "model_version": "",
+            "artifact_sha256": "",
+            "configuration_sha256": "",
+        },
+        "power_plan": {
+            "primary_metric": "paired_one_minus_mae_gain",
+            "clustering_unit": "participant",
+            "required_held_out_participants": 30,
+            "required_held_out_sessions": 50,
+            "required_paired_axis_observations": 150,
+            "alpha": 0.05,
+            "target_power": 0.8,
+            "minimally_detectable_gain": 0.01,
+            "planned_bootstrap_samples": 10000,
+        },
+        "reported_icc": None,
+        "reported_categorical_kappa": None,
+        "reliability_report_sha256": "",
+    }
+    (directory / MANIFEST_NAME).write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+    for filename, fields in (
+        (PARTICIPANTS_NAME, PARTICIPANT_FIELDS),
+        (LABELERS_NAME, LABELER_FIELDS),
+        (OBSERVATIONS_NAME, OBSERVATION_FIELDS),
+    ):
+        with (directory / filename).open("w", encoding="utf-8", newline="") as handle:
+            csv.writer(handle, lineterminator="\n").writerow(fields)
+    (directory / README_NAME).write_text(
+        "# G7 human voice-gain intake\n\n"
+        "이 디렉터리는 빈 입력 틀이지 증거가 아니다. `manifest.json`의 모든 provenance를 "
+        "실제 외부 연구 산출물 SHA-256으로 채우고 `template_only`를 `false`로 바꿔라. "
+        "CSV에는 비식별 키와 수치만 넣고 이름, 이메일, 원음, 축어록, 자유서술을 넣지 마라. "
+        "각 held-out 회기는 goal/task/bond 3축과 동일한 blind labeler panel을 가져야 한다.\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser(description=__doc__)
+    mode = result.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--create-template", type=Path, metavar="DIR")
+    mode.add_argument("--compile", type=Path, metavar="DIR")
+    result.add_argument("--out", type=Path, help="final pack JSON; compile only")
+    return result
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    args = parser().parse_args(list(argv) if argv is not None else None)
+    report = _base_report()
+    if args.create_template is not None:
+        if args.out is not None:
+            parser().error("--out is compile-only")
+        try:
+            create_template(args.create_template)
+        except IntakeError as exc:
+            report["errors"] = [_safe_error(exc)]
+            print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
+            return 1
+        report["template_created"] = True
+        report["template_is_evidence"] = False
+        print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
+        return 0
+
+    if args.out is None:
+        parser().error("--compile requires --out")
+    report, pack_payload = compile_intake(args.compile)
+    if pack_payload is not None:
+        try:
+            _atomic_create_json(args.out, pack_payload)
+        except IntakeError as exc:
+            report["passed"] = False
+            report["errors"] = [_safe_error(exc)]
+        else:
+            report["pack_written"] = True
+    print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
+    return 0 if report["passed"] is True and report["pack_written"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_prepare_g7_human_voice_gain_intake.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import contextlib
+import csv
+import importlib.util
+import io
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+API_ROOT = REPO_ROOT / "apps/api"
+if str(API_ROOT) not in sys.path:
+    sys.path.insert(0, str(API_ROOT))
+
+from app.contracts.g7_external_evidence import (  # noqa: E402
+    G7HumanVoiceGainEvidencePack,
+)
+from app.test_g7_voice_gain_evidence import _valid_payload  # noqa: E402
+from scripts.test_g7_external_proof import human_pack  # noqa: E402
+
+
+SCRIPT_PATH = Path(__file__).with_name("prepare-g7-human-voice-gain-intake.py")
+SPEC = importlib.util.spec_from_file_location(
+    "prepare_g7_human_voice_gain_intake", SCRIPT_PATH
+)
+assert SPEC is not None and SPEC.loader is not None
+MODULE = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = MODULE
+SPEC.loader.exec_module(MODULE)
+
+
+def _write_csv(
+    path: Path, fields: tuple[str, ...], rows: list[dict[str, object]]
+) -> None:
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def _write_intake(directory: Path, payload: dict[str, object]) -> None:
+    reliability = payload["reliability"]
+    assert isinstance(reliability, dict)
+    manifest = {
+        "intake_schema_version": MODULE.INTAKE_SCHEMA_VERSION,
+        "template_only": False,
+        "provenance": payload["provenance"],
+        "text_only_model": payload["text_only_model"],
+        "voice_enabled_model": payload["voice_enabled_model"],
+        "power_plan": payload["power_plan"],
+        "reported_icc": reliability["reported_icc"],
+        "reported_categorical_kappa": reliability["reported_categorical_kappa"],
+        "reliability_report_sha256": reliability["report_sha256"],
+    }
+    (directory / MODULE.MANIFEST_NAME).write_text(
+        json.dumps(manifest), encoding="utf-8"
+    )
+
+    participants = payload["participants"]
+    assert isinstance(participants, list)
+    _write_csv(
+        directory / MODULE.PARTICIPANTS_NAME,
+        MODULE.PARTICIPANT_FIELDS,
+        participants,
+    )
+
+    attestations = payload["labeler_attestations"]
+    assert isinstance(attestations, list)
+    labeler_rows = []
+    for item in attestations:
+        assert isinstance(item, dict)
+        labeler_rows.append(
+            {
+                "labeler_key": item["labeler_key"],
+                "blinded_to_model_condition": "true",
+                "blinded_to_other_labelers": "true",
+                "labeled_independently": "true",
+                "attestation_sha256": item["attestation_sha256"],
+            }
+        )
+    _write_csv(
+        directory / MODULE.LABELERS_NAME,
+        MODULE.LABELER_FIELDS,
+        labeler_rows,
+    )
+
+    observations = payload["observations"]
+    assert isinstance(observations, list)
+    observation_rows = []
+    for observation in observations:
+        assert isinstance(observation, dict)
+        labels = observation["labels"]
+        assert isinstance(labels, list)
+        for label in labels:
+            assert isinstance(label, dict)
+            observation_rows.append(
+                {
+                    "observation_id": observation["observation_id"],
+                    "participant_key": observation["participant_key"],
+                    "session_key": observation["session_key"],
+                    "axis": observation["axis"],
+                    "text_only_status": observation["text_only_status"],
+                    "text_only_score": (
+                        ""
+                        if observation.get("text_only_score") is None
+                        else observation["text_only_score"]
+                    ),
+                    "voice_enabled_status": observation["voice_enabled_status"],
+                    "voice_enabled_score": (
+                        ""
+                        if observation.get("voice_enabled_score") is None
+                        else observation["voice_enabled_score"]
+                    ),
+                    "labeler_key": label["labeler_key"],
+                    "label_score": label["score"],
+                    "label_category": label["category"],
+                }
+            )
+    _write_csv(
+        directory / MODULE.OBSERVATIONS_NAME,
+        MODULE.OBSERVATION_FIELDS,
+        observation_rows,
+    )
+
+
+class G7HumanVoiceGainIntakeTests(unittest.TestCase):
+    def test_empty_template_is_explicitly_not_evidence(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            target = Path(root) / "intake"
+            output = io.StringIO()
+            with contextlib.redirect_stdout(output):
+                exit_code = MODULE.main(["--create-template", str(target)])
+
+            self.assertEqual(0, exit_code)
+            self.assertEqual(
+                {
+                    MODULE.MANIFEST_NAME,
+                    MODULE.PARTICIPANTS_NAME,
+                    MODULE.LABELERS_NAME,
+                    MODULE.OBSERVATIONS_NAME,
+                    MODULE.README_NAME,
+                },
+                {item.name for item in target.iterdir()},
+            )
+            manifest = json.loads(
+                (target / MODULE.MANIFEST_NAME).read_text(encoding="utf-8")
+            )
+            self.assertIs(True, manifest["template_only"])
+            self.assertFalse(json.loads(output.getvalue())["template_is_evidence"])
+
+            report, pack = MODULE.compile_intake(target)
+            self.assertIsNone(pack)
+            self.assertEqual("template_cannot_compile", report["errors"][0]["code"])
+
+    def test_production_intake_compiles_and_checks_reported_reliability(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            directory = Path(root) / "private-human-intake"
+            directory.mkdir()
+            payload = human_pack()
+            _write_intake(directory, payload)
+            output_path = Path(root) / "compiled.json"
+            stdout = io.StringIO()
+            with contextlib.redirect_stdout(stdout):
+                exit_code = MODULE.main(
+                    ["--compile", str(directory), "--out", str(output_path)]
+                )
+
+            self.assertEqual(0, exit_code)
+            report = json.loads(stdout.getvalue())
+            self.assertTrue(report["passed"])
+            self.assertTrue(report["pack_written"])
+            self.assertEqual(30, report["result"]["held_out_participants"])
+            serialized_report = json.dumps(report, ensure_ascii=False)
+            self.assertNotIn("private-human-intake", serialized_report)
+            self.assertNotIn("held-000", serialized_report)
+            self.assertNotIn("labeler-001", serialized_report)
+
+            compiled = json.loads(output_path.read_text(encoding="utf-8"))
+            pack = G7HumanVoiceGainEvidencePack.model_validate(compiled)
+            self.assertEqual(1.0, pack.reliability.reported_icc)
+            self.assertEqual(1.0, pack.reliability.reported_categorical_kappa)
+            self.assertTrue(
+                all(not item.raw_audio_included for item in pack.observations)
+            )
+            self.assertTrue(
+                all(not item.transcript_included for item in pack.observations)
+            )
+
+    def test_underpowered_intake_never_writes_a_pack(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            directory = Path(root) / "intake"
+            directory.mkdir()
+            _write_intake(directory, _valid_payload())
+            output_path = Path(root) / "compiled.json"
+
+            report, pack = MODULE.compile_intake(directory)
+
+            self.assertIsNone(pack)
+            self.assertFalse(report["passed"])
+            self.assertIn(
+                "production_participant_floor",
+                report["result"]["failure_reasons"],
+            )
+            self.assertFalse(output_path.exists())
+
+    def test_reported_reliability_must_match_rows(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            directory = Path(root) / "intake"
+            directory.mkdir()
+            payload = human_pack()
+            reliability = payload["reliability"]
+            assert isinstance(reliability, dict)
+            reliability["reported_icc"] = 0.8
+            _write_intake(directory, payload)
+
+            report, pack = MODULE.compile_intake(directory)
+
+            self.assertIsNone(pack)
+            self.assertFalse(report["passed"])
+            self.assertIn(
+                "reported_icc_matches_rows",
+                report["result"]["failure_reasons"],
+            )
+
+    def test_unexpected_raw_material_column_is_rejected_without_echo(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            directory = Path(root) / "person-at-example.test"
+            directory.mkdir()
+            _write_intake(directory, human_pack())
+            path = directory / MODULE.OBSERVATIONS_NAME
+            rows = list(csv.reader(path.read_text(encoding="utf-8").splitlines()))
+            rows[0].append("raw_transcript")
+            rows[1].append("private words")
+            with path.open("w", encoding="utf-8", newline="") as handle:
+                csv.writer(handle, lineterminator="\n").writerows(rows)
+
+            report, pack = MODULE.compile_intake(directory)
+
+            self.assertIsNone(pack)
+            self.assertEqual("csv_headers_invalid", report["errors"][0]["code"])
+            serialized = json.dumps(report, ensure_ascii=False)
+            self.assertNotIn("private words", serialized)
+            self.assertNotIn("person-at-example.test", serialized)
+
+    def test_existing_output_is_not_overwritten(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            directory = Path(root) / "intake"
+            directory.mkdir()
+            _write_intake(directory, human_pack())
+            output_path = Path(root) / "compiled.json"
+            output_path.write_text("preserve", encoding="utf-8")
+            stdout = io.StringIO()
+            with contextlib.redirect_stdout(stdout):
+                exit_code = MODULE.main(
+                    ["--compile", str(directory), "--out", str(output_path)]
+                )
+
+            self.assertEqual(1, exit_code)
+            self.assertEqual("preserve", output_path.read_text(encoding="utf-8"))
+            report = json.loads(stdout.getvalue())
+            self.assertEqual("output_exists", report["errors"][0]["code"])
+            self.assertFalse(report["pack_written"])
+
+
+if __name__ == "__main__":
+    unittest.main()