95 lines
3.6 KiB
Python
95 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
API_ROOT = REPO_ROOT / "apps/api"
|
|
if str(API_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(API_ROOT))
|
|
|
|
from app.test_g7_voice_gain_evidence import _valid_payload # noqa: E402
|
|
from scripts.test_g7_external_proof import human_pack # noqa: E402
|
|
|
|
|
|
SCRIPT_PATH = Path(__file__).with_name("check-g7-human-voice-gain.py")
|
|
SPEC = importlib.util.spec_from_file_location("check_g7_human_voice_gain", SCRIPT_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = MODULE
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
|
|
class G7HumanVoiceGainCliTests(unittest.TestCase):
|
|
def test_production_pack_passes_with_aggregate_output_only(self) -> None:
|
|
report = MODULE.validate_payload(human_pack())
|
|
|
|
self.assertTrue(report["passed"])
|
|
self.assertEqual([], report["validation_errors"])
|
|
self.assertEqual(30, report["result"]["held_out_participants"])
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
self.assertNotIn("held-000", serialized)
|
|
self.assertNotIn("labeler-001", serialized)
|
|
self.assertFalse(report["privacy_boundary"]["participant_keys_logged"])
|
|
|
|
def test_underpowered_pack_fails_with_gate_names_but_no_identifiers(self) -> None:
|
|
report = MODULE.validate_payload(_valid_payload())
|
|
|
|
self.assertFalse(report["passed"])
|
|
self.assertIn(
|
|
"production_participant_floor",
|
|
report["result"]["failure_reasons"],
|
|
)
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
self.assertNotIn("held-out-001", serialized)
|
|
self.assertNotIn("labeler-001", serialized)
|
|
|
|
def test_invalid_rows_emit_only_json_pointer_and_error_type(self) -> None:
|
|
payload = _valid_payload()
|
|
observations = payload["observations"]
|
|
assert isinstance(observations, list)
|
|
labels = observations[0]["labels"]
|
|
assert isinstance(labels, list)
|
|
labels[0]["category"] = "person@example.test"
|
|
|
|
report = MODULE.validate_payload(payload)
|
|
|
|
self.assertFalse(report["passed"])
|
|
self.assertEqual({}, report["result"])
|
|
self.assertTrue(report["validation_errors"])
|
|
self.assertEqual(
|
|
"/observations/0/labels/0/category",
|
|
report["validation_errors"][0]["pointer"],
|
|
)
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
self.assertNotIn("person@example.test", serialized)
|
|
self.assertNotIn("held-out-001", serialized)
|
|
|
|
def test_cli_does_not_echo_input_path_and_schema_marks_kappa_required(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
secret_name = "person-at-example.test.json"
|
|
path = Path(directory) / secret_name
|
|
path.write_text(json.dumps(_valid_payload()), encoding="utf-8")
|
|
output = io.StringIO()
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = MODULE.main(["--input", str(path)])
|
|
|
|
self.assertEqual(1, exit_code)
|
|
self.assertNotIn(secret_name, output.getvalue())
|
|
|
|
schema = MODULE.G7HumanVoiceGainEvidencePack.model_json_schema()
|
|
reliability = schema["$defs"]["G7ReliabilityClaim"]
|
|
label = schema["$defs"]["G7HumanAxisLabel"]
|
|
self.assertIn("reported_categorical_kappa", reliability["required"])
|
|
self.assertIn("category", label["required"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|