주기 실회기 검증과 G7 종료계약 보강
This commit is contained in:
parent
83590e9ef7
commit
7b4955c3fc
23 changed files with 2916 additions and 117 deletions
129
scripts/check-g7-human-voice-gain.py
Normal file
129
scripts/check-g7-human-voice-gain.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Validate and evaluate one independent human-held-out G7 voice-gain pack.
|
||||
|
||||
The command emits only aggregate metrics and PII-safe JSON pointers. It never
|
||||
echoes input paths, participant/labeler keys, labels, or raw validation values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps/api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.contracts.g7_external_evidence import ( # noqa: E402
|
||||
G7HumanVoiceGainEvidencePack,
|
||||
)
|
||||
from app.services.g7_voice_gain_evidence import ( # noqa: E402
|
||||
evaluate_human_voice_gain,
|
||||
)
|
||||
|
||||
|
||||
def _json_pointer(location: tuple[int | str, ...]) -> str:
|
||||
if not location:
|
||||
return "/"
|
||||
parts = []
|
||||
for item in location:
|
||||
value = str(item).replace("~", "~0").replace("/", "~1")
|
||||
parts.append(value)
|
||||
return "/" + "/".join(parts)
|
||||
|
||||
|
||||
def _base_report() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "vignette.g7-human-voice-gain-check.v1",
|
||||
"passed": False,
|
||||
"clinical_claim_allowed": False,
|
||||
"privacy_boundary": {
|
||||
"input_path_logged": False,
|
||||
"participant_keys_logged": False,
|
||||
"labeler_keys_logged": False,
|
||||
"labels_logged": False,
|
||||
"raw_validation_values_logged": False,
|
||||
},
|
||||
"validation_errors": [],
|
||||
"result": {},
|
||||
}
|
||||
|
||||
|
||||
def validate_payload(payload: object) -> dict[str, Any]:
|
||||
report = _base_report()
|
||||
try:
|
||||
pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
|
||||
except ValidationError as exc:
|
||||
report["validation_errors"] = [
|
||||
{
|
||||
"pointer": _json_pointer(tuple(item["loc"])),
|
||||
"type": item["type"],
|
||||
}
|
||||
for item in exc.errors(
|
||||
include_url=False,
|
||||
include_context=False,
|
||||
include_input=False,
|
||||
)
|
||||
]
|
||||
return report
|
||||
|
||||
try:
|
||||
result = evaluate_human_voice_gain(pack)
|
||||
except Exception as exc:
|
||||
report["validation_errors"] = [
|
||||
{"pointer": "/", "type": f"evaluation:{type(exc).__name__}"}
|
||||
]
|
||||
return report
|
||||
|
||||
report["passed"] = result.passed
|
||||
report["result"] = result.model_dump(mode="json")
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser(description=__doc__)
|
||||
mode = result.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--input", type=Path, help="deidentified human pack JSON")
|
||||
mode.add_argument(
|
||||
"--print-schema",
|
||||
action="store_true",
|
||||
help="print the authoritative JSON Schema and exit",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
args = parser().parse_args(list(argv) if argv is not None else None)
|
||||
if args.print_schema:
|
||||
print(
|
||||
json.dumps(
|
||||
G7HumanVoiceGainEvidencePack.model_json_schema(),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
try:
|
||||
payload = json.loads(args.input.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
report = _base_report()
|
||||
report["validation_errors"] = [
|
||||
{"pointer": "/", "type": f"input:{type(exc).__name__}"}
|
||||
]
|
||||
else:
|
||||
report = validate_payload(payload)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if report["passed"] is True else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -42,6 +42,16 @@ from typing import Any, Callable, Iterable, Sequence
|
|||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = REPO_ROOT / "scripts"
|
||||
API_ROOT = REPO_ROOT / "apps/api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.contracts.g7_external_evidence import ( # noqa: E402
|
||||
G7HumanVoiceGainEvidencePack,
|
||||
)
|
||||
from app.services.g7_voice_gain_evidence import ( # noqa: E402
|
||||
evaluate_human_voice_gain,
|
||||
)
|
||||
|
||||
SOAK_SCRIPT = SCRIPTS / "soak-public-voice-websocket.py"
|
||||
RUNTIME_SCRIPT = SCRIPTS / "capture-g7-runtime-evidence.py"
|
||||
|
|
@ -63,6 +73,8 @@ MIN_PRODUCTION_SECONDS = (
|
|||
MIN_REQUIRED_OVERLAP_SECONDS + CAPTURE_START_SKEW_MARGIN_SECONDS
|
||||
)
|
||||
RUNTIME_SAMPLE_MARGIN = 1
|
||||
MAX_CHILD_INTERVAL_SECONDS = 60.0
|
||||
DEFAULT_SAMPLE_INTERVAL_SECONDS = 60.0
|
||||
|
||||
|
||||
class WindowError(RuntimeError):
|
||||
|
|
@ -172,6 +184,23 @@ def sample_plan(duration_seconds: float, interval_seconds: float) -> int:
|
|||
return math.ceil(duration_seconds / interval_seconds) + RUNTIME_SAMPLE_MARGIN
|
||||
|
||||
|
||||
def validate_human_voice_gain_pack(path: Path) -> None:
|
||||
"""52분 캡처를 열기 전에 사람 pack의 production gate를 완전히 계산한다."""
|
||||
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
pack = G7HumanVoiceGainEvidencePack.model_validate(payload)
|
||||
result = evaluate_human_voice_gain(pack)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
raise WindowError("human_voice_gain_pack_unreadable") from None
|
||||
except Exception:
|
||||
# validation 원문에는 입력값이 포함될 수 있으므로 오류 code만 낸다.
|
||||
raise WindowError("human_voice_gain_pack_invalid") from None
|
||||
if not result.passed:
|
||||
reasons = ",".join(result.failure_reasons)
|
||||
raise WindowError(f"human_voice_gain_pack_failed:{reasons}")
|
||||
|
||||
|
||||
def build_soak_leg(args: argparse.Namespace, output: Path) -> Leg:
|
||||
argv = [
|
||||
sys.executable,
|
||||
|
|
@ -533,8 +562,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
default=MIN_PRODUCTION_SECONDS,
|
||||
help="실제 캡처는 최소 3120초; 유효 artifact 교집합 기준은 3000초",
|
||||
)
|
||||
parser.add_argument("--runtime-interval-seconds", type=float, default=100.0)
|
||||
parser.add_argument("--topology-interval-seconds", type=float, default=100.0)
|
||||
parser.add_argument(
|
||||
"--runtime-interval-seconds",
|
||||
type=float,
|
||||
default=DEFAULT_SAMPLE_INTERVAL_SECONDS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--topology-interval-seconds",
|
||||
type=float,
|
||||
default=DEFAULT_SAMPLE_INTERVAL_SECONDS,
|
||||
)
|
||||
parser.add_argument("--human-voice-gain", type=Path)
|
||||
parser.add_argument("--out-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
|
|
@ -550,6 +587,12 @@ def validate(args: argparse.Namespace) -> None:
|
|||
raise WindowError("production_window_too_short")
|
||||
if not args.rehearse and args.human_voice_gain is None:
|
||||
raise WindowError("human_voice_gain_pack_required")
|
||||
for value, code in (
|
||||
(args.runtime_interval_seconds, "runtime_interval_out_of_bounds"),
|
||||
(args.topology_interval_seconds, "topology_interval_out_of_bounds"),
|
||||
):
|
||||
if not math.isfinite(value) or not 0.05 <= value <= MAX_CHILD_INTERVAL_SECONDS:
|
||||
raise WindowError(code)
|
||||
if args.topology_mode not in ("linux-compose", "windows-host"):
|
||||
raise WindowError("topology_mode_invalid")
|
||||
if args.topology_mode == "linux-compose":
|
||||
|
|
@ -589,6 +632,9 @@ def main(argv: Iterable[str] | None = None) -> int:
|
|||
args = build_parser().parse_args(list(argv) if argv is not None else None)
|
||||
try:
|
||||
validate(args)
|
||||
if not args.rehearse:
|
||||
assert args.human_voice_gain is not None
|
||||
validate_human_voice_gain_pack(args.human_voice_gain)
|
||||
args.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
legs = plan_legs(args, args.out_dir)
|
||||
except WindowError as exc:
|
||||
|
|
|
|||
|
|
@ -88,6 +88,19 @@ POSTDEPLOY_NAS_E2E_SPECS = (
|
|||
"e2e/multimodal-alliance.spec.ts",
|
||||
"e2e/continuous-improvement-admin.spec.ts",
|
||||
)
|
||||
# Only these specs use the configured NAS API/DB instead of replacing the
|
||||
# product API with Playwright route fixtures. Keep the split explicit in the
|
||||
# receipt so the 112-test browser total cannot be mistaken for 112 live
|
||||
# session/database proofs.
|
||||
POSTDEPLOY_NAS_REAL_API_E2E_SPECS = (
|
||||
"e2e/session-layout.spec.ts",
|
||||
"e2e/session-persistence.spec.ts",
|
||||
)
|
||||
POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS = tuple(
|
||||
spec
|
||||
for spec in POSTDEPLOY_NAS_E2E_SPECS
|
||||
if spec not in POSTDEPLOY_NAS_REAL_API_E2E_SPECS
|
||||
)
|
||||
POSTDEPLOY_SOURCE_ONLY_E2E_SPECS = ("e2e/insecure-context-uuid.spec.ts",)
|
||||
POSTDEPLOY_DISPOSABLE_DB_E2E_SPECS = (
|
||||
"e2e/returned-practice-db-closed-loop.spec.ts",
|
||||
|
|
@ -1572,8 +1585,13 @@ class ReleaseAgent:
|
|||
"base_url": self.config.target.base_url,
|
||||
"specs": list(POSTDEPLOY_NAS_E2E_SPECS),
|
||||
"projects": list(RELEASE_BROWSER_PROJECTS),
|
||||
"runtime_scope": {
|
||||
"real_api_db_specs": list(POSTDEPLOY_NAS_REAL_API_E2E_SPECS),
|
||||
"route_fixture_specs": list(POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS),
|
||||
},
|
||||
"source_only_candidate_specs": list(POSTDEPLOY_SOURCE_ONLY_E2E_SPECS),
|
||||
"separate_disposable_db_specs": list(POSTDEPLOY_DISPOSABLE_DB_E2E_SPECS),
|
||||
"separate_disposable_db_status": "not_run_by_release_agent",
|
||||
"uuid_runtime_route_specs": [
|
||||
"e2e/session-persistence.spec.ts",
|
||||
"e2e/self-directed-learning-loop.spec.ts",
|
||||
|
|
|
|||
1405
scripts/run-periodic-learner-e2e.py
Normal file
1405
scripts/run-periodic-learner-e2e.py
Normal file
File diff suppressed because it is too large
Load diff
95
scripts/test_check_g7_human_voice_gain.py
Normal file
95
scripts/test_check_g7_human_voice_gain.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps/api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.test_g7_voice_gain_evidence import _valid_payload # noqa: E402
|
||||
from scripts.test_g7_external_proof import human_pack # noqa: E402
|
||||
|
||||
|
||||
SCRIPT_PATH = Path(__file__).with_name("check-g7-human-voice-gain.py")
|
||||
SPEC = importlib.util.spec_from_file_location("check_g7_human_voice_gain", SCRIPT_PATH)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = MODULE
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class G7HumanVoiceGainCliTests(unittest.TestCase):
|
||||
def test_production_pack_passes_with_aggregate_output_only(self) -> None:
|
||||
report = MODULE.validate_payload(human_pack())
|
||||
|
||||
self.assertTrue(report["passed"])
|
||||
self.assertEqual([], report["validation_errors"])
|
||||
self.assertEqual(30, report["result"]["held_out_participants"])
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
self.assertNotIn("held-000", serialized)
|
||||
self.assertNotIn("labeler-001", serialized)
|
||||
self.assertFalse(report["privacy_boundary"]["participant_keys_logged"])
|
||||
|
||||
def test_underpowered_pack_fails_with_gate_names_but_no_identifiers(self) -> None:
|
||||
report = MODULE.validate_payload(_valid_payload())
|
||||
|
||||
self.assertFalse(report["passed"])
|
||||
self.assertIn(
|
||||
"production_participant_floor",
|
||||
report["result"]["failure_reasons"],
|
||||
)
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
self.assertNotIn("held-out-001", serialized)
|
||||
self.assertNotIn("labeler-001", serialized)
|
||||
|
||||
def test_invalid_rows_emit_only_json_pointer_and_error_type(self) -> None:
|
||||
payload = _valid_payload()
|
||||
observations = payload["observations"]
|
||||
assert isinstance(observations, list)
|
||||
labels = observations[0]["labels"]
|
||||
assert isinstance(labels, list)
|
||||
labels[0]["category"] = "person@example.test"
|
||||
|
||||
report = MODULE.validate_payload(payload)
|
||||
|
||||
self.assertFalse(report["passed"])
|
||||
self.assertEqual({}, report["result"])
|
||||
self.assertTrue(report["validation_errors"])
|
||||
self.assertEqual(
|
||||
"/observations/0/labels/0/category",
|
||||
report["validation_errors"][0]["pointer"],
|
||||
)
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
self.assertNotIn("person@example.test", serialized)
|
||||
self.assertNotIn("held-out-001", serialized)
|
||||
|
||||
def test_cli_does_not_echo_input_path_and_schema_marks_kappa_required(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
secret_name = "person-at-example.test.json"
|
||||
path = Path(directory) / secret_name
|
||||
path.write_text(json.dumps(_valid_payload()), encoding="utf-8")
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output):
|
||||
exit_code = MODULE.main(["--input", str(path)])
|
||||
|
||||
self.assertEqual(1, exit_code)
|
||||
self.assertNotIn(secret_name, output.getvalue())
|
||||
|
||||
schema = MODULE.G7HumanVoiceGainEvidencePack.model_json_schema()
|
||||
reliability = schema["$defs"]["G7ReliabilityClaim"]
|
||||
label = schema["$defs"]["G7HumanAxisLabel"]
|
||||
self.assertIn("reported_categorical_kappa", reliability["required"])
|
||||
self.assertIn("category", label["required"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -44,6 +44,7 @@ def human_pack() -> dict[str, object]:
|
|||
)
|
||||
observations = []
|
||||
axes = ("goal", "task", "bond")
|
||||
categories = ("low", "medium", "high")
|
||||
for session_index in range(50):
|
||||
participant = f"held-{session_index % 30:03d}"
|
||||
for axis_index, axis in enumerate(axes):
|
||||
|
|
@ -60,8 +61,16 @@ def human_pack() -> dict[str, object]:
|
|||
"voice_enabled_status": "observed",
|
||||
"voice_enabled_score": target,
|
||||
"labels": [
|
||||
{"labeler_key": "labeler-001", "score": target},
|
||||
{"labeler_key": "labeler-002", "score": target},
|
||||
{
|
||||
"labeler_key": "labeler-001",
|
||||
"score": target,
|
||||
"category": categories[axis_index],
|
||||
},
|
||||
{
|
||||
"labeler_key": "labeler-002",
|
||||
"score": target,
|
||||
"category": categories[axis_index],
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
|
@ -108,6 +117,7 @@ def human_pack() -> dict[str, object]:
|
|||
"reliability": {
|
||||
"labeler_keys": ["labeler-001", "labeler-002"],
|
||||
"reported_icc": 1.0,
|
||||
"reported_categorical_kappa": 1.0,
|
||||
"report_sha256": _sha(112),
|
||||
},
|
||||
"observations": observations,
|
||||
|
|
@ -432,6 +442,25 @@ class G7ExternalProofTests(unittest.TestCase):
|
|||
self.assertEqual([], errors)
|
||||
self.assertTrue(gain["passed"])
|
||||
self.assertGreaterEqual(gain["held_out_participants"], 30)
|
||||
self.assertGreaterEqual(gain["recomputed_categorical_kappa"], 0.70)
|
||||
|
||||
def test_human_pack_cannot_bypass_the_required_categorical_kappa(self) -> None:
|
||||
errors: list[str] = []
|
||||
payload = human_pack()
|
||||
reliability = payload["reliability"]
|
||||
assert isinstance(reliability, dict)
|
||||
reliability.pop("reported_categorical_kappa")
|
||||
observations = payload["observations"]
|
||||
assert isinstance(observations, list)
|
||||
for observation in observations:
|
||||
labels = observation["labels"]
|
||||
for label in labels:
|
||||
label.pop("category")
|
||||
|
||||
gain = self.checker.validate_human_gain(payload, errors)
|
||||
|
||||
self.assertEqual({}, gain)
|
||||
self.assertIn("human_gain:invalid:ValidationError", errors)
|
||||
|
||||
def test_windows_host_topology_passes_without_weakening_compose(self) -> None:
|
||||
errors: list[str] = []
|
||||
|
|
|
|||
|
|
@ -693,6 +693,15 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
|
|||
set(self.agent_module.POSTDEPLOY_NAS_E2E_SPECS)
|
||||
| set(self.agent_module.POSTDEPLOY_SOURCE_ONLY_E2E_SPECS),
|
||||
)
|
||||
self.assertEqual(
|
||||
set(self.agent_module.POSTDEPLOY_NAS_E2E_SPECS),
|
||||
set(self.agent_module.POSTDEPLOY_NAS_REAL_API_E2E_SPECS)
|
||||
| set(self.agent_module.POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS),
|
||||
)
|
||||
self.assertFalse(
|
||||
set(self.agent_module.POSTDEPLOY_NAS_REAL_API_E2E_SPECS)
|
||||
& set(self.agent_module.POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS)
|
||||
)
|
||||
self.assertLess(
|
||||
runner.event_log.index("runner:postdeploy_browser_review"),
|
||||
runner.event_log.index("deployment:commit_active_state"),
|
||||
|
|
@ -750,6 +759,22 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
|
|||
["e2e/returned-practice-db-closed-loop.spec.ts"],
|
||||
evidence["browser_review_proof"]["separate_disposable_db_specs"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"real_api_db_specs": [
|
||||
"e2e/session-layout.spec.ts",
|
||||
"e2e/session-persistence.spec.ts",
|
||||
],
|
||||
"route_fixture_specs": list(
|
||||
self.agent_module.POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS
|
||||
),
|
||||
},
|
||||
evidence["browser_review_proof"]["runtime_scope"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"not_run_by_release_agent",
|
||||
evidence["browser_review_proof"]["separate_disposable_db_status"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
"docs/dev_dashboard.html",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -50,8 +52,8 @@ def args(**overrides):
|
|||
"microphone_device": "",
|
||||
"confirm_physical_capture": False,
|
||||
"duration_seconds": 3_120.0,
|
||||
"runtime_interval_seconds": 100.0,
|
||||
"topology_interval_seconds": 100.0,
|
||||
"runtime_interval_seconds": 60.0,
|
||||
"topology_interval_seconds": 60.0,
|
||||
"human_voice_gain": Path("pack.json"),
|
||||
"out_dir": Path("out"),
|
||||
"rehearse": False,
|
||||
|
|
@ -154,6 +156,8 @@ class HostAlignmentTest(unittest.TestCase):
|
|||
self.assertEqual(MODULE.DEFAULT_BROWSER_ORIGIN, parsed.origin)
|
||||
self.assertEqual(MODULE.DEFAULT_ADMIN_RUNTIME_URL, parsed.admin_runtime_url)
|
||||
self.assertEqual(3_120.0, parsed.duration_seconds)
|
||||
self.assertEqual(60.0, parsed.runtime_interval_seconds)
|
||||
self.assertEqual(60.0, parsed.topology_interval_seconds)
|
||||
self.assertIsNone(parsed.allowed_browser_origins)
|
||||
help_text = parser.format_help()
|
||||
for expected in (
|
||||
|
|
@ -168,13 +172,13 @@ class HostAlignmentTest(unittest.TestCase):
|
|||
|
||||
class SamplePlanTest(unittest.TestCase):
|
||||
def test_samples_cover_the_whole_window(self) -> None:
|
||||
self.assertEqual(MODULE.sample_plan(3_120.0, 100.0), 33)
|
||||
self.assertEqual(MODULE.sample_plan(3_120.0, 60.0), 53)
|
||||
|
||||
def test_fractional_interval_always_gets_a_terminal_sample(self) -> None:
|
||||
self.assertEqual(MODULE.sample_plan(3_121.0, 100.0), 33)
|
||||
self.assertEqual(MODULE.sample_plan(3_121.0, 60.0), 54)
|
||||
|
||||
def test_invalid_plan_fails_closed(self) -> None:
|
||||
for duration, interval in ((0, 100.0), (3_000.0, 0)):
|
||||
for duration, interval in ((0, 60.0), (3_000.0, 0)):
|
||||
with self.subTest(duration=duration, interval=interval):
|
||||
with self.assertRaises(MODULE.WindowError):
|
||||
MODULE.sample_plan(duration, interval)
|
||||
|
|
@ -226,6 +230,44 @@ class ConsentGateTest(unittest.TestCase):
|
|||
def test_rehearse_relaxes_only_the_two_human_inputs(self) -> None:
|
||||
MODULE.validate(args(rehearse=True, duration_seconds=30.0, human_voice_gain=None))
|
||||
|
||||
def test_child_sampling_intervals_are_bounded_before_capture(self) -> None:
|
||||
for field in ("runtime_interval_seconds", "topology_interval_seconds"):
|
||||
with self.subTest(field=field):
|
||||
with self.assertRaises(MODULE.WindowError) as ctx:
|
||||
MODULE.validate(args(**{field: 60.001}))
|
||||
self.assertEqual(
|
||||
f"{field.removesuffix('_seconds')}_out_of_bounds",
|
||||
str(ctx.exception),
|
||||
)
|
||||
|
||||
def test_human_pack_is_fully_validated_before_capture(self) -> None:
|
||||
from app.test_g7_voice_gain_evidence import _valid_payload
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
missing = root / "missing.json"
|
||||
with self.assertRaises(MODULE.WindowError) as ctx:
|
||||
MODULE.validate_human_voice_gain_pack(missing)
|
||||
self.assertEqual("human_voice_gain_pack_unreadable", str(ctx.exception))
|
||||
|
||||
malformed = root / "malformed.json"
|
||||
malformed.write_text("{}", encoding="utf-8")
|
||||
with self.assertRaises(MODULE.WindowError) as ctx:
|
||||
MODULE.validate_human_voice_gain_pack(malformed)
|
||||
self.assertEqual("human_voice_gain_pack_invalid", str(ctx.exception))
|
||||
|
||||
underpowered = root / "underpowered.json"
|
||||
underpowered.write_text(
|
||||
json.dumps(_valid_payload()),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(MODULE.WindowError) as ctx:
|
||||
MODULE.validate_human_voice_gain_pack(underpowered)
|
||||
self.assertTrue(
|
||||
str(ctx.exception).startswith("human_voice_gain_pack_failed:")
|
||||
)
|
||||
self.assertIn("production_participant_floor", str(ctx.exception))
|
||||
|
||||
|
||||
class LegCompositionTest(unittest.TestCase):
|
||||
def test_expected_providers_default_to_the_decided_local_stack(self) -> None:
|
||||
|
|
|
|||
360
scripts/test_run_periodic_learner_e2e.py
Normal file
360
scripts/test_run_periodic_learner_e2e.py
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("run-periodic-learner-e2e.py")
|
||||
|
||||
|
||||
def load_module() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location("run_periodic_learner_e2e", SCRIPT)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("cannot load periodic learner runner")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
runner = load_module()
|
||||
|
||||
|
||||
class FakeController:
|
||||
def __init__(self, *, tracked_status: str = "") -> None:
|
||||
self.tracked_status = tracked_status
|
||||
self.commands: list[tuple[str, list[str]]] = []
|
||||
self.starts: list[tuple[str, list[str]]] = []
|
||||
self.stops: list[int] = []
|
||||
|
||||
def run(
|
||||
self,
|
||||
stage,
|
||||
argv,
|
||||
*,
|
||||
cwd,
|
||||
env=None,
|
||||
timeout,
|
||||
check=True,
|
||||
):
|
||||
del cwd, env, timeout, check
|
||||
self.commands.append((stage, list(argv)))
|
||||
stdout = ""
|
||||
if stage == "source_head":
|
||||
stdout = "1" * 40 + "\n"
|
||||
elif stage == "source_tree":
|
||||
stdout = "2" * 40 + "\n"
|
||||
elif stage == "source_tracked_clean":
|
||||
stdout = self.tracked_status
|
||||
elif stage == "source_untracked_inventory":
|
||||
stdout = "?? apps/api/engine.err.log.bak\n"
|
||||
elif stage == "docker_context_show":
|
||||
stdout = "desktop-linux\n"
|
||||
elif stage == "docker_context_inspect":
|
||||
stdout = json.dumps(
|
||||
[
|
||||
{
|
||||
"Endpoints": {
|
||||
"docker": {
|
||||
"Host": "npipe:////./pipe/dockerDesktopLinuxEngine"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
)
|
||||
return runner.CommandResult(0, stdout, "", 0.01)
|
||||
|
||||
def start(
|
||||
self,
|
||||
stage,
|
||||
argv,
|
||||
*,
|
||||
cwd,
|
||||
env,
|
||||
stdout_path,
|
||||
stderr_path,
|
||||
):
|
||||
del cwd, env, stdout_path, stderr_path
|
||||
self.starts.append((stage, list(argv)))
|
||||
return runner.ProcessHandle(4242)
|
||||
|
||||
def stop_exact(self, handle, *, timeout):
|
||||
del timeout
|
||||
self.stops.append(handle.pid)
|
||||
return True
|
||||
|
||||
def http_json(self, url, *, headers=None, timeout):
|
||||
del url, headers, timeout
|
||||
return {"ok": True, "status": "ok", "db": True, "engine": True}
|
||||
|
||||
def http_text(self, url, *, timeout):
|
||||
del url, timeout
|
||||
return '<div id="root"></div>'
|
||||
|
||||
def tcp_listening(self, host, port, *, timeout=0.25):
|
||||
del host, port, timeout
|
||||
return False
|
||||
|
||||
def sleep(self, seconds):
|
||||
del seconds
|
||||
|
||||
|
||||
def config(receipt_path: Path) -> runner.RunnerConfig:
|
||||
return runner.RunnerConfig(
|
||||
receipt_path=receipt_path,
|
||||
python_exe="python.exe",
|
||||
node_exe="node.exe",
|
||||
docker_exe="docker.exe",
|
||||
execute=True,
|
||||
)
|
||||
|
||||
|
||||
class SafetyContractTests(unittest.TestCase):
|
||||
def test_rejects_every_protected_port_from_argv_and_runtime_env(self) -> None:
|
||||
for port in sorted(runner.FORBIDDEN_PORTS):
|
||||
with self.subTest(port=port):
|
||||
with self.assertRaises(runner.GateError):
|
||||
runner.assert_safe_invocation([f"http://127.0.0.1:{port}"], {})
|
||||
with self.assertRaises(runner.GateError):
|
||||
runner.assert_safe_invocation([], {"DATABASE_URL": f"postgresql://x@127.0.0.1:{port}/db"})
|
||||
|
||||
def test_rejects_public_and_nas_markers(self) -> None:
|
||||
values = [
|
||||
"https://vignette.chanpaca.net",
|
||||
"https://api-vignette.chanpaca.net/health",
|
||||
"postgresql://app@100.116.83.60:55433/vignette",
|
||||
"vignette-preview-20260807",
|
||||
"vignette-dev-db",
|
||||
"/volume1/docker/vignette",
|
||||
]
|
||||
for value in values:
|
||||
with self.subTest(value=value), self.assertRaises(runner.GateError):
|
||||
runner.assert_safe_value("contract", value)
|
||||
|
||||
def test_rejects_inherited_runtime_targets_even_when_the_port_looks_local(self) -> None:
|
||||
for key, value in (
|
||||
("DATABASE_URL", "postgresql://app@127.0.0.1:55439/vignette"),
|
||||
("ENGINE_URL", "http://127.0.0.1:9199"),
|
||||
("PLAYWRIGHT_BASE_URL", "http://127.0.0.1:5199"),
|
||||
("COMPOSE_PROJECT_NAME", "some-existing-project"),
|
||||
):
|
||||
with self.subTest(key=key), self.assertRaises(runner.GateError):
|
||||
runner.assert_no_inherited_runtime_targets({key: value})
|
||||
|
||||
def test_runtime_identity_and_generated_environment_are_disposable(self) -> None:
|
||||
source = runner.SourceIdentity("1" * 40, "2" * 40)
|
||||
with patch.object(runner, "allocate_unique_ports", return_value=(18080, 18443, 15439, 19199)):
|
||||
runtime = runner.build_runtime_identity(source)
|
||||
runner.validate_runtime_identity(runtime)
|
||||
values = runner.build_stack_environment(runtime)
|
||||
|
||||
self.assertEqual(len(set(runtime.ports)), 4)
|
||||
self.assertTrue(runner.PROJECT_RE.fullmatch(runtime.project))
|
||||
self.assertEqual(values["HTTP_PORT"], "18080")
|
||||
self.assertEqual(values["DB_HOST_PORT"], "15439")
|
||||
self.assertEqual(values["ENGINE_URL"], "http://host.docker.internal:19199")
|
||||
self.assertNotIn("vignette.chanpaca.net", json.dumps(values))
|
||||
self.assertFalse(set(runtime.ports) & runner.FORBIDDEN_PORTS)
|
||||
|
||||
def test_compose_override_labels_every_resource_and_loopback_binds_database(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
path = Path(raw) / "override.yml"
|
||||
runner.write_compose_override(path)
|
||||
text = path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertEqual(text.count("com.vignette.periodic-e2e-sentinel"), 8)
|
||||
self.assertIn('127.0.0.1:${DB_HOST_PORT}:5432', text)
|
||||
self.assertIn("pgdata:", text)
|
||||
self.assertIn("apiuploads:", text)
|
||||
self.assertIn("caddydata:", text)
|
||||
self.assertIn("vignette:", text)
|
||||
|
||||
def test_resolved_compose_contract_binds_only_loopback_and_internal_db(self) -> None:
|
||||
runtime = runner.RuntimeIdentity(
|
||||
"20260809T120000-abcdef12",
|
||||
"vignette-periodic-11111111-abcdef12",
|
||||
"periodic:" + "1" * 40 + ":" + "2" * 40 + ":run",
|
||||
18080,
|
||||
18443,
|
||||
15439,
|
||||
19199,
|
||||
)
|
||||
stack_env = {
|
||||
"POSTGRES_DB": "vignette_periodic",
|
||||
}
|
||||
payload = {
|
||||
"services": {
|
||||
name: {
|
||||
"labels": {
|
||||
"com.vignette.periodic-e2e-sentinel": runtime.sentinel
|
||||
},
|
||||
"ports": [],
|
||||
}
|
||||
for name in ("db", "api", "web", "proxy")
|
||||
},
|
||||
"volumes": {
|
||||
name: {
|
||||
"labels": {
|
||||
"com.vignette.periodic-e2e-sentinel": runtime.sentinel
|
||||
}
|
||||
}
|
||||
for name in ("pgdata", "apiuploads", "caddydata")
|
||||
},
|
||||
"networks": {
|
||||
"vignette": {
|
||||
"labels": {
|
||||
"com.vignette.periodic-e2e-sentinel": runtime.sentinel
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
payload["services"]["db"]["ports"] = [
|
||||
{"host_ip": "127.0.0.1", "target": 5432, "published": "15439"}
|
||||
]
|
||||
payload["services"]["proxy"]["ports"] = [
|
||||
{"host_ip": "127.0.0.1", "target": 80, "published": "18080"},
|
||||
{"host_ip": "127.0.0.1", "target": 443, "published": "18443"},
|
||||
]
|
||||
payload["services"]["api"]["environment"] = {
|
||||
"DATABASE_URL": "postgresql://app:secret@db:5432/vignette_periodic",
|
||||
"ENGINE_URL": "http://host.docker.internal:19199",
|
||||
"FRONTEND_BASE_URL": "http://127.0.0.1:18080",
|
||||
"OAUTH_REDIRECT_URI": "http://127.0.0.1:18080/api/auth/callback",
|
||||
}
|
||||
|
||||
class ConfigFake(FakeController):
|
||||
def run(self, stage, argv, **kwargs):
|
||||
result = super().run(stage, argv, **kwargs)
|
||||
if stage == "resolved_compose_config":
|
||||
return runner.CommandResult(0, json.dumps(payload), "", 0.01)
|
||||
return result
|
||||
|
||||
fake = ConfigFake()
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
periodic = runner.PeriodicRunner(config(Path(raw) / "receipt.json"), fake)
|
||||
periodic.state.runtime = runtime
|
||||
periodic.state.docker_context = "desktop-linux"
|
||||
periodic.state.env_file = Path(raw) / "stack.env"
|
||||
periodic.state.override_file = Path(raw) / "override.yml"
|
||||
digest = periodic._validate_resolved_compose_config(stack_env)
|
||||
|
||||
self.assertEqual(len(digest), 64)
|
||||
payload["services"]["proxy"]["ports"][0]["host_ip"] = "0.0.0.0"
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
periodic = runner.PeriodicRunner(config(Path(raw) / "receipt.json"), ConfigFake())
|
||||
periodic.state.runtime = runtime
|
||||
periodic.state.docker_context = "desktop-linux"
|
||||
periodic.state.env_file = Path(raw) / "stack.env"
|
||||
periodic.state.override_file = Path(raw) / "override.yml"
|
||||
with self.assertRaises(runner.GateError):
|
||||
periodic._validate_resolved_compose_config(stack_env)
|
||||
|
||||
def test_dirty_source_fails_before_process_or_compose_mutation(self) -> None:
|
||||
fake = FakeController(tracked_status=" M apps/web/src/App.tsx\n")
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
receipt = Path(raw) / "receipt.json"
|
||||
execution = runner.PeriodicRunner(config(receipt), fake).execute()
|
||||
stored = json.loads(receipt.read_text(encoding="utf-8"))
|
||||
|
||||
self.assertEqual(execution["status"], "FAILED")
|
||||
self.assertEqual(stored["status"], "FAILED")
|
||||
self.assertIn("tracked worktree differs", execution["error"])
|
||||
self.assertEqual(fake.starts, [])
|
||||
self.assertFalse(any("compose" in command for _, command in fake.commands))
|
||||
|
||||
def test_docker_context_is_pinned_to_local_windows_named_pipe(self) -> None:
|
||||
fake = FakeController()
|
||||
periodic = runner.PeriodicRunner(config(Path("receipt.json")), fake)
|
||||
proof = periodic._pin_local_docker_context()
|
||||
self.assertEqual(proof, {"context": "desktop-linux", "transport": "npipe", "remote": False})
|
||||
self.assertEqual(periodic.state.docker_context, "desktop-linux")
|
||||
|
||||
class RemoteDockerFake(FakeController):
|
||||
def run(self, stage, argv, **kwargs):
|
||||
result = super().run(stage, argv, **kwargs)
|
||||
if stage == "docker_context_inspect":
|
||||
return runner.CommandResult(
|
||||
0,
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"Endpoints": {
|
||||
"docker": {"Host": "tcp://127.0.0.1:2375"}
|
||||
}
|
||||
}
|
||||
]
|
||||
),
|
||||
"",
|
||||
0.01,
|
||||
)
|
||||
return result
|
||||
|
||||
with self.assertRaises(runner.GateError):
|
||||
runner.PeriodicRunner(
|
||||
config(Path("receipt.json")), RemoteDockerFake()
|
||||
)._pin_local_docker_context()
|
||||
|
||||
def test_cleanup_targets_only_exact_project_and_proves_listener_zero(self) -> None:
|
||||
fake = FakeController()
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
periodic = runner.PeriodicRunner(config(Path(raw) / "receipt.json"), fake)
|
||||
periodic.state.source = runner.SourceIdentity("1" * 40, "2" * 40)
|
||||
periodic.state.runtime = runner.RuntimeIdentity(
|
||||
"20260809T120000-abcdef12",
|
||||
"vignette-periodic-11111111-abcdef12",
|
||||
"periodic:" + "1" * 40 + ":" + "2" * 40 + ":run",
|
||||
18080,
|
||||
18443,
|
||||
15439,
|
||||
19199,
|
||||
)
|
||||
periodic.state.docker_context = "desktop-linux"
|
||||
periodic.state.env_file = Path(raw) / "stack.env"
|
||||
periodic.state.override_file = Path(raw) / "override.yml"
|
||||
periodic.state.stack_attempted = True
|
||||
periodic.state.engine = runner.ProcessHandle(4242)
|
||||
proof = periodic._cleanup()
|
||||
|
||||
down = next(command for stage, command in fake.commands if stage == "compose_down")
|
||||
self.assertIn("vignette-periodic-11111111-abcdef12", down)
|
||||
self.assertIn("--remove-orphans", down)
|
||||
self.assertIn("--volumes", down)
|
||||
self.assertEqual(fake.stops, [4242])
|
||||
self.assertEqual(proof["container_remainder"], 0)
|
||||
self.assertEqual(proof["volume_remainder"], 0)
|
||||
self.assertEqual(proof["network_remainder"], 0)
|
||||
self.assertEqual(set(proof["listener_counts"].values()), {0})
|
||||
self.assertTrue(periodic._cleanup_green(proof))
|
||||
|
||||
def test_untracked_runtime_source_is_rejected_but_log_backup_is_not(self) -> None:
|
||||
fake = FakeController()
|
||||
periodic = runner.PeriodicRunner(config(Path("receipt.json")), fake)
|
||||
identity = periodic.source_identity(require_clean=True)
|
||||
self.assertEqual(identity.head, "1" * 40)
|
||||
|
||||
class DangerousFake(FakeController):
|
||||
def run(self, stage, argv, **kwargs):
|
||||
result = super().run(stage, argv, **kwargs)
|
||||
if stage == "source_untracked_inventory":
|
||||
return runner.CommandResult(
|
||||
0,
|
||||
"?? apps/web/e2e/uncommitted-runtime.spec.ts\n",
|
||||
"",
|
||||
0.01,
|
||||
)
|
||||
return result
|
||||
|
||||
with self.assertRaises(runner.GateError):
|
||||
runner.PeriodicRunner(
|
||||
config(Path("receipt.json")), DangerousFake()
|
||||
).source_identity(require_clean=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue