주기 회기 검증 격리 보강
This commit is contained in:
Yun Chan
parent
9a7dd98cb6
commit
4cd354881f
10 changed files with 181 additions and 69 deletions
|
|
@ -906,6 +906,65 @@ class PeriodicRunner:
|
|||
values["E2E_PERIODIC_LEARNER_RESULT"] = str(result)
|
||||
return _clean_child_env(values)
|
||||
|
||||
def _prepare_returned_practice_fixture(
|
||||
self,
|
||||
*,
|
||||
stage: str,
|
||||
base_url: str,
|
||||
database_url: str,
|
||||
database_admin_url: str,
|
||||
stack_env: Mapping[str, str],
|
||||
fixture_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
harness_env = _clean_child_env(
|
||||
{
|
||||
"PYTHONPATH": str(REPO_ROOT / "apps" / "api"),
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
self._run(
|
||||
stage,
|
||||
[
|
||||
self.config.python_exe,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(FIXTURE_SCRIPT),
|
||||
"--api-base-url",
|
||||
f"{base_url}/api",
|
||||
"--database-url",
|
||||
database_url,
|
||||
"--database-admin-url",
|
||||
database_admin_url,
|
||||
"--practice-internal-token",
|
||||
stack_env["VIGNETTE_PRACTICE_INTERNAL_TOKEN"],
|
||||
"--transfer-internal-token",
|
||||
stack_env["VIGNETTE_CALIBRATION_TRANSFER_INTERNAL_TOKEN"],
|
||||
"--out",
|
||||
str(fixture_path),
|
||||
"--request-timeout",
|
||||
str(self.config.request_timeout),
|
||||
"--review-poll-timeout",
|
||||
str(self.config.readiness_timeout),
|
||||
],
|
||||
env=harness_env,
|
||||
timeout=20 * 60,
|
||||
)
|
||||
fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
setup = fixture.get("setup_proof") or {}
|
||||
expected_setup = {
|
||||
"source_review_ready": True,
|
||||
"follow_up_review_ready": True,
|
||||
"distinct_persona": True,
|
||||
"initial_runtime_observation_count": 0,
|
||||
"initial_actual_transfer_execution_count": 0,
|
||||
}
|
||||
if any(setup.get(key) != value for key, value in expected_setup.items()):
|
||||
raise GateError("returned-practice fixture setup proof is incomplete")
|
||||
return {
|
||||
**expected_setup,
|
||||
"fixture_sha256": _sha256_path(fixture_path),
|
||||
}
|
||||
|
||||
def _validate_periodic_result(self, result_path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
|
|
@ -1070,7 +1129,8 @@ class PeriodicRunner:
|
|||
self.state.temp_dir = temp_dir
|
||||
self.state.env_file = temp_dir / "stack.env"
|
||||
self.state.override_file = temp_dir / "compose.periodic.yml"
|
||||
fixture_path = temp_dir / "returned-practice-fixture.json"
|
||||
periodic_fixture_path = temp_dir / "periodic-learner-fixture.json"
|
||||
returned_fixture_path = temp_dir / "returned-regression-fixture.json"
|
||||
periodic_result = temp_dir / "periodic-result.json"
|
||||
stack_env = build_stack_environment(runtime)
|
||||
write_env_file(self.state.env_file, stack_env)
|
||||
|
|
@ -1184,53 +1244,15 @@ class PeriodicRunner:
|
|||
"database_sentinel_sha256": sentinel_sha,
|
||||
}
|
||||
|
||||
harness_env = _clean_child_env(
|
||||
{
|
||||
"PYTHONPATH": str(REPO_ROOT / "apps" / "api"),
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
self._run(
|
||||
"prepare_returned_practice_fixture",
|
||||
[
|
||||
self.config.python_exe,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(FIXTURE_SCRIPT),
|
||||
"--api-base-url",
|
||||
f"{base_url}/api",
|
||||
"--database-url",
|
||||
database_url,
|
||||
"--database-admin-url",
|
||||
database_admin_url,
|
||||
"--practice-internal-token",
|
||||
stack_env["VIGNETTE_PRACTICE_INTERNAL_TOKEN"],
|
||||
"--transfer-internal-token",
|
||||
stack_env["VIGNETTE_CALIBRATION_TRANSFER_INTERNAL_TOKEN"],
|
||||
"--out",
|
||||
str(fixture_path),
|
||||
"--request-timeout",
|
||||
str(self.config.request_timeout),
|
||||
"--review-poll-timeout",
|
||||
str(self.config.readiness_timeout),
|
||||
],
|
||||
env=harness_env,
|
||||
timeout=20 * 60,
|
||||
)
|
||||
fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
setup = fixture.get("setup_proof") or {}
|
||||
expected_setup = {
|
||||
"source_review_ready": True,
|
||||
"follow_up_review_ready": True,
|
||||
"distinct_persona": True,
|
||||
"initial_runtime_observation_count": 0,
|
||||
"initial_actual_transfer_execution_count": 0,
|
||||
}
|
||||
if any(setup.get(key) != value for key, value in expected_setup.items()):
|
||||
raise GateError("returned-practice fixture setup proof is incomplete")
|
||||
self.state.proof["fixture"] = {
|
||||
**expected_setup,
|
||||
"fixture_sha256": _sha256_path(fixture_path),
|
||||
self.state.proof["fixtures"] = {
|
||||
"periodic": self._prepare_returned_practice_fixture(
|
||||
stage="prepare_periodic_learner_fixture",
|
||||
base_url=base_url,
|
||||
database_url=database_url,
|
||||
database_admin_url=database_admin_url,
|
||||
stack_env=stack_env,
|
||||
fixture_path=periodic_fixture_path,
|
||||
)
|
||||
}
|
||||
|
||||
web_dir = REPO_ROOT / "apps" / "web"
|
||||
|
|
@ -1248,7 +1270,11 @@ class PeriodicRunner:
|
|||
"--reporter=line",
|
||||
],
|
||||
cwd=web_dir,
|
||||
env=self._browser_env(runtime, fixture_path, periodic_result),
|
||||
env=self._browser_env(
|
||||
runtime,
|
||||
periodic_fixture_path,
|
||||
periodic_result,
|
||||
),
|
||||
timeout=25 * 60,
|
||||
)
|
||||
if not re.search(r"\b1 passed\b", periodic_run.stdout + periodic_run.stderr):
|
||||
|
|
@ -1257,6 +1283,17 @@ class PeriodicRunner:
|
|||
periodic_result
|
||||
)
|
||||
|
||||
self.state.proof["fixtures"]["returned_regression"] = (
|
||||
self._prepare_returned_practice_fixture(
|
||||
stage="prepare_returned_regression_fixture",
|
||||
base_url=base_url,
|
||||
database_url=database_url,
|
||||
database_admin_url=database_admin_url,
|
||||
stack_env=stack_env,
|
||||
fixture_path=returned_fixture_path,
|
||||
)
|
||||
)
|
||||
|
||||
returned_run = self._run(
|
||||
"returned_practice_desktop_mobile",
|
||||
[
|
||||
|
|
@ -1271,7 +1308,7 @@ class PeriodicRunner:
|
|||
"--reporter=line",
|
||||
],
|
||||
cwd=web_dir,
|
||||
env=self._browser_env(runtime, fixture_path),
|
||||
env=self._browser_env(runtime, returned_fixture_path),
|
||||
timeout=25 * 60,
|
||||
)
|
||||
if not re.search(r"\b4 passed\b", returned_run.stdout + returned_run.stderr):
|
||||
|
|
|
|||
|
|
@ -30,6 +30,68 @@ class PeriodicRunnerStaticContractTests(unittest.TestCase):
|
|||
def test_browser_failures_are_not_retried_automatically(self) -> None:
|
||||
self.assertEqual(runner.PLAYWRIGHT_RETRIES, 0)
|
||||
|
||||
def test_browser_gates_prepare_distinct_zero_state_fixtures(self) -> None:
|
||||
class FixtureController(FakeController):
|
||||
def run(self, stage, argv, **kwargs):
|
||||
result = super().run(stage, argv, **kwargs)
|
||||
if stage.startswith("prepare_"):
|
||||
output = Path(argv[argv.index("--out") + 1])
|
||||
output.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"fixture_id": stage,
|
||||
"setup_proof": {
|
||||
"source_review_ready": True,
|
||||
"follow_up_review_ready": True,
|
||||
"distinct_persona": True,
|
||||
"initial_runtime_observation_count": 0,
|
||||
"initial_actual_transfer_execution_count": 0,
|
||||
}
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
root = Path(raw)
|
||||
fake = FixtureController()
|
||||
periodic = runner.PeriodicRunner(config(root / "receipt.json"), fake)
|
||||
stack_env = {
|
||||
"VIGNETTE_PRACTICE_INTERNAL_TOKEN": "practice-token",
|
||||
"VIGNETTE_CALIBRATION_TRANSFER_INTERNAL_TOKEN": "transfer-token",
|
||||
}
|
||||
periodic_path = root / "periodic.json"
|
||||
regression_path = root / "regression.json"
|
||||
periodic_proof = periodic._prepare_returned_practice_fixture(
|
||||
stage="prepare_periodic_learner_fixture",
|
||||
base_url="http://127.0.0.1:18080",
|
||||
database_url="postgresql://app:secret@127.0.0.1:15439/vignette",
|
||||
database_admin_url="postgresql://owner:secret@127.0.0.1:15439/vignette",
|
||||
stack_env=stack_env,
|
||||
fixture_path=periodic_path,
|
||||
)
|
||||
regression_proof = periodic._prepare_returned_practice_fixture(
|
||||
stage="prepare_returned_regression_fixture",
|
||||
base_url="http://127.0.0.1:18080",
|
||||
database_url="postgresql://app:secret@127.0.0.1:15439/vignette",
|
||||
database_admin_url="postgresql://owner:secret@127.0.0.1:15439/vignette",
|
||||
stack_env=stack_env,
|
||||
fixture_path=regression_path,
|
||||
)
|
||||
|
||||
self.assertNotEqual(periodic_path.name, regression_path.name)
|
||||
self.assertEqual(
|
||||
[stage for stage, _ in fake.commands if stage.startswith("prepare_")],
|
||||
["prepare_periodic_learner_fixture", "prepare_returned_regression_fixture"],
|
||||
)
|
||||
self.assertEqual(periodic_proof["initial_runtime_observation_count"], 0)
|
||||
self.assertEqual(regression_proof["initial_actual_transfer_execution_count"], 0)
|
||||
self.assertNotEqual(
|
||||
periodic_proof["fixture_sha256"],
|
||||
regression_proof["fixture_sha256"],
|
||||
)
|
||||
|
||||
|
||||
class FakeController:
|
||||
def __init__(self, *, tracked_status: str = "") -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue