yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
vignette/scripts/run-outcome-os-release-agent.py

1928 lines
77 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Fail-closed G8 release agent for the isolated Vignette NAS preview.
The default is a read-only dry run. ``--execute`` is intentionally narrow:
it accepts only the dedicated Tailnet preview, builds either the default
verified HEAD+patch candidate or an explicitly requested clean-HEAD archive
candidate, runs release/API/Web/session gates, snapshots the current API and
Web images, promotes only a changed deterministic SHA, and rolls back the
previous images when any post-promotion proof fails.
This program never runs ``docker compose down`` against the NAS, never removes
remote volumes or networks, and never targets another Compose project. A
successful changed-SHA promotion emits an SSOT evidence contract; a separate
agent must bind that exact SHA into the dashboard/TODO/backlog/manifest and run
their canonical checkers.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shlex
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.error
import urllib.request
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Protocol
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MANIFEST = REPO_ROOT / "docs/ops/outcome-os-clean-release-manifest-2026-08-07.json"
DEFAULT_HUNK_MAP = REPO_ROOT / "docs/ops/outcome-os-release-hunk-map-2026-08-07.json"
DEFAULT_NAS_ENV = REPO_ROOT / "infra/.env.nas-preview"
BUILDER = REPO_ROOT / "scripts/build-outcome-os-release-patch.py"
MANIFEST_CHECKER = REPO_ROOT / "scripts/check-outcome-os-release-manifest.py"
PREFLIGHT = REPO_ROOT / "scripts/check-deploy-preflight.py"
SSOT_REQUIRED_PATHS = (
"docs/dev_dashboard.html",
"docs/TODO.md",
"docs/ops/backlog-2026-06-26.md",
"docs/ops/outcome-os-clean-release-manifest-2026-08-07.json",
)
# A promotion is allowed only after every Outcome OS goal's primary learner or
# operator surface has exercised its browser contract. Keep this list explicit
# so a newly added goal cannot disappear behind a broad directory glob.
RELEASE_UI_E2E_SPECS = (
"e2e/insecure-context-uuid.spec.ts",
"e2e/session-layout.spec.ts",
"e2e/session-persistence.spec.ts",
"e2e/self-directed-learning-loop.spec.ts",
"e2e/alliance-pulse.spec.ts",
"e2e/outcome-trajectory.spec.ts",
"e2e/rupture-repair.spec.ts",
"e2e/deliberate-practice.spec.ts",
"e2e/calibration-transfer.spec.ts",
"e2e/supervision-research.spec.ts",
"e2e/multimodal-alliance.spec.ts",
"e2e/continuous-improvement-admin.spec.ts",
)
# The UUID source-contract spec intentionally imports ``/src/lib/uuid.ts`` and
# scans the local source tree, so it belongs to the Vite candidate gate above.
# A production NAS build serves the SPA HTML for ``/src/*``. The postdeploy
# gate therefore exercises the exact built review routes that previously
# crashed on insecure HTTP, while retaining every production-origin-compatible
# release spec. The real returned-practice DB closed loop remains an explicit
# disposable-DB-only gate and must never be silently skipped against NAS.
POSTDEPLOY_NAS_E2E_SPECS = (
"e2e/session-layout.spec.ts",
"e2e/session-persistence.spec.ts",
"e2e/self-directed-learning-loop.spec.ts",
"e2e/alliance-pulse.spec.ts",
"e2e/outcome-trajectory.spec.ts",
"e2e/rupture-repair.spec.ts",
"e2e/deliberate-practice.spec.ts",
"e2e/calibration-transfer.spec.ts",
"e2e/supervision-research.spec.ts",
"e2e/multimodal-alliance.spec.ts",
"e2e/continuous-improvement-admin.spec.ts",
)
# Only these specs use the configured NAS API/DB instead of replacing the
# product API with Playwright route fixtures. Keep the split explicit in the
# receipt so the 112-test browser total cannot be mistaken for 112 live
# session/database proofs.
POSTDEPLOY_NAS_REAL_API_E2E_SPECS = (
"e2e/session-layout.spec.ts",
"e2e/session-persistence.spec.ts",
)
POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS = tuple(
spec
for spec in POSTDEPLOY_NAS_E2E_SPECS
if spec not in POSTDEPLOY_NAS_REAL_API_E2E_SPECS
)
POSTDEPLOY_SOURCE_ONLY_E2E_SPECS = ("e2e/insecure-context-uuid.spec.ts",)
POSTDEPLOY_DISPOSABLE_DB_E2E_SPECS = (
"e2e/returned-practice-db-closed-loop.spec.ts",
)
SOURCE_ONLY_E2E_PORT = 5198
RELEASE_BROWSER_PROJECTS = (
"chromium-desktop",
"chromium-mobile",
"chromium-single-run",
)
RELEASE_E2E_TIMEOUT_SECONDS = 15 * 60
# Existing preview volumes do not replay docker-entrypoint-initdb.d. These
# forward-compatible, idempotent Outcome OS migrations must therefore land
# before the new API image is restarted. Every file is applied atomically.
RELEASE_DB_MIGRATIONS = (
"14_continuous_improvement.sql",
"15_self_directed_practice_runtime.sql",
"16_calibration_transfer_actual_execution.sql",
"17_improvement_workbook_contracts.sql",
"19_auth_identity_alias.sql",
)
# These files are runtime-critical but were added after the first manifest
# snapshot. A release must classify them as whole-file payloads; otherwise a
# clean candidate can pass source tests while silently shipping the old mobile
# shell or omitting a migration that only fails after remote promotion starts.
REQUIRED_RELEASE_PAYLOAD_PATHS = (
"apps/web/src/components/shell/shell.css",
"infra/db/init/15_self_directed_practice_runtime.sql",
"infra/db/init/16_calibration_transfer_actual_execution.sql",
"infra/db/init/17_improvement_workbook_contracts.sql",
"infra/db/init/19_auth_identity_alias.sql",
)
REQUIRED_OPENAPI_PATHS = {
"G1": "/sessions/{session_id}/alliance-pulses",
"G2": "/sessions/{session_id}/outcome-trajectory",
"G3": "/sessions/{session_id}/ruptures",
"G4": "/practice/learners/me",
"G5": "/calibration/learners/me",
"G6": "/supervision-research/supervision-view",
"G7": "/sessions/{session_id}/multimodal-alliance",
"G8": "/continuous-improvement",
}
IMAGE_ID_RE = re.compile(r"^sha256:[a-f0-9]{64}$")
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
GIT_OBJECT_RE = re.compile(r"^(?:[a-f0-9]{40}|[a-f0-9]{64})$")
SSH_TARGET_RE = re.compile(r"^(?:[A-Za-z0-9._-]+@)?100\.116\.83\.60$")
RELEASE_KEY_RE = re.compile(r"^[a-f0-9]{16}-\d{8}T\d{6}Z-\d+$")
SOURCE_MODES = ("patch", "clean-head")
@dataclass(frozen=True)
class DeploymentTarget:
base_url: str
compose_project: str
remote_root: str
NAS_PREVIEW_TARGET = DeploymentTarget(
base_url="http://100.116.83.60:8088",
compose_project="vignette-preview-20260807",
remote_root="/volume1/docker/vignette-preview-20260807",
)
@dataclass(frozen=True)
class MaterialMilestone:
milestone_id: str
material: bool
goals: tuple[str, ...]
evidence_refs: tuple[str, ...]
@dataclass(frozen=True)
class CommandResult:
returncode: int
stdout: str
stderr: str
@dataclass(frozen=True)
class DeploymentSnapshot:
api_image: str
web_image: str
compose_file: str
env_file: str
active_sha: str | None
@dataclass
class ReleaseAgentConfig:
repo_root: Path = REPO_ROOT
manifest_path: Path = DEFAULT_MANIFEST
hunk_map_path: Path = DEFAULT_HUNK_MAP
source_mode: str = "patch"
execute: bool = False
milestone: MaterialMilestone | None = None
target: DeploymentTarget = NAS_PREVIEW_TARGET
evidence_out: Path | None = None
nas_env_file: Path = DEFAULT_NAS_ENV
candidate_base_url: str = "http://127.0.0.1:18088"
candidate_http_port: int = 18088
candidate_https_port: int = 18443
run_candidate_stack: bool = False
runtime_ready_timeout_seconds: float = 180.0
runtime_retry_interval_seconds: float = 3.0
class StageFailure(RuntimeError):
def __init__(self, stage: str, detail: str):
self.stage = stage
self.detail = detail
super().__init__(f"{stage}: {detail}")
class ReleaseAgentFailure(RuntimeError):
def __init__(self, report: dict[str, Any]):
self.report = report
super().__init__(
f"release agent failed at {report.get('failed_stage', 'unknown')}: "
f"{report.get('error', 'unknown error')}"
)
class Runner(Protocol):
def run(
self,
stage: str,
argv: list[str],
*,
cwd: Path | None = None,
env: dict[str, str] | None = None,
timeout: int | None = None,
allowed_returncodes: tuple[int, ...] = (0,),
) -> CommandResult: ...
class DeploymentDriver(Protocol):
def read_active_state(self) -> dict[str, Any] | None: ...
def snapshot(self) -> DeploymentSnapshot: ...
def promote(
self,
candidate: Path,
desired_sha: str,
snapshot: DeploymentSnapshot,
) -> dict[str, Any]: ...
def commit_active_state(self, state: dict[str, Any]) -> None: ...
def rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]: ...
def verify_rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]: ...
class RuntimeProbe(Protocol):
def verify(self) -> dict[str, Any]: ...
class CandidateManager(Protocol):
def materialize(self, base_commit: str, patch_path: str) -> Path: ...
def materialize_archive(self, archive_path: Path, expected_sha256: str) -> Path: ...
def cleanup(self) -> None: ...
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def sha256_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def sha256_file(path: Path) -> tuple[str, int]:
digest = hashlib.sha256()
size = 0
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
digest.update(chunk)
size += len(chunk)
return digest.hexdigest(), size
def validate_target(target: DeploymentTarget) -> None:
if target != NAS_PREVIEW_TARGET:
raise ValueError(
"only the isolated NAS preview target is allowed: "
f"{NAS_PREVIEW_TARGET!r}"
)
root = PurePosixPath(target.remote_root)
if not root.is_absolute() or ".." in root.parts:
raise ValueError("isolated NAS preview root must be an absolute canonical path")
def validate_source_repo_root(repo_root: Path) -> None:
if not repo_root.is_absolute():
raise StageFailure("source_repository", "source repository root must be absolute")
if not repo_root.exists():
raise StageFailure("source_repository", "source repository root does not exist")
if not repo_root.is_dir():
raise StageFailure("source_repository", "source repository root is not a directory")
def validate_required_release_payload(manifest: dict[str, Any]) -> None:
classifications = manifest.get("classifications")
related = classifications.get("related") if isinstance(classifications, dict) else None
if not isinstance(related, list):
raise StageFailure("release_manifest_binding", "related release classification is missing")
related_paths = {
path
for entry in related
if isinstance(entry, dict) and entry.get("disposition") == "candidate-whole-file"
for path in entry.get("paths", [])
if isinstance(path, str)
}
missing = sorted(set(REQUIRED_RELEASE_PAYLOAD_PATHS) - related_paths)
if missing:
raise StageFailure(
"release_manifest_binding",
f"runtime-critical release payload is not classified as related: {missing}",
)
def validate_remote_compose_path(
value: Any,
target: DeploymentTarget,
*,
stage: str,
) -> str:
if not isinstance(value, str):
raise StageFailure(stage, "remote Compose file is missing")
root = PurePosixPath(target.remote_root)
candidate = PurePosixPath(value)
if (
not candidate.is_absolute()
or ".." in candidate.parts
or str(candidate) != value
):
raise StageFailure(stage, "remote Compose file is not canonical")
try:
relative = candidate.relative_to(root)
except ValueError as exc:
raise StageFailure(stage, "remote Compose file escapes preview root") from exc
is_root_compose = relative.parts == ("infra", "docker-compose.yml")
is_legacy_release_compose = relative.parts == (
"release",
"infra",
"docker-compose.yml",
)
is_release_compose = (
len(relative.parts) == 4
and relative.parts[0] == "releases"
and RELEASE_KEY_RE.fullmatch(relative.parts[1]) is not None
and relative.parts[2:] == ("infra", "docker-compose.yml")
)
if not (is_root_compose or is_legacy_release_compose or is_release_compose):
raise StageFailure(stage, "remote Compose file is outside an approved release path")
return value
def validate_remote_env_file(
value: Any,
target: DeploymentTarget,
*,
stage: str,
) -> str:
if not isinstance(value, str):
raise StageFailure(stage, "remote env file is missing")
root = PurePosixPath(target.remote_root)
candidate = PurePosixPath(value)
if (
not candidate.is_absolute()
or ".." in candidate.parts
or str(candidate) != value
):
raise StageFailure(stage, "remote env file is not canonical")
try:
relative = candidate.relative_to(root)
except ValueError as exc:
raise StageFailure(stage, "remote env file escapes preview root") from exc
allowed = {
("infra", ".env"),
("release", "infra", ".env"),
}
if relative.parts not in allowed:
raise StageFailure(stage, "remote env file is outside the approved stable paths")
return value
def candidate_compose_project(desired_sha: str, *, process_id: int | None = None) -> str:
if not SHA256_RE.fullmatch(desired_sha):
raise StageFailure("candidate_stack", "desired SHA is invalid")
pid = os.getpid() if process_id is None else process_id
if pid <= 0:
raise StageFailure("candidate_stack", "process ID is invalid")
return f"vignette-release-gate-{desired_sha[:12]}-p{pid}"
def validate_milestone(milestone: MaterialMilestone | None) -> None:
if milestone is None:
raise StageFailure("material_milestone", "material milestone evidence is required")
if not milestone.material:
raise StageFailure("material_milestone", "milestone is not material")
if not milestone.milestone_id.strip():
raise StageFailure("material_milestone", "milestone_id is empty")
allowed_goals = {f"G{number}" for number in range(9)}
if not milestone.goals or not set(milestone.goals) <= allowed_goals:
raise StageFailure("material_milestone", "goals must be a non-empty G0-G8 subset")
if not milestone.evidence_refs or any(not value.strip() for value in milestone.evidence_refs):
raise StageFailure("material_milestone", "evidence_refs must be non-empty")
def load_json_object(path: Path, stage: str) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise StageFailure(stage, f"cannot read {path}: {exc}") from exc
if not isinstance(value, dict):
raise StageFailure(stage, f"JSON root must be an object: {path}")
return value
def parse_json_stdout(result: CommandResult, stage: str) -> dict[str, Any]:
try:
value = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise StageFailure(stage, f"command did not emit JSON: {exc}") from exc
if not isinstance(value, dict) or value.get("ok") is not True:
raise StageFailure(stage, f"command reported failure: {value!r}")
return value
class SubprocessRunner:
"""Run one deterministic stage without leaking full command output."""
def run(
self,
stage: str,
argv: list[str],
*,
cwd: Path | None = None,
env: dict[str, str] | None = None,
timeout: int | None = None,
allowed_returncodes: tuple[int, ...] = (0,),
) -> CommandResult:
try:
completed = subprocess.run(
argv,
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
timeout=timeout,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise StageFailure(stage, str(exc)) from exc
stdout = completed.stdout.decode("utf-8", errors="replace")
stderr = completed.stderr.decode("utf-8", errors="replace")
if completed.returncode not in allowed_returncodes:
raise StageFailure(
stage,
f"exit={completed.returncode}; stdout={stdout[-1600:]}; "
f"stderr={stderr[-2400:]}",
)
return CommandResult(completed.returncode, stdout, stderr)
class CleanCandidateManager:
"""Materialize canonical HEAD and apply only the verified release patch."""
def __init__(self, repo_root: Path, runner: Runner):
self.repo_root = repo_root
self.runner = runner
self._temp: tempfile.TemporaryDirectory[str] | None = None
def materialize(self, base_commit: str, patch_path: str) -> Path:
self._temp = tempfile.TemporaryDirectory(prefix="vignette-release-agent-")
root = Path(self._temp.name) / "candidate"
root.mkdir(parents=True)
archive = Path(self._temp.name) / "base.tar"
self.runner.run(
"candidate_archive",
["git", "archive", "--format=tar", "--output", str(archive), base_commit],
cwd=self.repo_root,
timeout=120,
)
with tarfile.open(archive, "r:") as tar:
_safe_extract(tar, root)
self.runner.run(
"candidate_patch_apply",
["git", "apply", "--whitespace=nowarn", str(self.repo_root / patch_path)],
cwd=root,
timeout=120,
)
generated_api = root / "apps/web/src/lib/api.gen.ts"
if not generated_api.is_file():
raise StageFailure("candidate_patch_apply", "generated API contract is missing")
_normalize_candidate_file_to_lf(generated_api, "generated API contract")
for shell_script in sorted(root.rglob("*.sh")):
_normalize_candidate_file_to_lf(shell_script, "shell script")
return root
def materialize_archive(self, archive_path: Path, expected_sha256: str) -> Path:
"""Extract an exact clean-HEAD archive without applying a patch."""
if not SHA256_RE.fullmatch(expected_sha256):
raise StageFailure("candidate_archive_binding", "expected archive SHA is invalid")
try:
actual_sha256, _ = sha256_file(archive_path)
except OSError as exc:
raise StageFailure("candidate_archive_binding", str(exc)) from exc
if actual_sha256 != expected_sha256:
raise StageFailure(
"candidate_archive_binding",
f"archive SHA drifted: expected={expected_sha256} actual={actual_sha256}",
)
self._temp = tempfile.TemporaryDirectory(prefix="vignette-release-agent-")
root = Path(self._temp.name) / "candidate"
try:
root.mkdir(parents=True)
with tarfile.open(archive_path, "r:") as archive:
_safe_extract(archive, root)
generated_api = root / "apps/web/src/lib/api.gen.ts"
if not generated_api.is_file():
raise StageFailure("candidate_archive", "generated API contract is missing")
# git archive stores canonical LF blobs, while a Windows checkout can
# materialize this generated file with CRLF. openapi-typescript
# --check compares bytes, so apply the same platform normalization as
# patch-mode after the archive SHA has already been verified.
_normalize_candidate_file_to_lf(generated_api, "generated API contract")
for shell_script in sorted(root.rglob("*.sh")):
_normalize_candidate_file_to_lf(shell_script, "shell script")
except StageFailure:
self.cleanup()
raise
except (OSError, tarfile.TarError) as exc:
self.cleanup()
raise StageFailure("candidate_archive", str(exc)) from exc
return root
def cleanup(self) -> None:
if self._temp is not None:
self._temp.cleanup()
self._temp = None
def _normalize_candidate_file_to_lf(path: Path, label: str) -> None:
raw = path.read_bytes()
normalized = raw.replace(b"\r\n", b"\n")
if b"\r" in normalized:
raise StageFailure(
"candidate_patch_apply",
f"{label} contains unsupported carriage returns: {path}",
)
if normalized != raw:
path.write_bytes(normalized)
def _safe_extract(archive: tarfile.TarFile, destination: Path) -> None:
destination_resolved = destination.resolve()
for member in archive.getmembers():
target = (destination / member.name).resolve()
if target != destination_resolved and destination_resolved not in target.parents:
raise StageFailure("candidate_archive", f"archive path escapes candidate: {member.name}")
archive.extractall(destination, filter="data")
class HttpRuntimeProbe:
"""Verify semantic health, unauthenticated auth, and G0-G8 OpenAPI surface."""
def __init__(self, base_url: str, *, timeout_seconds: float = 15.0):
self.base_url = base_url.rstrip("/")
self.timeout_seconds = timeout_seconds
def _json(self, path: str, expected_status: int = 200) -> tuple[int, Any]:
request = urllib.request.Request(
self.base_url + path,
headers={"accept": "application/json", "user-agent": "vignette-release-agent/1"},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
status = response.status
body = response.read()
except urllib.error.HTTPError as exc:
status = exc.code
body = exc.read()
except (OSError, urllib.error.URLError) as exc:
raise StageFailure("runtime_contract", f"{path} unavailable: {exc}") from exc
if status != expected_status:
raise StageFailure(
"runtime_contract",
f"{path} status={status}, expected={expected_status}",
)
if not body:
return status, None
try:
return status, json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise StageFailure("runtime_contract", f"{path} returned invalid JSON") from exc
def verify(self) -> dict[str, Any]:
_, health = self._json("/api/health")
if not isinstance(health, dict) or not (
health.get("status") == "ok"
and health.get("db") is True
and health.get("engine") is True
):
raise StageFailure("runtime_contract", f"semantic health failed: {health!r}")
auth_status, _ = self._json("/api/auth/me", expected_status=401)
_, openapi = self._json("/api/openapi.json")
if not isinstance(openapi, dict) or not isinstance(openapi.get("paths"), dict):
raise StageFailure("runtime_contract", "OpenAPI paths object is missing")
paths = openapi["paths"]
missing = {
goal: path for goal, path in REQUIRED_OPENAPI_PATHS.items() if path not in paths
}
schemas = openapi.get("components", {}).get("schemas", {})
if "AllianceScores" not in schemas:
missing["G0"] = "components.schemas.AllianceScores"
if missing:
raise StageFailure("runtime_contract", f"OpenAPI G0-G8 coverage missing: {missing}")
return {
"health": {
"status": health.get("status"),
"environment": health.get("environment"),
"db": health.get("db"),
"engine": health.get("engine"),
"engine_mode": health.get("engine_mode"),
},
"auth_status": auth_status,
"openapi_paths": len(paths),
"goals": [f"G{number}" for number in range(9)],
}
class NasPreviewDeploymentDriver:
"""Promote API/Web only inside the dedicated Compose project.
No command in this driver removes a volume, network, project, or the legacy
root. Candidate releases are immutable subdirectories. Rollback recreates
only api/web/proxy with the snapshotted image IDs and prior Compose file.
"""
def __init__(
self,
runner: Runner,
*,
ssh_target: str,
target: DeploymentTarget = NAS_PREVIEW_TARGET,
):
validate_target(target)
if not SSH_TARGET_RE.fullmatch(ssh_target):
raise ValueError("ssh target must be [user@]100.116.83.60")
self.runner = runner
self.ssh_target = ssh_target
self.target = target
self._active_state: dict[str, Any] | None = None
@property
def _container_prefix(self) -> str:
return self.target.compose_project
def _remote(
self,
stage: str,
script: str,
*,
allowed_returncodes: tuple[int, ...] = (0,),
timeout: int = 120,
) -> CommandResult:
command = "sh -lc " + shlex.quote(script)
return self.runner.run(
stage,
[
"ssh.exe",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
self.ssh_target,
command,
],
timeout=timeout,
allowed_returncodes=allowed_returncodes,
)
def _scp(self, stage: str, source: Path, destination: str) -> None:
if not destination.startswith(self.target.remote_root.rstrip("/") + "/"):
raise StageFailure(stage, "remote copy destination escapes isolated preview root")
self.runner.run(
stage,
[
"scp.exe",
"-O",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
str(source),
f"{self.ssh_target}:{destination}",
],
timeout=600,
)
def read_active_state(self) -> dict[str, Any] | None:
state_path = f"{self.target.remote_root}/release-agent-active.json"
result = self._remote(
"nas_read_active_state",
f"test -f {shlex.quote(state_path)} || exit 3; cat {shlex.quote(state_path)}",
allowed_returncodes=(0, 3),
)
if result.returncode == 3:
self._active_state = None
return None
try:
payload = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise StageFailure("nas_read_active_state", "remote state is invalid JSON") from exc
if not isinstance(payload, dict) or payload.get("schema") != "vignette.release-agent-active.v1":
raise StageFailure("nas_read_active_state", "remote state schema is invalid")
active_sha = payload.get("active_sha")
if not isinstance(active_sha, str) or not SHA256_RE.fullmatch(active_sha):
raise StageFailure("nas_read_active_state", "remote active_sha is invalid")
if payload.get("compose_project") != self.target.compose_project:
raise StageFailure("nas_read_active_state", "remote Compose project is invalid")
if payload.get("remote_root") != self.target.remote_root:
raise StageFailure("nas_read_active_state", "remote preview root is invalid")
validate_remote_compose_path(
payload.get("compose_file"),
self.target,
stage="nas_read_active_state",
)
env_file = payload.get("env_file")
if env_file is None:
compose_file = PurePosixPath(payload["compose_file"])
env_file = str(compose_file.parent / ".env")
env_file = validate_remote_env_file(
env_file,
self.target,
stage="nas_read_active_state",
)
for key in ("api_image", "web_image"):
image = payload.get(key)
if not isinstance(image, str) or not IMAGE_ID_RE.fullmatch(image):
raise StageFailure("nas_read_active_state", f"remote {key} is invalid")
normalized_payload = dict(payload)
normalized_payload["env_file"] = env_file
self._active_state = normalized_payload
return normalized_payload
def snapshot(self) -> DeploymentSnapshot:
root = self.target.remote_root
project = self.target.compose_project
previous_compose = (
self._active_state.get("compose_file")
if isinstance(self._active_state, dict)
else f"{root}/infra/docker-compose.yml"
)
previous_compose = validate_remote_compose_path(
previous_compose,
self.target,
stage="nas_snapshot",
)
previous_env = (
self._active_state.get("env_file")
if isinstance(self._active_state, dict)
else str(PurePosixPath(previous_compose).parent / ".env")
)
previous_env = validate_remote_env_file(
previous_env,
self.target,
stage="nas_snapshot",
)
api_container = f"{self._container_prefix}-api-1"
web_container = f"{self._container_prefix}-web-1"
script = "\n".join(
[
"set -eu",
f"test -f {shlex.quote(previous_compose)}",
f"test -f {shlex.quote(previous_env)}",
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(api_container)})",
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(web_container)})",
f"ap=$(docker inspect --format '{{{{index .Config.Labels \"com.docker.compose.project\"}}}}' {shlex.quote(api_container)})",
f"wp=$(docker inspect --format '{{{{index .Config.Labels \"com.docker.compose.project\"}}}}' {shlex.quote(web_container)})",
f"test \"$ap\" = {shlex.quote(project)}",
f"test \"$wp\" = {shlex.quote(project)}",
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
]
)
result = self._remote("nas_snapshot", script)
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
if len(lines) != 2 or any(not IMAGE_ID_RE.fullmatch(line) for line in lines):
raise StageFailure("nas_snapshot", f"invalid image snapshot: {lines!r}")
active_sha = None
if isinstance(self._active_state, dict):
value = self._active_state.get("active_sha")
if isinstance(value, str):
active_sha = value
return DeploymentSnapshot(
lines[0],
lines[1],
previous_compose,
previous_env,
active_sha,
)
def promote(
self,
candidate: Path,
desired_sha: str,
snapshot: DeploymentSnapshot,
) -> dict[str, Any]:
if not SHA256_RE.fullmatch(desired_sha):
raise StageFailure("nas_promote", "desired SHA is invalid")
root = self.target.remote_root
project = self.target.compose_project
run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + f"-{os.getpid()}"
release_key = f"{desired_sha[:16]}-{run_id}"
release_root = f"{root}/releases/{release_key}"
incoming = f"{root}/incoming"
env_file = validate_remote_env_file(
snapshot.env_file,
self.target,
stage="nas_promote",
)
with tempfile.TemporaryDirectory(prefix="vignette-nas-promotion-") as temp_name:
temp = Path(temp_name)
archive = temp / f"{release_key}.tar.gz"
_archive_candidate(candidate, archive)
override = temp / "release-agent.override.yml"
override.write_text(
"services:\n"
" api:\n"
f" image: {project}-api:{desired_sha[:16]}\n"
" web:\n"
f" image: {project}-web:{desired_sha[:16]}\n",
encoding="utf-8",
newline="\n",
)
self._remote(
"nas_prepare_release",
"\n".join(
[
"set -eu",
f"test -d {shlex.quote(root)}",
f"if [ ! -f {shlex.quote(env_file)} ]; then printf '%s\\n' {shlex.quote('required env file missing: ' + env_file)} >&2; exit 4; fi",
f"mkdir -p {shlex.quote(incoming)} {shlex.quote(root + '/releases')}",
f"test ! -e {shlex.quote(release_root)}",
]
),
)
self._scp("nas_upload_archive", archive, f"{incoming}/{archive.name}")
self._scp("nas_upload_override", override, f"{incoming}/{release_key}.override.yml")
compose = f"{release_root}/infra/docker-compose.yml"
remote_override = f"{release_root}/infra/release-agent.override.yml"
migration_commands: list[str] = []
for migration_name in RELEASE_DB_MIGRATIONS:
migration_path = f"{release_root}/infra/db/init/{migration_name}"
migration_commands.extend(
[
f"test -f {shlex.quote(migration_path)}",
(
f"docker compose -p {shlex.quote(project)} "
f"--env-file {shlex.quote(env_file)} "
f"-f {shlex.quote(compose)} -f {shlex.quote(remote_override)} "
"exec -T db sh -lc "
+ shlex.quote(
'psql -v ON_ERROR_STOP=1 --single-transaction '
'-U "$POSTGRES_USER" -d "$POSTGRES_DB"'
)
+ f" < {shlex.quote(migration_path)}"
),
]
)
script = "\n".join(
[
"set -eu",
f"mkdir {shlex.quote(release_root)}",
f"tar -xzf {shlex.quote(incoming + '/' + archive.name)} -C {shlex.quote(release_root)}",
f"mv {shlex.quote(incoming + '/' + release_key + '.override.yml')} {shlex.quote(remote_override)}",
f"test -f {shlex.quote(env_file)}",
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(compose)} -f {shlex.quote(remote_override)} build api web",
*migration_commands,
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(compose)} -f {shlex.quote(remote_override)} up -d --no-deps --force-recreate api web proxy",
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-api-1')})",
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-web-1')})",
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
]
)
result = self._remote("nas_promote", script, timeout=1800)
images = [line.strip() for line in result.stdout.splitlines() if IMAGE_ID_RE.fullmatch(line.strip())]
if len(images) < 2:
raise StageFailure("nas_promote", "promoted image IDs were not returned")
return {
"schema": "vignette.release-agent-active.v1",
"active_sha": desired_sha,
"compose_project": project,
"remote_root": root,
"release_root": release_root,
"compose_file": compose,
"env_file": env_file,
"api_image": images[-2],
"web_image": images[-1],
"previous_api_image": snapshot.api_image,
"previous_web_image": snapshot.web_image,
"deployed_at": utc_now(),
"database_migrations": list(RELEASE_DB_MIGRATIONS),
}
def commit_active_state(self, state: dict[str, Any]) -> None:
desired_sha = state.get("active_sha")
if not isinstance(desired_sha, str) or not SHA256_RE.fullmatch(desired_sha):
raise StageFailure("nas_commit_state", "active state SHA is invalid")
root = self.target.remote_root
with tempfile.TemporaryDirectory(prefix="vignette-nas-state-") as temp_name:
state_file = Path(temp_name) / f"{desired_sha}.json"
state_file.write_text(
json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
newline="\n",
)
remote_temp = f"{root}/incoming/{desired_sha}.state.json"
self._scp("nas_upload_state", state_file, remote_temp)
self._remote(
"nas_commit_state",
f"set -eu; mv {shlex.quote(remote_temp)} {shlex.quote(root + '/release-agent-active.json')}",
)
self._active_state = state
def rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]:
for image in (snapshot.api_image, snapshot.web_image):
if not IMAGE_ID_RE.fullmatch(image):
raise StageFailure("nas_rollback", "snapshot image ID is invalid")
root = self.target.remote_root
validate_remote_compose_path(
snapshot.compose_file,
self.target,
stage="nas_rollback",
)
if snapshot.active_sha is None or not SHA256_RE.fullmatch(snapshot.active_sha):
raise StageFailure("nas_rollback", "snapshot active SHA is missing or invalid")
project = self.target.compose_project
env_file = validate_remote_env_file(
snapshot.env_file,
self.target,
stage="nas_rollback",
)
self._remote(
"nas_rollback_preflight",
"\n".join(
[
"set -eu",
f"if [ ! -f {shlex.quote(env_file)} ]; then printf '%s\\n' {shlex.quote('required env file missing: ' + env_file)} >&2; exit 4; fi",
f"test -f {shlex.quote(snapshot.compose_file)}",
]
),
)
with tempfile.TemporaryDirectory(prefix="vignette-nas-rollback-") as temp_name:
temp = Path(temp_name)
override = temp / "rollback.override.yml"
override.write_text(
"services:\n"
" api:\n"
f" image: {snapshot.api_image}\n"
" web:\n"
f" image: {snapshot.web_image}\n",
encoding="utf-8",
newline="\n",
)
restored_state = temp / "restored-state.json"
restored_state.write_text(
json.dumps(
{
"schema": "vignette.release-agent-active.v1",
"active_sha": snapshot.active_sha,
"compose_project": project,
"remote_root": root,
"compose_file": snapshot.compose_file,
"env_file": env_file,
"api_image": snapshot.api_image,
"web_image": snapshot.web_image,
"rollback_restored_at": utc_now(),
},
ensure_ascii=False,
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
newline="\n",
)
remote_override = f"{root}/incoming/rollback-{os.getpid()}.override.yml"
remote_state = f"{root}/incoming/rollback-{os.getpid()}.state.json"
self._scp("nas_upload_rollback", override, remote_override)
self._scp("nas_upload_rollback_state", restored_state, remote_state)
self._remote(
"nas_rollback",
"\n".join(
[
"set -eu",
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(snapshot.compose_file)} -f {shlex.quote(remote_override)} up -d --no-build --no-deps --force-recreate api web proxy",
f"mv {shlex.quote(remote_state)} {shlex.quote(root + '/release-agent-active.json')}",
f"rm -f {shlex.quote(remote_override)}",
]
),
timeout=600,
)
self._active_state = {
"schema": "vignette.release-agent-active.v1",
"active_sha": snapshot.active_sha,
"compose_file": snapshot.compose_file,
"env_file": env_file,
"api_image": snapshot.api_image,
"web_image": snapshot.web_image,
}
return {"api_image": snapshot.api_image, "web_image": snapshot.web_image}
def verify_rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]:
project = self.target.compose_project
result = self._remote(
"nas_verify_rollback_images",
"\n".join(
[
"set -eu",
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-api-1')})",
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-web-1')})",
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
]
),
)
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
ok = lines == [snapshot.api_image, snapshot.web_image]
return {
"ok": ok,
"api_image": lines[0] if lines else None,
"web_image": lines[1] if len(lines) > 1 else None,
}
def _archive_candidate(candidate: Path, destination: Path) -> None:
excluded_parts = {
"node_modules",
".git",
".devlogs",
".release-agent",
"playwright-report",
}
with tarfile.open(destination, "w:gz", format=tarfile.PAX_FORMAT) as archive:
for path in sorted(candidate.rglob("*"), key=lambda value: value.as_posix()):
relative = path.relative_to(candidate)
if any(part in excluded_parts for part in relative.parts):
continue
if path.is_symlink():
continue
archive.add(path, arcname=relative.as_posix(), recursive=False)
class ReleaseAgent:
def __init__(
self,
config: ReleaseAgentConfig,
*,
runner: Runner,
deployment: DeploymentDriver,
runtime_probe: RuntimeProbe,
candidate_manager: CandidateManager,
):
self.config = config
self.runner = runner
self.deployment = deployment
self.runtime_probe = runtime_probe
self.candidate_manager = candidate_manager
self.stage_evidence: list[dict[str, Any]] = []
self.source_evidence: dict[str, Any] | None = None
self._source_temp: tempfile.TemporaryDirectory[str] | None = None
def _run(
self,
stage: str,
argv: list[str],
*,
cwd: Path | None = None,
env: dict[str, str] | None = None,
timeout: int | None = None,
) -> CommandResult:
started = time.monotonic()
result = self.runner.run(stage, argv, cwd=cwd, env=env, timeout=timeout)
self.stage_evidence.append(
{
"stage": stage,
"status": "passed",
"duration_ms": int((time.monotonic() - started) * 1000),
"stdout_sha256": sha256_bytes(result.stdout.encode("utf-8")),
"stderr_sha256": sha256_bytes(result.stderr.encode("utf-8")),
}
)
return result
@staticmethod
def _is_transient_runtime_failure(exc: StageFailure) -> bool:
if exc.stage != "runtime_contract":
return False
detail = exc.detail.lower()
return any(
marker in detail
for marker in (
"unavailable:",
"status=502",
"status=503",
"status=504",
"semantic health failed",
)
)
def _verify_runtime_ready(self) -> dict[str, Any]:
deadline = time.monotonic() + self.config.runtime_ready_timeout_seconds
attempts = 0
while True:
attempts += 1
try:
report = dict(self.runtime_probe.verify())
report["readiness_attempts"] = attempts
return report
except StageFailure as exc:
if (
not self._is_transient_runtime_failure(exc)
or time.monotonic() >= deadline
):
raise
time.sleep(self.config.runtime_retry_interval_seconds)
def _build_release_twice(self) -> dict[str, Any]:
reports: list[dict[str, Any]] = []
for run_number in (1, 2):
stage = f"release_patch_run_{run_number}"
result = self._run(
stage,
[
sys.executable,
"-X",
"utf8",
str(BUILDER if self.config.repo_root == REPO_ROOT else self.config.repo_root / "scripts/build-outcome-os-release-patch.py"),
"--map",
str(self.config.hunk_map_path),
"--json",
],
cwd=self.config.repo_root,
timeout=900,
)
reports.append(parse_json_stdout(result, stage))
required = {
"patch_sha256",
"patch_bytes",
"patch_files",
"base_commit",
"output_patch",
"git_apply_check",
"git_apply_cached_check",
"patch_line_endings",
"source_worktree",
"api_generation",
}
for report in reports:
missing = required - report.keys()
if missing:
raise StageFailure("release_determinism", f"builder report missing {sorted(missing)}")
if report["git_apply_cached_check"] != "passed-on-canonical-clean-index":
raise StageFailure("release_determinism", "canonical clean-index proof is missing")
if report["git_apply_check"] != "passed-on-clean-temporary-head":
raise StageFailure("release_determinism", "clean HEAD apply proof is missing")
comparable = (
"patch_sha256",
"patch_bytes",
"patch_files",
"base_commit",
"output_patch",
"git_apply_check",
"git_apply_cached_check",
"patch_line_endings",
"source_worktree",
"api_generation",
)
drift = {
key: [reports[0].get(key), reports[1].get(key)]
for key in comparable
if reports[0].get(key) != reports[1].get(key)
}
if drift:
raise StageFailure("release_determinism", f"two release builds differ: {drift}")
sha = reports[1]["patch_sha256"]
if not isinstance(sha, str) or not SHA256_RE.fullmatch(sha):
raise StageFailure("release_determinism", "builder patch SHA is invalid")
return reports[1]
def _clean_head_identity(self, suffix: str) -> tuple[str, str]:
status = self._run(
f"clean_head_worktree_{suffix}",
["git", "status", "--porcelain=v1", "--untracked-files=no"],
cwd=self.config.repo_root,
timeout=120,
)
if status.stdout.strip():
raise StageFailure(
"clean_head_worktree",
"tracked worktree/index changes are forbidden in clean-head mode",
)
head_result = self._run(
f"clean_head_head_{suffix}",
["git", "rev-parse", "--verify", "HEAD"],
cwd=self.config.repo_root,
timeout=120,
)
tree_result = self._run(
f"clean_head_tree_{suffix}",
["git", "rev-parse", "--verify", "HEAD^{tree}"],
cwd=self.config.repo_root,
timeout=120,
)
head = head_result.stdout.strip()
tree = tree_result.stdout.strip()
if not GIT_OBJECT_RE.fullmatch(head):
raise StageFailure("clean_head_identity", "Git HEAD is invalid")
if not GIT_OBJECT_RE.fullmatch(tree):
raise StageFailure("clean_head_identity", "Git tree is invalid")
return head, tree
def _validate_clean_head_worktree_root(self) -> None:
result = self._run(
"clean_head_repository",
["git", "rev-parse", "--show-toplevel"],
cwd=self.config.repo_root,
timeout=120,
)
reported_root = result.stdout.strip()
if not reported_root:
raise StageFailure("clean_head_repository", "Git worktree root is missing")
try:
actual_root = Path(reported_root).resolve(strict=True)
expected_root = self.config.repo_root.resolve(strict=True)
except OSError as exc:
raise StageFailure("clean_head_repository", str(exc)) from exc
if actual_root != expected_root:
raise StageFailure(
"clean_head_repository",
f"source repository is not the Git worktree root: {actual_root}",
)
def _build_clean_head_source_twice(self) -> dict[str, Any]:
self._validate_clean_head_worktree_root()
before_head, before_tree = self._clean_head_identity("before")
self._source_temp = tempfile.TemporaryDirectory(prefix="vignette-release-source-")
source_root = Path(self._source_temp.name)
archives: list[Path] = []
archive_reports: list[dict[str, Any]] = []
for run_number in (1, 2):
archive_path = source_root / f"clean-head-{run_number}.tar"
self._run(
f"clean_head_archive_run_{run_number}",
[
"git",
"archive",
"--format=tar",
"--output",
str(archive_path),
before_head,
],
cwd=self.config.repo_root,
timeout=900,
)
try:
archive_sha256, archive_size = sha256_file(archive_path)
except OSError as exc:
raise StageFailure("clean_head_archive", str(exc)) from exc
archives.append(archive_path)
archive_reports.append(
{
"sha256": archive_sha256,
"bytes": archive_size,
}
)
after_head, after_tree = self._clean_head_identity("after")
if (before_head, before_tree) != (after_head, after_tree):
raise StageFailure(
"clean_head_identity",
"Git HEAD/tree changed while clean-head archives were built",
)
if archive_reports[0] != archive_reports[1]:
raise StageFailure(
"clean_head_determinism",
f"two clean-head archives differ: {archive_reports}",
)
archive_sha256 = archive_reports[1]["sha256"]
if not SHA256_RE.fullmatch(archive_sha256):
raise StageFailure("clean_head_determinism", "archive SHA is invalid")
return {
"source_mode": "clean-head",
"head": after_head,
"tree": after_tree,
"archive_sha256": archive_sha256,
"archive_bytes": archive_reports[1]["bytes"],
"archive_path": archives[1],
"deterministic_runs": 2,
}
def _prepare_source(self) -> tuple[dict[str, Any], str]:
if self.config.source_mode == "patch":
release = self._build_release_twice()
self._check_manifest_binding(release)
self.source_evidence = {
"repo_root": str(self.config.repo_root),
"base_commit": release["base_commit"],
"patch": {
"sha256": release["patch_sha256"],
"bytes": release["patch_bytes"],
"files": release["patch_files"],
"output": release["output_patch"],
"deterministic_runs": 2,
},
}
return release, release["patch_sha256"]
if self.config.source_mode == "clean-head":
release = self._build_clean_head_source_twice()
self.source_evidence = {
"repo_root": str(self.config.repo_root),
"head": release["head"],
"tree": release["tree"],
"archive": {
"sha256": release["archive_sha256"],
"bytes": release["archive_bytes"],
"deterministic_runs": release["deterministic_runs"],
},
}
return release, release["archive_sha256"]
raise StageFailure(
"configuration",
f"unsupported source mode: {self.config.source_mode!r}",
)
def _check_manifest_binding(self, release: dict[str, Any]) -> None:
result = self._run(
"release_manifest",
[
sys.executable,
"-X",
"utf8",
str(MANIFEST_CHECKER if self.config.repo_root == REPO_ROOT else self.config.repo_root / "scripts/check-outcome-os-release-manifest.py"),
"--manifest",
str(self.config.manifest_path),
"--json",
],
cwd=self.config.repo_root,
timeout=300,
)
parse_json_stdout(result, "release_manifest")
manifest = load_json_object(self.config.manifest_path, "release_manifest_binding")
validate_required_release_payload(manifest)
assembly = manifest.get("release_assembly")
if not isinstance(assembly, dict):
raise StageFailure("release_manifest_binding", "release_assembly is missing")
if assembly.get("patch_sha256") != release["patch_sha256"]:
raise StageFailure("release_manifest_binding", "manifest is not bound to current patch SHA")
if assembly.get("output_patch") != release["output_patch"]:
raise StageFailure("release_manifest_binding", "manifest output patch path drifted")
def _validate_snapshot_binding(
self,
active_state: dict[str, Any],
snapshot: DeploymentSnapshot,
) -> None:
expected = {
"active_sha": snapshot.active_sha,
"compose_file": snapshot.compose_file,
"env_file": snapshot.env_file,
"api_image": snapshot.api_image,
"web_image": snapshot.web_image,
}
drift = {
key: {"tracked": active_state.get(key), "runtime": value}
for key, value in expected.items()
if active_state.get(key) != value
}
if drift:
raise StageFailure(
"deployment_state",
f"tracked active state does not match runtime snapshot: {drift}",
)
def _run_candidate_gates(self, candidate: Path, desired_sha: str) -> None:
python = sys.executable
candidate_preflight = candidate / "scripts/check-deploy-preflight.py"
self._run(
"nas_preflight",
[
python,
"-X",
"utf8",
str(candidate_preflight),
"--env-file",
str(self.config.nas_env_file),
"--deployment-profile",
"compose",
"--skip-db",
],
cwd=candidate,
timeout=180,
)
self._run(
"api_tests",
[python, "-X", "utf8", "-m", "pytest", "-p", "no:cacheprovider", "-q"],
cwd=candidate / "apps/api",
timeout=1800,
)
web = candidate / "apps/web"
self._run("web_install", ["npm.cmd", "ci", "--ignore-scripts"], cwd=web, timeout=900)
self._run("web_api_contract", ["npm.cmd", "run", "check:api-types"], cwd=web, timeout=300)
self._run("web_typecheck", ["npm.cmd", "run", "typecheck"], cwd=web, timeout=300)
self._run("web_build", ["npm.cmd", "run", "build"], cwd=web, timeout=600)
# This spec imports /src/lib/uuid.ts and therefore must run against a
# source Vite server, not the production Compose build (which correctly
# serves the SPA document for /src/*). CI=1 prevents reuse of an
# unrelated long-lived Vite process on the workstation.
source_e2e_env = os.environ.copy()
source_e2e_env.pop("PLAYWRIGHT_BASE_URL", None)
source_e2e_env.pop("PLAYWRIGHT_SKIP_WEB_SERVER", None)
source_e2e_env["PLAYWRIGHT_HOST"] = "127.0.0.1"
source_e2e_env["PLAYWRIGHT_PORT"] = str(SOURCE_ONLY_E2E_PORT)
source_e2e_env["CI"] = "1"
self._run(
"source_insecure_context_e2e",
[
"node.exe",
str(web / "node_modules/@playwright/test/cli.js"),
"test",
*POSTDEPLOY_SOURCE_ONLY_E2E_SPECS,
"--project=chromium-desktop",
"--project=chromium-mobile",
"--workers=1",
"--reporter=line",
],
cwd=web,
env=source_e2e_env,
timeout=300,
)
stack_attempted = False
stack_env: Path | None = None
project = candidate_compose_project(desired_sha)
try:
if self.config.run_candidate_stack:
stack_env = self._candidate_stack_env(candidate)
stack_attempted = True
self._run(
"candidate_stack_up",
[
"docker.exe",
"compose",
"-p",
project,
"--env-file",
str(stack_env),
"-f",
str(candidate / "infra/docker-compose.yml"),
"up",
"-d",
"--build",
"--wait",
"--wait-timeout",
"600",
],
cwd=candidate,
timeout=1800,
)
HttpRuntimeProbe(self.config.candidate_base_url, timeout_seconds=20).verify()
e2e_env = os.environ.copy()
e2e_env["PLAYWRIGHT_BASE_URL"] = self.config.candidate_base_url
e2e_env["PLAYWRIGHT_SKIP_WEB_SERVER"] = "1"
self._run(
"session_e2e",
[
"node.exe",
str(web / "node_modules/@playwright/test/cli.js"),
"test",
*POSTDEPLOY_NAS_E2E_SPECS,
"--project=chromium-desktop",
"--project=chromium-mobile",
"--project=chromium-single-run",
"--workers=1",
"--reporter=line",
],
cwd=web,
env=e2e_env,
timeout=RELEASE_E2E_TIMEOUT_SECONDS,
)
finally:
if stack_attempted and stack_env is not None:
self._run(
"candidate_stack_down",
[
"docker.exe",
"compose",
"-p",
project,
"--env-file",
str(stack_env),
"-f",
str(candidate / "infra/docker-compose.yml"),
"down",
"--remove-orphans",
"--volumes",
],
cwd=candidate,
timeout=600,
)
def _candidate_stack_env(self, candidate: Path) -> Path:
try:
lines = self.config.nas_env_file.read_text(encoding="utf-8").splitlines()
except OSError as exc:
raise StageFailure("candidate_stack_env", str(exc)) from exc
overrides = {
"HTTP_PORT": str(self.config.candidate_http_port),
"HTTPS_PORT": str(self.config.candidate_https_port),
"FRONTEND_BASE_URL": self.config.candidate_base_url,
"FRONTEND_ORIGIN_MAP": json.dumps(
{"default": self.config.candidate_base_url}, separators=(",", ":")
),
"CORS_ORIGINS": json.dumps([self.config.candidate_base_url], separators=(",", ":")),
"AUTH_DEV_LOGIN_EXTRA_ORIGINS": json.dumps(
[self.config.candidate_base_url], separators=(",", ":")
),
"AUTH_DEV_LOGIN_ENABLED": "true",
"PUBLIC_API_BASE": "/api",
}
seen: set[str] = set()
output: list[str] = []
for line in lines:
if not line or line.lstrip().startswith("#") or "=" not in line:
output.append(line)
continue
key = line.split("=", 1)[0].strip()
if key in overrides:
output.append(f"{key}={overrides[key]}")
seen.add(key)
else:
output.append(line)
for key in sorted(overrides.keys() - seen):
output.append(f"{key}={overrides[key]}")
destination = candidate / ".release-agent/candidate.env"
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text("\n".join(output) + "\n", encoding="utf-8", newline="\n")
return destination
def _run_postdeploy_browser_review(self, candidate: Path) -> dict[str, Any]:
web = candidate / "apps/web"
env = os.environ.copy()
env["PLAYWRIGHT_BASE_URL"] = self.config.target.base_url
env["PLAYWRIGHT_SKIP_WEB_SERVER"] = "1"
result = self._run(
"postdeploy_browser_review",
[
"node.exe",
str(web / "node_modules/@playwright/test/cli.js"),
"test",
*POSTDEPLOY_NAS_E2E_SPECS,
*(f"--project={project}" for project in RELEASE_BROWSER_PROJECTS),
"--workers=1",
"--reporter=line",
],
cwd=web,
env=env,
timeout=RELEASE_E2E_TIMEOUT_SECONDS,
)
return {
"status": "passed",
"base_url": self.config.target.base_url,
"specs": list(POSTDEPLOY_NAS_E2E_SPECS),
"projects": list(RELEASE_BROWSER_PROJECTS),
"runtime_scope": {
"real_api_db_specs": list(POSTDEPLOY_NAS_REAL_API_E2E_SPECS),
"route_fixture_specs": list(POSTDEPLOY_NAS_ROUTE_FIXTURE_E2E_SPECS),
},
"source_only_candidate_specs": list(POSTDEPLOY_SOURCE_ONLY_E2E_SPECS),
"separate_disposable_db_specs": list(POSTDEPLOY_DISPOSABLE_DB_E2E_SPECS),
"separate_disposable_db_status": "not_run_by_release_agent",
"uuid_runtime_route_specs": [
"e2e/session-persistence.spec.ts",
"e2e/self-directed-learning-loop.spec.ts",
"e2e/rupture-repair.spec.ts",
"e2e/deliberate-practice.spec.ts",
"e2e/calibration-transfer.spec.ts",
],
"physical_microphone_used": False,
"stdout_sha256": sha256_bytes(result.stdout.encode("utf-8")),
}
def _base_report(self, desired_sha: str | None = None) -> dict[str, Any]:
return {
"schema": "vignette.outcome-os-release-agent-evidence.v1",
"captured_at": utc_now(),
"ok": False,
"mode": "execute" if self.config.execute else "dry-run",
"source_mode": self.config.source_mode,
"source": self.source_evidence,
"target": asdict(self.config.target),
"milestone": asdict(self.config.milestone) if self.config.milestone else None,
"desired_sha": desired_sha,
"stages": self.stage_evidence,
}
def _write_evidence(self, report: dict[str, Any]) -> None:
if self.config.evidence_out is None:
return
path = self.config.evidence_out
path.parent.mkdir(parents=True, exist_ok=True)
candidate = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
candidate.write_text(
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
newline="\n",
)
os.replace(candidate, path)
finally:
if candidate.exists():
candidate.unlink()
def run(self) -> dict[str, Any]:
candidate: Path | None = None
snapshot: DeploymentSnapshot | None = None
promotion_attempted = False
desired_sha: str | None = None
try:
validate_target(self.config.target)
validate_source_repo_root(self.config.repo_root)
validate_milestone(self.config.milestone)
release, desired_sha = self._prepare_source()
active_state = self.deployment.read_active_state()
active_sha = active_state.get("active_sha") if active_state else None
if active_sha is not None and (
not isinstance(active_sha, str) or not SHA256_RE.fullmatch(active_sha)
):
raise StageFailure("deployment_state", "active deployment SHA is invalid")
if active_sha == desired_sha:
snapshot = self.deployment.snapshot()
self._validate_snapshot_binding(active_state, snapshot)
snapshot = None
runtime = self.runtime_probe.verify()
report = self._base_report(desired_sha)
report.update(
{
"ok": True,
"status": "noop_same_sha",
"active_sha": active_sha,
"runtime": runtime,
"deployment_mutated": False,
}
)
self._write_evidence(report)
return report
if not self.config.execute:
report = self._base_report(desired_sha)
report.update(
{
"ok": True,
"status": "dry_run_changed_sha" if active_sha else "dry_run_untracked_state",
"active_sha": active_sha,
"deployment_mutated": False,
"would_promote": active_sha is not None and active_sha != desired_sha,
}
)
self._write_evidence(report)
return report
if active_sha is None:
raise StageFailure(
"deployment_state",
"remote release-agent state is untracked; adopt it only after an image/Compose rollback snapshot audit",
)
if self.config.source_mode == "clean-head":
candidate = self.candidate_manager.materialize_archive(
release["archive_path"], release["archive_sha256"]
)
else:
candidate = self.candidate_manager.materialize(
release["base_commit"], release["output_patch"]
)
self._run_candidate_gates(candidate, desired_sha)
snapshot = self.deployment.snapshot()
self._validate_snapshot_binding(active_state, snapshot)
promotion_attempted = True
promoted_state = self.deployment.promote(candidate, desired_sha, snapshot)
runtime = self._verify_runtime_ready()
browser_review = self._run_postdeploy_browser_review(candidate)
self.deployment.commit_active_state(promoted_state)
report = self._base_report(desired_sha)
report.update(
{
"ok": True,
"status": "deployed_ssot_sync_required",
"active_sha": desired_sha,
"previous_sha": active_sha,
"deployment": {
"desired_sha": desired_sha,
"previous_sha": active_sha,
"api_image": promoted_state.get("api_image"),
"web_image": promoted_state.get("web_image"),
"compose_project": self.config.target.compose_project,
},
"runtime": runtime,
"browser_review_proof": browser_review,
"deployment_mutated": True,
"ssot_sync": {
"status": "required",
"deployment_sha": desired_sha,
"required_paths": list(SSOT_REQUIRED_PATHS),
"required_anchor": desired_sha,
"checker_commands": [
"py -3.11 -X utf8 scripts/check-dev-dashboard-ssot.py --json",
"py -3.11 -X utf8 scripts/check-outcome-os-release-manifest.py --json",
],
},
}
)
self._write_evidence(report)
return report
except (StageFailure, ValueError, OSError) as exc:
stage = exc.stage if isinstance(exc, StageFailure) else "configuration"
rollback: dict[str, Any] = {"status": "not-required"}
if promotion_attempted and snapshot is not None:
rollback = {"status": "failed"}
try:
rollback_effect = self.deployment.rollback(snapshot)
restored_runtime = self._verify_runtime_ready()
rollback_proof = self.deployment.verify_rollback(snapshot)
if rollback_proof.get("ok") is not True:
raise StageFailure(
"rollback_proof",
f"rollback image proof mismatch: {rollback_proof}",
)
rollback = {
"status": "verified",
"effect": rollback_effect,
"runtime": restored_runtime,
"image_proof": rollback_proof,
}
except (StageFailure, ValueError, OSError) as rollback_exc:
rollback = {"status": "failed", "error": str(rollback_exc)}
report = self._base_report(desired_sha)
report.update(
{
"ok": False,
"status": "failed_closed",
"failed_stage": stage,
"error": str(exc),
"deployment_mutated": promotion_attempted,
"rollback": rollback,
}
)
self._write_evidence(report)
raise ReleaseAgentFailure(report) from exc
finally:
if candidate is not None:
self.candidate_manager.cleanup()
if self._source_temp is not None:
self._source_temp.cleanup()
self._source_temp = None
def load_milestone(path: Path) -> MaterialMilestone:
payload = load_json_object(path, "material_milestone")
if payload.get("schema") != "vignette.material-milestone.v1":
raise StageFailure("material_milestone", "milestone schema is invalid")
goals = payload.get("goals")
evidence_refs = payload.get("evidence_refs")
if not isinstance(goals, list) or any(not isinstance(value, str) for value in goals):
raise StageFailure("material_milestone", "goals must be strings")
if not isinstance(evidence_refs, list) or any(
not isinstance(value, str) for value in evidence_refs
):
raise StageFailure("material_milestone", "evidence_refs must be strings")
milestone_id = payload.get("milestone_id")
if not isinstance(milestone_id, str):
raise StageFailure("material_milestone", "milestone_id must be a string")
return MaterialMilestone(
milestone_id=milestone_id,
material=payload.get("material") is True,
goals=tuple(goals),
evidence_refs=tuple(evidence_refs),
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--milestone", type=Path, required=True)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--hunk-map", type=Path, default=DEFAULT_HUNK_MAP)
parser.add_argument(
"--source-mode",
choices=SOURCE_MODES,
default="patch",
help="Release source: verified patch (default) or exact clean Git HEAD archive",
)
parser.add_argument(
"--source-repo-root",
type=Path,
default=REPO_ROOT,
help="Absolute Git worktree root used as the release source",
)
parser.add_argument("--nas-env-file", type=Path, default=DEFAULT_NAS_ENV)
parser.add_argument("--evidence-out", type=Path)
parser.add_argument("--ssh-target", help="Required for live state inspection/execution")
parser.add_argument("--current-deployment-sha", help="Dry-run state override")
parser.add_argument(
"--execute",
action="store_true",
help="Promote only the dedicated NAS preview; default is read-only dry-run",
)
parser.add_argument(
"--run-candidate-stack",
action="store_true",
help="Start a uniquely named local Compose stack for pre-deploy session E2E",
)
parser.add_argument("--json", action="store_true")
return parser
class StaticDeploymentState:
"""Dry-run-only state provider; every mutating method is a hard failure."""
def __init__(self, active_sha: str | None):
if active_sha is not None and not SHA256_RE.fullmatch(active_sha):
raise ValueError("current deployment SHA must be 64 lowercase hex characters")
self.active_sha = active_sha
def read_active_state(self) -> dict[str, Any] | None:
if self.active_sha is None:
return None
return {"schema": "vignette.release-agent-active.v1", "active_sha": self.active_sha}
def _deny(self, *_: Any, **__: Any):
raise StageFailure("deployment_driver", "static dry-run driver cannot mutate")
snapshot = _deny
promote = _deny
commit_active_state = _deny
rollback = _deny
verify_rollback = _deny
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
milestone = load_milestone(args.milestone)
runner = SubprocessRunner()
if args.execute:
if not args.ssh_target:
raise StageFailure("configuration", "--execute requires --ssh-target")
deployment: DeploymentDriver = NasPreviewDeploymentDriver(
runner,
ssh_target=args.ssh_target,
)
elif args.ssh_target:
deployment = NasPreviewDeploymentDriver(runner, ssh_target=args.ssh_target)
else:
deployment = StaticDeploymentState(args.current_deployment_sha)
config = ReleaseAgentConfig(
repo_root=args.source_repo_root,
manifest_path=args.manifest,
hunk_map_path=args.hunk_map,
source_mode=args.source_mode,
execute=args.execute,
milestone=milestone,
target=NAS_PREVIEW_TARGET,
evidence_out=args.evidence_out,
nas_env_file=args.nas_env_file,
run_candidate_stack=args.run_candidate_stack,
)
agent = ReleaseAgent(
config,
runner=runner,
deployment=deployment,
runtime_probe=HttpRuntimeProbe(NAS_PREVIEW_TARGET.base_url),
candidate_manager=CleanCandidateManager(args.source_repo_root, runner),
)
report = agent.run()
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
else:
print(
f"PASS release agent: {report['status']} "
f"desired={report.get('desired_sha')} active={report.get('active_sha')}"
)
return 0
except ReleaseAgentFailure as exc:
if args.json:
print(json.dumps(exc.report, ensure_ascii=False, indent=2, sort_keys=True))
else:
print(str(exc), file=sys.stderr)
return 1
except (StageFailure, ValueError, OSError) as exc:
report = {"ok": False, "status": "failed_closed", "error": str(exc)}
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
else:
print(f"FAIL release agent: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
Reference in a new issue View git blame Copy permalink