8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
1605 lines
63 KiB
Python
1605 lines
63 KiB
Python
#!/usr/bin/env python3
|
|
"""Fail-closed G8 release agent for the isolated Vignette NAS preview.
|
|
|
|
The default is a read-only dry run. ``--execute`` is intentionally narrow:
|
|
it accepts only the dedicated Tailnet preview, builds a clean HEAD+patch
|
|
candidate, runs release/API/Web/session gates, snapshots the current API and
|
|
Web images, promotes only a changed deterministic SHA, and rolls back the
|
|
previous images when any post-promotion proof fails.
|
|
|
|
This program never runs ``docker compose down`` against the NAS, never removes
|
|
remote volumes or networks, and never targets another Compose project. A
|
|
successful changed-SHA promotion emits an SSOT evidence contract; a separate
|
|
agent must bind that exact SHA into the dashboard/TODO/backlog/manifest and run
|
|
their canonical checkers.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import asdict, dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, Protocol
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_MANIFEST = REPO_ROOT / "docs/ops/outcome-os-clean-release-manifest-2026-08-07.json"
|
|
DEFAULT_HUNK_MAP = REPO_ROOT / "docs/ops/outcome-os-release-hunk-map-2026-08-07.json"
|
|
DEFAULT_NAS_ENV = REPO_ROOT / "infra/.env.nas-preview"
|
|
BUILDER = REPO_ROOT / "scripts/build-outcome-os-release-patch.py"
|
|
MANIFEST_CHECKER = REPO_ROOT / "scripts/check-outcome-os-release-manifest.py"
|
|
PREFLIGHT = REPO_ROOT / "scripts/check-deploy-preflight.py"
|
|
|
|
SSOT_REQUIRED_PATHS = (
|
|
"docs/dev_dashboard.html",
|
|
"docs/TODO.md",
|
|
"docs/ops/backlog-2026-06-26.md",
|
|
"docs/ops/outcome-os-clean-release-manifest-2026-08-07.json",
|
|
)
|
|
|
|
# A promotion is allowed only after every Outcome OS goal's primary learner or
|
|
# operator surface has exercised its browser contract. Keep this list explicit
|
|
# so a newly added goal cannot disappear behind a broad directory glob.
|
|
RELEASE_UI_E2E_SPECS = (
|
|
"e2e/session-layout.spec.ts",
|
|
"e2e/session-persistence.spec.ts",
|
|
"e2e/self-directed-learning-loop.spec.ts",
|
|
"e2e/alliance-pulse.spec.ts",
|
|
"e2e/outcome-trajectory.spec.ts",
|
|
"e2e/rupture-repair.spec.ts",
|
|
"e2e/deliberate-practice.spec.ts",
|
|
"e2e/calibration-transfer.spec.ts",
|
|
"e2e/supervision-research.spec.ts",
|
|
"e2e/multimodal-alliance.spec.ts",
|
|
"e2e/continuous-improvement-admin.spec.ts",
|
|
)
|
|
RELEASE_E2E_TIMEOUT_SECONDS = 15 * 60
|
|
|
|
# Existing preview volumes do not replay docker-entrypoint-initdb.d. These
|
|
# forward-compatible, idempotent Outcome OS migrations must therefore land
|
|
# before the new API image is restarted. Every file is applied atomically.
|
|
RELEASE_DB_MIGRATIONS = (
|
|
"14_continuous_improvement.sql",
|
|
"15_self_directed_practice_runtime.sql",
|
|
"16_calibration_transfer_actual_execution.sql",
|
|
)
|
|
|
|
# These files are runtime-critical but were added after the first manifest
|
|
# snapshot. A release must classify them as whole-file payloads; otherwise a
|
|
# clean candidate can pass source tests while silently shipping the old mobile
|
|
# shell or omitting a migration that only fails after remote promotion starts.
|
|
REQUIRED_RELEASE_PAYLOAD_PATHS = (
|
|
"apps/web/src/components/shell/shell.css",
|
|
"infra/db/init/15_self_directed_practice_runtime.sql",
|
|
"infra/db/init/16_calibration_transfer_actual_execution.sql",
|
|
)
|
|
|
|
REQUIRED_OPENAPI_PATHS = {
|
|
"G1": "/sessions/{session_id}/alliance-pulses",
|
|
"G2": "/sessions/{session_id}/outcome-trajectory",
|
|
"G3": "/sessions/{session_id}/ruptures",
|
|
"G4": "/practice/learners/me",
|
|
"G5": "/calibration/learners/me",
|
|
"G6": "/supervision-research/supervision-view",
|
|
"G7": "/sessions/{session_id}/multimodal-alliance",
|
|
"G8": "/continuous-improvement",
|
|
}
|
|
|
|
IMAGE_ID_RE = re.compile(r"^sha256:[a-f0-9]{64}$")
|
|
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
SSH_TARGET_RE = re.compile(r"^(?:[A-Za-z0-9._-]+@)?100\.116\.83\.60$")
|
|
RELEASE_KEY_RE = re.compile(r"^[a-f0-9]{16}-\d{8}T\d{6}Z-\d+$")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeploymentTarget:
|
|
base_url: str
|
|
compose_project: str
|
|
remote_root: str
|
|
|
|
|
|
NAS_PREVIEW_TARGET = DeploymentTarget(
|
|
base_url="http://100.116.83.60:8088",
|
|
compose_project="vignette-preview-20260807",
|
|
remote_root="/volume1/docker/vignette-preview-20260807",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MaterialMilestone:
|
|
milestone_id: str
|
|
material: bool
|
|
goals: tuple[str, ...]
|
|
evidence_refs: tuple[str, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CommandResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeploymentSnapshot:
|
|
api_image: str
|
|
web_image: str
|
|
compose_file: str
|
|
env_file: str
|
|
active_sha: str | None
|
|
|
|
|
|
@dataclass
|
|
class ReleaseAgentConfig:
|
|
repo_root: Path = REPO_ROOT
|
|
manifest_path: Path = DEFAULT_MANIFEST
|
|
hunk_map_path: Path = DEFAULT_HUNK_MAP
|
|
execute: bool = False
|
|
milestone: MaterialMilestone | None = None
|
|
target: DeploymentTarget = NAS_PREVIEW_TARGET
|
|
evidence_out: Path | None = None
|
|
nas_env_file: Path = DEFAULT_NAS_ENV
|
|
candidate_base_url: str = "http://127.0.0.1:18088"
|
|
candidate_http_port: int = 18088
|
|
candidate_https_port: int = 18443
|
|
run_candidate_stack: bool = False
|
|
runtime_ready_timeout_seconds: float = 180.0
|
|
runtime_retry_interval_seconds: float = 3.0
|
|
|
|
|
|
class StageFailure(RuntimeError):
|
|
def __init__(self, stage: str, detail: str):
|
|
self.stage = stage
|
|
self.detail = detail
|
|
super().__init__(f"{stage}: {detail}")
|
|
|
|
|
|
class ReleaseAgentFailure(RuntimeError):
|
|
def __init__(self, report: dict[str, Any]):
|
|
self.report = report
|
|
super().__init__(
|
|
f"release agent failed at {report.get('failed_stage', 'unknown')}: "
|
|
f"{report.get('error', 'unknown error')}"
|
|
)
|
|
|
|
|
|
class Runner(Protocol):
|
|
def run(
|
|
self,
|
|
stage: str,
|
|
argv: list[str],
|
|
*,
|
|
cwd: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
timeout: int | None = None,
|
|
allowed_returncodes: tuple[int, ...] = (0,),
|
|
) -> CommandResult: ...
|
|
|
|
|
|
class DeploymentDriver(Protocol):
|
|
def read_active_state(self) -> dict[str, Any] | None: ...
|
|
|
|
def snapshot(self) -> DeploymentSnapshot: ...
|
|
|
|
def promote(
|
|
self,
|
|
candidate: Path,
|
|
desired_sha: str,
|
|
snapshot: DeploymentSnapshot,
|
|
) -> dict[str, Any]: ...
|
|
|
|
def commit_active_state(self, state: dict[str, Any]) -> None: ...
|
|
|
|
def rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]: ...
|
|
|
|
def verify_rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]: ...
|
|
|
|
|
|
class RuntimeProbe(Protocol):
|
|
def verify(self) -> dict[str, Any]: ...
|
|
|
|
|
|
class CandidateManager(Protocol):
|
|
def materialize(self, base_commit: str, patch_path: str) -> Path: ...
|
|
|
|
def cleanup(self) -> None: ...
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def validate_target(target: DeploymentTarget) -> None:
|
|
if target != NAS_PREVIEW_TARGET:
|
|
raise ValueError(
|
|
"only the isolated NAS preview target is allowed: "
|
|
f"{NAS_PREVIEW_TARGET!r}"
|
|
)
|
|
root = PurePosixPath(target.remote_root)
|
|
if not root.is_absolute() or ".." in root.parts:
|
|
raise ValueError("isolated NAS preview root must be an absolute canonical path")
|
|
|
|
|
|
def validate_required_release_payload(manifest: dict[str, Any]) -> None:
|
|
classifications = manifest.get("classifications")
|
|
related = classifications.get("related") if isinstance(classifications, dict) else None
|
|
if not isinstance(related, list):
|
|
raise StageFailure("release_manifest_binding", "related release classification is missing")
|
|
related_paths = {
|
|
path
|
|
for entry in related
|
|
if isinstance(entry, dict) and entry.get("disposition") == "candidate-whole-file"
|
|
for path in entry.get("paths", [])
|
|
if isinstance(path, str)
|
|
}
|
|
missing = sorted(set(REQUIRED_RELEASE_PAYLOAD_PATHS) - related_paths)
|
|
if missing:
|
|
raise StageFailure(
|
|
"release_manifest_binding",
|
|
f"runtime-critical release payload is not classified as related: {missing}",
|
|
)
|
|
|
|
|
|
def validate_remote_compose_path(
|
|
value: Any,
|
|
target: DeploymentTarget,
|
|
*,
|
|
stage: str,
|
|
) -> str:
|
|
if not isinstance(value, str):
|
|
raise StageFailure(stage, "remote Compose file is missing")
|
|
root = PurePosixPath(target.remote_root)
|
|
candidate = PurePosixPath(value)
|
|
if (
|
|
not candidate.is_absolute()
|
|
or ".." in candidate.parts
|
|
or str(candidate) != value
|
|
):
|
|
raise StageFailure(stage, "remote Compose file is not canonical")
|
|
try:
|
|
relative = candidate.relative_to(root)
|
|
except ValueError as exc:
|
|
raise StageFailure(stage, "remote Compose file escapes preview root") from exc
|
|
is_root_compose = relative.parts == ("infra", "docker-compose.yml")
|
|
is_legacy_release_compose = relative.parts == (
|
|
"release",
|
|
"infra",
|
|
"docker-compose.yml",
|
|
)
|
|
is_release_compose = (
|
|
len(relative.parts) == 4
|
|
and relative.parts[0] == "releases"
|
|
and RELEASE_KEY_RE.fullmatch(relative.parts[1]) is not None
|
|
and relative.parts[2:] == ("infra", "docker-compose.yml")
|
|
)
|
|
if not (is_root_compose or is_legacy_release_compose or is_release_compose):
|
|
raise StageFailure(stage, "remote Compose file is outside an approved release path")
|
|
return value
|
|
|
|
|
|
def validate_remote_env_file(
|
|
value: Any,
|
|
target: DeploymentTarget,
|
|
*,
|
|
stage: str,
|
|
) -> str:
|
|
if not isinstance(value, str):
|
|
raise StageFailure(stage, "remote env file is missing")
|
|
root = PurePosixPath(target.remote_root)
|
|
candidate = PurePosixPath(value)
|
|
if (
|
|
not candidate.is_absolute()
|
|
or ".." in candidate.parts
|
|
or str(candidate) != value
|
|
):
|
|
raise StageFailure(stage, "remote env file is not canonical")
|
|
try:
|
|
relative = candidate.relative_to(root)
|
|
except ValueError as exc:
|
|
raise StageFailure(stage, "remote env file escapes preview root") from exc
|
|
allowed = {
|
|
("infra", ".env"),
|
|
("release", "infra", ".env"),
|
|
}
|
|
if relative.parts not in allowed:
|
|
raise StageFailure(stage, "remote env file is outside the approved stable paths")
|
|
return value
|
|
|
|
|
|
def candidate_compose_project(desired_sha: str, *, process_id: int | None = None) -> str:
|
|
if not SHA256_RE.fullmatch(desired_sha):
|
|
raise StageFailure("candidate_stack", "desired SHA is invalid")
|
|
pid = os.getpid() if process_id is None else process_id
|
|
if pid <= 0:
|
|
raise StageFailure("candidate_stack", "process ID is invalid")
|
|
return f"vignette-release-gate-{desired_sha[:12]}-p{pid}"
|
|
|
|
|
|
def validate_milestone(milestone: MaterialMilestone | None) -> None:
|
|
if milestone is None:
|
|
raise StageFailure("material_milestone", "material milestone evidence is required")
|
|
if not milestone.material:
|
|
raise StageFailure("material_milestone", "milestone is not material")
|
|
if not milestone.milestone_id.strip():
|
|
raise StageFailure("material_milestone", "milestone_id is empty")
|
|
allowed_goals = {f"G{number}" for number in range(9)}
|
|
if not milestone.goals or not set(milestone.goals) <= allowed_goals:
|
|
raise StageFailure("material_milestone", "goals must be a non-empty G0-G8 subset")
|
|
if not milestone.evidence_refs or any(not value.strip() for value in milestone.evidence_refs):
|
|
raise StageFailure("material_milestone", "evidence_refs must be non-empty")
|
|
|
|
|
|
def load_json_object(path: Path, stage: str) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise StageFailure(stage, f"cannot read {path}: {exc}") from exc
|
|
if not isinstance(value, dict):
|
|
raise StageFailure(stage, f"JSON root must be an object: {path}")
|
|
return value
|
|
|
|
|
|
def parse_json_stdout(result: CommandResult, stage: str) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(result.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise StageFailure(stage, f"command did not emit JSON: {exc}") from exc
|
|
if not isinstance(value, dict) or value.get("ok") is not True:
|
|
raise StageFailure(stage, f"command reported failure: {value!r}")
|
|
return value
|
|
|
|
|
|
class SubprocessRunner:
|
|
"""Run one deterministic stage without leaking full command output."""
|
|
|
|
def run(
|
|
self,
|
|
stage: str,
|
|
argv: list[str],
|
|
*,
|
|
cwd: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
timeout: int | None = None,
|
|
allowed_returncodes: tuple[int, ...] = (0,),
|
|
) -> CommandResult:
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=cwd,
|
|
env=env,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
timeout=timeout,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise StageFailure(stage, str(exc)) from exc
|
|
stdout = completed.stdout.decode("utf-8", errors="replace")
|
|
stderr = completed.stderr.decode("utf-8", errors="replace")
|
|
if completed.returncode not in allowed_returncodes:
|
|
raise StageFailure(
|
|
stage,
|
|
f"exit={completed.returncode}; stdout={stdout[-1600:]}; "
|
|
f"stderr={stderr[-2400:]}",
|
|
)
|
|
return CommandResult(completed.returncode, stdout, stderr)
|
|
|
|
|
|
class CleanCandidateManager:
|
|
"""Materialize canonical HEAD and apply only the verified release patch."""
|
|
|
|
def __init__(self, repo_root: Path, runner: Runner):
|
|
self.repo_root = repo_root
|
|
self.runner = runner
|
|
self._temp: tempfile.TemporaryDirectory[str] | None = None
|
|
|
|
def materialize(self, base_commit: str, patch_path: str) -> Path:
|
|
self._temp = tempfile.TemporaryDirectory(prefix="vignette-release-agent-")
|
|
root = Path(self._temp.name) / "candidate"
|
|
root.mkdir(parents=True)
|
|
archive = Path(self._temp.name) / "base.tar"
|
|
self.runner.run(
|
|
"candidate_archive",
|
|
["git", "archive", "--format=tar", "--output", str(archive), base_commit],
|
|
cwd=self.repo_root,
|
|
timeout=120,
|
|
)
|
|
with tarfile.open(archive, "r:") as tar:
|
|
_safe_extract(tar, root)
|
|
self.runner.run(
|
|
"candidate_patch_apply",
|
|
["git", "apply", "--whitespace=nowarn", str(self.repo_root / patch_path)],
|
|
cwd=root,
|
|
timeout=120,
|
|
)
|
|
generated_api = root / "apps/web/src/lib/api.gen.ts"
|
|
if not generated_api.is_file():
|
|
raise StageFailure("candidate_patch_apply", "generated API contract is missing")
|
|
_normalize_candidate_file_to_lf(generated_api, "generated API contract")
|
|
for shell_script in sorted(root.rglob("*.sh")):
|
|
_normalize_candidate_file_to_lf(shell_script, "shell script")
|
|
return root
|
|
|
|
def cleanup(self) -> None:
|
|
if self._temp is not None:
|
|
self._temp.cleanup()
|
|
self._temp = None
|
|
|
|
|
|
def _normalize_candidate_file_to_lf(path: Path, label: str) -> None:
|
|
raw = path.read_bytes()
|
|
normalized = raw.replace(b"\r\n", b"\n")
|
|
if b"\r" in normalized:
|
|
raise StageFailure(
|
|
"candidate_patch_apply",
|
|
f"{label} contains unsupported carriage returns: {path}",
|
|
)
|
|
if normalized != raw:
|
|
path.write_bytes(normalized)
|
|
|
|
|
|
def _safe_extract(archive: tarfile.TarFile, destination: Path) -> None:
|
|
destination_resolved = destination.resolve()
|
|
for member in archive.getmembers():
|
|
target = (destination / member.name).resolve()
|
|
if target != destination_resolved and destination_resolved not in target.parents:
|
|
raise StageFailure("candidate_archive", f"archive path escapes candidate: {member.name}")
|
|
archive.extractall(destination, filter="data")
|
|
|
|
|
|
class HttpRuntimeProbe:
|
|
"""Verify semantic health, unauthenticated auth, and G0-G8 OpenAPI surface."""
|
|
|
|
def __init__(self, base_url: str, *, timeout_seconds: float = 15.0):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.timeout_seconds = timeout_seconds
|
|
|
|
def _json(self, path: str, expected_status: int = 200) -> tuple[int, Any]:
|
|
request = urllib.request.Request(
|
|
self.base_url + path,
|
|
headers={"accept": "application/json", "user-agent": "vignette-release-agent/1"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
|
|
status = response.status
|
|
body = response.read()
|
|
except urllib.error.HTTPError as exc:
|
|
status = exc.code
|
|
body = exc.read()
|
|
except (OSError, urllib.error.URLError) as exc:
|
|
raise StageFailure("runtime_contract", f"{path} unavailable: {exc}") from exc
|
|
if status != expected_status:
|
|
raise StageFailure(
|
|
"runtime_contract",
|
|
f"{path} status={status}, expected={expected_status}",
|
|
)
|
|
if not body:
|
|
return status, None
|
|
try:
|
|
return status, json.loads(body.decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise StageFailure("runtime_contract", f"{path} returned invalid JSON") from exc
|
|
|
|
def verify(self) -> dict[str, Any]:
|
|
_, health = self._json("/api/health")
|
|
if not isinstance(health, dict) or not (
|
|
health.get("status") == "ok"
|
|
and health.get("db") is True
|
|
and health.get("engine") is True
|
|
):
|
|
raise StageFailure("runtime_contract", f"semantic health failed: {health!r}")
|
|
auth_status, _ = self._json("/api/auth/me", expected_status=401)
|
|
_, openapi = self._json("/api/openapi.json")
|
|
if not isinstance(openapi, dict) or not isinstance(openapi.get("paths"), dict):
|
|
raise StageFailure("runtime_contract", "OpenAPI paths object is missing")
|
|
paths = openapi["paths"]
|
|
missing = {
|
|
goal: path for goal, path in REQUIRED_OPENAPI_PATHS.items() if path not in paths
|
|
}
|
|
schemas = openapi.get("components", {}).get("schemas", {})
|
|
if "AllianceScores" not in schemas:
|
|
missing["G0"] = "components.schemas.AllianceScores"
|
|
if missing:
|
|
raise StageFailure("runtime_contract", f"OpenAPI G0-G8 coverage missing: {missing}")
|
|
return {
|
|
"health": {
|
|
"status": health.get("status"),
|
|
"environment": health.get("environment"),
|
|
"db": health.get("db"),
|
|
"engine": health.get("engine"),
|
|
"engine_mode": health.get("engine_mode"),
|
|
},
|
|
"auth_status": auth_status,
|
|
"openapi_paths": len(paths),
|
|
"goals": [f"G{number}" for number in range(9)],
|
|
}
|
|
|
|
|
|
class NasPreviewDeploymentDriver:
|
|
"""Promote API/Web only inside the dedicated Compose project.
|
|
|
|
No command in this driver removes a volume, network, project, or the legacy
|
|
root. Candidate releases are immutable subdirectories. Rollback recreates
|
|
only api/web/proxy with the snapshotted image IDs and prior Compose file.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
runner: Runner,
|
|
*,
|
|
ssh_target: str,
|
|
target: DeploymentTarget = NAS_PREVIEW_TARGET,
|
|
):
|
|
validate_target(target)
|
|
if not SSH_TARGET_RE.fullmatch(ssh_target):
|
|
raise ValueError("ssh target must be [user@]100.116.83.60")
|
|
self.runner = runner
|
|
self.ssh_target = ssh_target
|
|
self.target = target
|
|
self._active_state: dict[str, Any] | None = None
|
|
|
|
@property
|
|
def _container_prefix(self) -> str:
|
|
return self.target.compose_project
|
|
|
|
def _remote(
|
|
self,
|
|
stage: str,
|
|
script: str,
|
|
*,
|
|
allowed_returncodes: tuple[int, ...] = (0,),
|
|
timeout: int = 120,
|
|
) -> CommandResult:
|
|
command = "sh -lc " + shlex.quote(script)
|
|
return self.runner.run(
|
|
stage,
|
|
[
|
|
"ssh.exe",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"ConnectTimeout=10",
|
|
self.ssh_target,
|
|
command,
|
|
],
|
|
timeout=timeout,
|
|
allowed_returncodes=allowed_returncodes,
|
|
)
|
|
|
|
def _scp(self, stage: str, source: Path, destination: str) -> None:
|
|
if not destination.startswith(self.target.remote_root.rstrip("/") + "/"):
|
|
raise StageFailure(stage, "remote copy destination escapes isolated preview root")
|
|
self.runner.run(
|
|
stage,
|
|
[
|
|
"scp.exe",
|
|
"-O",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"ConnectTimeout=10",
|
|
str(source),
|
|
f"{self.ssh_target}:{destination}",
|
|
],
|
|
timeout=600,
|
|
)
|
|
|
|
def read_active_state(self) -> dict[str, Any] | None:
|
|
state_path = f"{self.target.remote_root}/release-agent-active.json"
|
|
result = self._remote(
|
|
"nas_read_active_state",
|
|
f"test -f {shlex.quote(state_path)} || exit 3; cat {shlex.quote(state_path)}",
|
|
allowed_returncodes=(0, 3),
|
|
)
|
|
if result.returncode == 3:
|
|
self._active_state = None
|
|
return None
|
|
try:
|
|
payload = json.loads(result.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise StageFailure("nas_read_active_state", "remote state is invalid JSON") from exc
|
|
if not isinstance(payload, dict) or payload.get("schema") != "vignette.release-agent-active.v1":
|
|
raise StageFailure("nas_read_active_state", "remote state schema is invalid")
|
|
active_sha = payload.get("active_sha")
|
|
if not isinstance(active_sha, str) or not SHA256_RE.fullmatch(active_sha):
|
|
raise StageFailure("nas_read_active_state", "remote active_sha is invalid")
|
|
if payload.get("compose_project") != self.target.compose_project:
|
|
raise StageFailure("nas_read_active_state", "remote Compose project is invalid")
|
|
if payload.get("remote_root") != self.target.remote_root:
|
|
raise StageFailure("nas_read_active_state", "remote preview root is invalid")
|
|
validate_remote_compose_path(
|
|
payload.get("compose_file"),
|
|
self.target,
|
|
stage="nas_read_active_state",
|
|
)
|
|
env_file = payload.get("env_file")
|
|
if env_file is None:
|
|
compose_file = PurePosixPath(payload["compose_file"])
|
|
env_file = str(compose_file.parent / ".env")
|
|
env_file = validate_remote_env_file(
|
|
env_file,
|
|
self.target,
|
|
stage="nas_read_active_state",
|
|
)
|
|
for key in ("api_image", "web_image"):
|
|
image = payload.get(key)
|
|
if not isinstance(image, str) or not IMAGE_ID_RE.fullmatch(image):
|
|
raise StageFailure("nas_read_active_state", f"remote {key} is invalid")
|
|
normalized_payload = dict(payload)
|
|
normalized_payload["env_file"] = env_file
|
|
self._active_state = normalized_payload
|
|
return normalized_payload
|
|
|
|
def snapshot(self) -> DeploymentSnapshot:
|
|
root = self.target.remote_root
|
|
project = self.target.compose_project
|
|
previous_compose = (
|
|
self._active_state.get("compose_file")
|
|
if isinstance(self._active_state, dict)
|
|
else f"{root}/infra/docker-compose.yml"
|
|
)
|
|
previous_compose = validate_remote_compose_path(
|
|
previous_compose,
|
|
self.target,
|
|
stage="nas_snapshot",
|
|
)
|
|
previous_env = (
|
|
self._active_state.get("env_file")
|
|
if isinstance(self._active_state, dict)
|
|
else str(PurePosixPath(previous_compose).parent / ".env")
|
|
)
|
|
previous_env = validate_remote_env_file(
|
|
previous_env,
|
|
self.target,
|
|
stage="nas_snapshot",
|
|
)
|
|
api_container = f"{self._container_prefix}-api-1"
|
|
web_container = f"{self._container_prefix}-web-1"
|
|
script = "\n".join(
|
|
[
|
|
"set -eu",
|
|
f"test -f {shlex.quote(previous_compose)}",
|
|
f"test -f {shlex.quote(previous_env)}",
|
|
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(api_container)})",
|
|
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(web_container)})",
|
|
f"ap=$(docker inspect --format '{{{{index .Config.Labels \"com.docker.compose.project\"}}}}' {shlex.quote(api_container)})",
|
|
f"wp=$(docker inspect --format '{{{{index .Config.Labels \"com.docker.compose.project\"}}}}' {shlex.quote(web_container)})",
|
|
f"test \"$ap\" = {shlex.quote(project)}",
|
|
f"test \"$wp\" = {shlex.quote(project)}",
|
|
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
|
|
]
|
|
)
|
|
result = self._remote("nas_snapshot", script)
|
|
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
|
if len(lines) != 2 or any(not IMAGE_ID_RE.fullmatch(line) for line in lines):
|
|
raise StageFailure("nas_snapshot", f"invalid image snapshot: {lines!r}")
|
|
active_sha = None
|
|
if isinstance(self._active_state, dict):
|
|
value = self._active_state.get("active_sha")
|
|
if isinstance(value, str):
|
|
active_sha = value
|
|
return DeploymentSnapshot(
|
|
lines[0],
|
|
lines[1],
|
|
previous_compose,
|
|
previous_env,
|
|
active_sha,
|
|
)
|
|
|
|
def promote(
|
|
self,
|
|
candidate: Path,
|
|
desired_sha: str,
|
|
snapshot: DeploymentSnapshot,
|
|
) -> dict[str, Any]:
|
|
if not SHA256_RE.fullmatch(desired_sha):
|
|
raise StageFailure("nas_promote", "desired SHA is invalid")
|
|
root = self.target.remote_root
|
|
project = self.target.compose_project
|
|
run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + f"-{os.getpid()}"
|
|
release_key = f"{desired_sha[:16]}-{run_id}"
|
|
release_root = f"{root}/releases/{release_key}"
|
|
incoming = f"{root}/incoming"
|
|
env_file = validate_remote_env_file(
|
|
snapshot.env_file,
|
|
self.target,
|
|
stage="nas_promote",
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="vignette-nas-promotion-") as temp_name:
|
|
temp = Path(temp_name)
|
|
archive = temp / f"{release_key}.tar.gz"
|
|
_archive_candidate(candidate, archive)
|
|
override = temp / "release-agent.override.yml"
|
|
override.write_text(
|
|
"services:\n"
|
|
" api:\n"
|
|
f" image: {project}-api:{desired_sha[:16]}\n"
|
|
" web:\n"
|
|
f" image: {project}-web:{desired_sha[:16]}\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
self._remote(
|
|
"nas_prepare_release",
|
|
"\n".join(
|
|
[
|
|
"set -eu",
|
|
f"test -d {shlex.quote(root)}",
|
|
f"if [ ! -f {shlex.quote(env_file)} ]; then printf '%s\\n' {shlex.quote('required env file missing: ' + env_file)} >&2; exit 4; fi",
|
|
f"mkdir -p {shlex.quote(incoming)} {shlex.quote(root + '/releases')}",
|
|
f"test ! -e {shlex.quote(release_root)}",
|
|
]
|
|
),
|
|
)
|
|
self._scp("nas_upload_archive", archive, f"{incoming}/{archive.name}")
|
|
self._scp("nas_upload_override", override, f"{incoming}/{release_key}.override.yml")
|
|
compose = f"{release_root}/infra/docker-compose.yml"
|
|
remote_override = f"{release_root}/infra/release-agent.override.yml"
|
|
migration_commands: list[str] = []
|
|
for migration_name in RELEASE_DB_MIGRATIONS:
|
|
migration_path = f"{release_root}/infra/db/init/{migration_name}"
|
|
migration_commands.extend(
|
|
[
|
|
f"test -f {shlex.quote(migration_path)}",
|
|
(
|
|
f"docker compose -p {shlex.quote(project)} "
|
|
f"--env-file {shlex.quote(env_file)} "
|
|
f"-f {shlex.quote(compose)} -f {shlex.quote(remote_override)} "
|
|
"exec -T db sh -lc "
|
|
+ shlex.quote(
|
|
'psql -v ON_ERROR_STOP=1 --single-transaction '
|
|
'-U "$POSTGRES_USER" -d "$POSTGRES_DB"'
|
|
)
|
|
+ f" < {shlex.quote(migration_path)}"
|
|
),
|
|
]
|
|
)
|
|
script = "\n".join(
|
|
[
|
|
"set -eu",
|
|
f"mkdir {shlex.quote(release_root)}",
|
|
f"tar -xzf {shlex.quote(incoming + '/' + archive.name)} -C {shlex.quote(release_root)}",
|
|
f"mv {shlex.quote(incoming + '/' + release_key + '.override.yml')} {shlex.quote(remote_override)}",
|
|
f"test -f {shlex.quote(env_file)}",
|
|
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(compose)} -f {shlex.quote(remote_override)} build api web",
|
|
*migration_commands,
|
|
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(compose)} -f {shlex.quote(remote_override)} up -d --no-deps --force-recreate api web proxy",
|
|
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-api-1')})",
|
|
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-web-1')})",
|
|
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
|
|
]
|
|
)
|
|
result = self._remote("nas_promote", script, timeout=1800)
|
|
images = [line.strip() for line in result.stdout.splitlines() if IMAGE_ID_RE.fullmatch(line.strip())]
|
|
if len(images) < 2:
|
|
raise StageFailure("nas_promote", "promoted image IDs were not returned")
|
|
return {
|
|
"schema": "vignette.release-agent-active.v1",
|
|
"active_sha": desired_sha,
|
|
"compose_project": project,
|
|
"remote_root": root,
|
|
"release_root": release_root,
|
|
"compose_file": compose,
|
|
"env_file": env_file,
|
|
"api_image": images[-2],
|
|
"web_image": images[-1],
|
|
"previous_api_image": snapshot.api_image,
|
|
"previous_web_image": snapshot.web_image,
|
|
"deployed_at": utc_now(),
|
|
"database_migrations": list(RELEASE_DB_MIGRATIONS),
|
|
}
|
|
|
|
def commit_active_state(self, state: dict[str, Any]) -> None:
|
|
desired_sha = state.get("active_sha")
|
|
if not isinstance(desired_sha, str) or not SHA256_RE.fullmatch(desired_sha):
|
|
raise StageFailure("nas_commit_state", "active state SHA is invalid")
|
|
root = self.target.remote_root
|
|
with tempfile.TemporaryDirectory(prefix="vignette-nas-state-") as temp_name:
|
|
state_file = Path(temp_name) / f"{desired_sha}.json"
|
|
state_file.write_text(
|
|
json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
remote_temp = f"{root}/incoming/{desired_sha}.state.json"
|
|
self._scp("nas_upload_state", state_file, remote_temp)
|
|
self._remote(
|
|
"nas_commit_state",
|
|
f"set -eu; mv {shlex.quote(remote_temp)} {shlex.quote(root + '/release-agent-active.json')}",
|
|
)
|
|
self._active_state = state
|
|
|
|
def rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]:
|
|
for image in (snapshot.api_image, snapshot.web_image):
|
|
if not IMAGE_ID_RE.fullmatch(image):
|
|
raise StageFailure("nas_rollback", "snapshot image ID is invalid")
|
|
root = self.target.remote_root
|
|
validate_remote_compose_path(
|
|
snapshot.compose_file,
|
|
self.target,
|
|
stage="nas_rollback",
|
|
)
|
|
if snapshot.active_sha is None or not SHA256_RE.fullmatch(snapshot.active_sha):
|
|
raise StageFailure("nas_rollback", "snapshot active SHA is missing or invalid")
|
|
project = self.target.compose_project
|
|
env_file = validate_remote_env_file(
|
|
snapshot.env_file,
|
|
self.target,
|
|
stage="nas_rollback",
|
|
)
|
|
self._remote(
|
|
"nas_rollback_preflight",
|
|
"\n".join(
|
|
[
|
|
"set -eu",
|
|
f"if [ ! -f {shlex.quote(env_file)} ]; then printf '%s\\n' {shlex.quote('required env file missing: ' + env_file)} >&2; exit 4; fi",
|
|
f"test -f {shlex.quote(snapshot.compose_file)}",
|
|
]
|
|
),
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="vignette-nas-rollback-") as temp_name:
|
|
temp = Path(temp_name)
|
|
override = temp / "rollback.override.yml"
|
|
override.write_text(
|
|
"services:\n"
|
|
" api:\n"
|
|
f" image: {snapshot.api_image}\n"
|
|
" web:\n"
|
|
f" image: {snapshot.web_image}\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
restored_state = temp / "restored-state.json"
|
|
restored_state.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema": "vignette.release-agent-active.v1",
|
|
"active_sha": snapshot.active_sha,
|
|
"compose_project": project,
|
|
"remote_root": root,
|
|
"compose_file": snapshot.compose_file,
|
|
"env_file": env_file,
|
|
"api_image": snapshot.api_image,
|
|
"web_image": snapshot.web_image,
|
|
"rollback_restored_at": utc_now(),
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
remote_override = f"{root}/incoming/rollback-{os.getpid()}.override.yml"
|
|
remote_state = f"{root}/incoming/rollback-{os.getpid()}.state.json"
|
|
self._scp("nas_upload_rollback", override, remote_override)
|
|
self._scp("nas_upload_rollback_state", restored_state, remote_state)
|
|
self._remote(
|
|
"nas_rollback",
|
|
"\n".join(
|
|
[
|
|
"set -eu",
|
|
f"docker compose -p {shlex.quote(project)} --env-file {shlex.quote(env_file)} -f {shlex.quote(snapshot.compose_file)} -f {shlex.quote(remote_override)} up -d --no-build --no-deps --force-recreate api web proxy",
|
|
f"mv {shlex.quote(remote_state)} {shlex.quote(root + '/release-agent-active.json')}",
|
|
f"rm -f {shlex.quote(remote_override)}",
|
|
]
|
|
),
|
|
timeout=600,
|
|
)
|
|
self._active_state = {
|
|
"schema": "vignette.release-agent-active.v1",
|
|
"active_sha": snapshot.active_sha,
|
|
"compose_file": snapshot.compose_file,
|
|
"env_file": env_file,
|
|
"api_image": snapshot.api_image,
|
|
"web_image": snapshot.web_image,
|
|
}
|
|
return {"api_image": snapshot.api_image, "web_image": snapshot.web_image}
|
|
|
|
def verify_rollback(self, snapshot: DeploymentSnapshot) -> dict[str, Any]:
|
|
project = self.target.compose_project
|
|
result = self._remote(
|
|
"nas_verify_rollback_images",
|
|
"\n".join(
|
|
[
|
|
"set -eu",
|
|
f"api=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-api-1')})",
|
|
f"web=$(docker inspect --format '{{{{.Image}}}}' {shlex.quote(project + '-web-1')})",
|
|
"printf '%s\\n%s\\n' \"$api\" \"$web\"",
|
|
]
|
|
),
|
|
)
|
|
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
|
ok = lines == [snapshot.api_image, snapshot.web_image]
|
|
return {
|
|
"ok": ok,
|
|
"api_image": lines[0] if lines else None,
|
|
"web_image": lines[1] if len(lines) > 1 else None,
|
|
}
|
|
|
|
|
|
def _archive_candidate(candidate: Path, destination: Path) -> None:
|
|
excluded_parts = {
|
|
"node_modules",
|
|
".git",
|
|
".devlogs",
|
|
".release-agent",
|
|
"playwright-report",
|
|
}
|
|
with tarfile.open(destination, "w:gz", format=tarfile.PAX_FORMAT) as archive:
|
|
for path in sorted(candidate.rglob("*"), key=lambda value: value.as_posix()):
|
|
relative = path.relative_to(candidate)
|
|
if any(part in excluded_parts for part in relative.parts):
|
|
continue
|
|
if path.is_symlink():
|
|
continue
|
|
archive.add(path, arcname=relative.as_posix(), recursive=False)
|
|
|
|
|
|
class ReleaseAgent:
|
|
def __init__(
|
|
self,
|
|
config: ReleaseAgentConfig,
|
|
*,
|
|
runner: Runner,
|
|
deployment: DeploymentDriver,
|
|
runtime_probe: RuntimeProbe,
|
|
candidate_manager: CandidateManager,
|
|
):
|
|
self.config = config
|
|
self.runner = runner
|
|
self.deployment = deployment
|
|
self.runtime_probe = runtime_probe
|
|
self.candidate_manager = candidate_manager
|
|
self.stage_evidence: list[dict[str, Any]] = []
|
|
|
|
def _run(
|
|
self,
|
|
stage: str,
|
|
argv: list[str],
|
|
*,
|
|
cwd: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
timeout: int | None = None,
|
|
) -> CommandResult:
|
|
started = time.monotonic()
|
|
result = self.runner.run(stage, argv, cwd=cwd, env=env, timeout=timeout)
|
|
self.stage_evidence.append(
|
|
{
|
|
"stage": stage,
|
|
"status": "passed",
|
|
"duration_ms": int((time.monotonic() - started) * 1000),
|
|
"stdout_sha256": sha256_bytes(result.stdout.encode("utf-8")),
|
|
"stderr_sha256": sha256_bytes(result.stderr.encode("utf-8")),
|
|
}
|
|
)
|
|
return result
|
|
|
|
@staticmethod
|
|
def _is_transient_runtime_failure(exc: StageFailure) -> bool:
|
|
if exc.stage != "runtime_contract":
|
|
return False
|
|
detail = exc.detail.lower()
|
|
return any(
|
|
marker in detail
|
|
for marker in (
|
|
"unavailable:",
|
|
"status=502",
|
|
"status=503",
|
|
"status=504",
|
|
"semantic health failed",
|
|
)
|
|
)
|
|
|
|
def _verify_runtime_ready(self) -> dict[str, Any]:
|
|
deadline = time.monotonic() + self.config.runtime_ready_timeout_seconds
|
|
attempts = 0
|
|
while True:
|
|
attempts += 1
|
|
try:
|
|
report = dict(self.runtime_probe.verify())
|
|
report["readiness_attempts"] = attempts
|
|
return report
|
|
except StageFailure as exc:
|
|
if (
|
|
not self._is_transient_runtime_failure(exc)
|
|
or time.monotonic() >= deadline
|
|
):
|
|
raise
|
|
time.sleep(self.config.runtime_retry_interval_seconds)
|
|
|
|
def _build_release_twice(self) -> dict[str, Any]:
|
|
reports: list[dict[str, Any]] = []
|
|
for run_number in (1, 2):
|
|
stage = f"release_patch_run_{run_number}"
|
|
result = self._run(
|
|
stage,
|
|
[
|
|
sys.executable,
|
|
"-X",
|
|
"utf8",
|
|
str(BUILDER if self.config.repo_root == REPO_ROOT else self.config.repo_root / "scripts/build-outcome-os-release-patch.py"),
|
|
"--map",
|
|
str(self.config.hunk_map_path),
|
|
"--json",
|
|
],
|
|
cwd=self.config.repo_root,
|
|
timeout=900,
|
|
)
|
|
reports.append(parse_json_stdout(result, stage))
|
|
required = {
|
|
"patch_sha256",
|
|
"patch_bytes",
|
|
"patch_files",
|
|
"base_commit",
|
|
"output_patch",
|
|
"git_apply_check",
|
|
"git_apply_cached_check",
|
|
"patch_line_endings",
|
|
"source_worktree",
|
|
"api_generation",
|
|
}
|
|
for report in reports:
|
|
missing = required - report.keys()
|
|
if missing:
|
|
raise StageFailure("release_determinism", f"builder report missing {sorted(missing)}")
|
|
if report["git_apply_cached_check"] != "passed-on-canonical-clean-index":
|
|
raise StageFailure("release_determinism", "canonical clean-index proof is missing")
|
|
if report["git_apply_check"] != "passed-on-clean-temporary-head":
|
|
raise StageFailure("release_determinism", "clean HEAD apply proof is missing")
|
|
comparable = (
|
|
"patch_sha256",
|
|
"patch_bytes",
|
|
"patch_files",
|
|
"base_commit",
|
|
"output_patch",
|
|
"git_apply_check",
|
|
"git_apply_cached_check",
|
|
"patch_line_endings",
|
|
"source_worktree",
|
|
"api_generation",
|
|
)
|
|
drift = {
|
|
key: [reports[0].get(key), reports[1].get(key)]
|
|
for key in comparable
|
|
if reports[0].get(key) != reports[1].get(key)
|
|
}
|
|
if drift:
|
|
raise StageFailure("release_determinism", f"two release builds differ: {drift}")
|
|
sha = reports[1]["patch_sha256"]
|
|
if not isinstance(sha, str) or not SHA256_RE.fullmatch(sha):
|
|
raise StageFailure("release_determinism", "builder patch SHA is invalid")
|
|
return reports[1]
|
|
|
|
def _check_manifest_binding(self, release: dict[str, Any]) -> None:
|
|
result = self._run(
|
|
"release_manifest",
|
|
[
|
|
sys.executable,
|
|
"-X",
|
|
"utf8",
|
|
str(MANIFEST_CHECKER if self.config.repo_root == REPO_ROOT else self.config.repo_root / "scripts/check-outcome-os-release-manifest.py"),
|
|
"--manifest",
|
|
str(self.config.manifest_path),
|
|
"--json",
|
|
],
|
|
cwd=self.config.repo_root,
|
|
timeout=300,
|
|
)
|
|
parse_json_stdout(result, "release_manifest")
|
|
manifest = load_json_object(self.config.manifest_path, "release_manifest_binding")
|
|
validate_required_release_payload(manifest)
|
|
assembly = manifest.get("release_assembly")
|
|
if not isinstance(assembly, dict):
|
|
raise StageFailure("release_manifest_binding", "release_assembly is missing")
|
|
if assembly.get("patch_sha256") != release["patch_sha256"]:
|
|
raise StageFailure("release_manifest_binding", "manifest is not bound to current patch SHA")
|
|
if assembly.get("output_patch") != release["output_patch"]:
|
|
raise StageFailure("release_manifest_binding", "manifest output patch path drifted")
|
|
|
|
def _validate_snapshot_binding(
|
|
self,
|
|
active_state: dict[str, Any],
|
|
snapshot: DeploymentSnapshot,
|
|
) -> None:
|
|
expected = {
|
|
"active_sha": snapshot.active_sha,
|
|
"compose_file": snapshot.compose_file,
|
|
"env_file": snapshot.env_file,
|
|
"api_image": snapshot.api_image,
|
|
"web_image": snapshot.web_image,
|
|
}
|
|
drift = {
|
|
key: {"tracked": active_state.get(key), "runtime": value}
|
|
for key, value in expected.items()
|
|
if active_state.get(key) != value
|
|
}
|
|
if drift:
|
|
raise StageFailure(
|
|
"deployment_state",
|
|
f"tracked active state does not match runtime snapshot: {drift}",
|
|
)
|
|
|
|
def _run_candidate_gates(self, candidate: Path, desired_sha: str) -> None:
|
|
python = sys.executable
|
|
candidate_preflight = candidate / "scripts/check-deploy-preflight.py"
|
|
self._run(
|
|
"nas_preflight",
|
|
[
|
|
python,
|
|
"-X",
|
|
"utf8",
|
|
str(candidate_preflight),
|
|
"--env-file",
|
|
str(self.config.nas_env_file),
|
|
"--deployment-profile",
|
|
"compose",
|
|
"--skip-db",
|
|
],
|
|
cwd=candidate,
|
|
timeout=180,
|
|
)
|
|
self._run(
|
|
"api_tests",
|
|
[python, "-X", "utf8", "-m", "pytest", "-p", "no:cacheprovider", "-q"],
|
|
cwd=candidate / "apps/api",
|
|
timeout=1800,
|
|
)
|
|
web = candidate / "apps/web"
|
|
self._run("web_install", ["npm.cmd", "ci", "--ignore-scripts"], cwd=web, timeout=900)
|
|
self._run("web_api_contract", ["npm.cmd", "run", "check:api-types"], cwd=web, timeout=300)
|
|
self._run("web_typecheck", ["npm.cmd", "run", "typecheck"], cwd=web, timeout=300)
|
|
self._run("web_build", ["npm.cmd", "run", "build"], cwd=web, timeout=600)
|
|
|
|
stack_attempted = False
|
|
stack_env: Path | None = None
|
|
project = candidate_compose_project(desired_sha)
|
|
try:
|
|
if self.config.run_candidate_stack:
|
|
stack_env = self._candidate_stack_env(candidate)
|
|
stack_attempted = True
|
|
self._run(
|
|
"candidate_stack_up",
|
|
[
|
|
"docker.exe",
|
|
"compose",
|
|
"-p",
|
|
project,
|
|
"--env-file",
|
|
str(stack_env),
|
|
"-f",
|
|
str(candidate / "infra/docker-compose.yml"),
|
|
"up",
|
|
"-d",
|
|
"--build",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"600",
|
|
],
|
|
cwd=candidate,
|
|
timeout=1800,
|
|
)
|
|
HttpRuntimeProbe(self.config.candidate_base_url, timeout_seconds=20).verify()
|
|
e2e_env = os.environ.copy()
|
|
e2e_env["PLAYWRIGHT_BASE_URL"] = self.config.candidate_base_url
|
|
e2e_env["PLAYWRIGHT_SKIP_WEB_SERVER"] = "1"
|
|
self._run(
|
|
"session_e2e",
|
|
[
|
|
"node.exe",
|
|
str(web / "node_modules/@playwright/test/cli.js"),
|
|
"test",
|
|
*RELEASE_UI_E2E_SPECS,
|
|
"--project=chromium-desktop",
|
|
"--project=chromium-mobile",
|
|
"--project=chromium-single-run",
|
|
"--workers=1",
|
|
"--reporter=line",
|
|
],
|
|
cwd=web,
|
|
env=e2e_env,
|
|
timeout=RELEASE_E2E_TIMEOUT_SECONDS,
|
|
)
|
|
finally:
|
|
if stack_attempted and stack_env is not None:
|
|
self._run(
|
|
"candidate_stack_down",
|
|
[
|
|
"docker.exe",
|
|
"compose",
|
|
"-p",
|
|
project,
|
|
"--env-file",
|
|
str(stack_env),
|
|
"-f",
|
|
str(candidate / "infra/docker-compose.yml"),
|
|
"down",
|
|
"--remove-orphans",
|
|
"--volumes",
|
|
],
|
|
cwd=candidate,
|
|
timeout=600,
|
|
)
|
|
|
|
def _candidate_stack_env(self, candidate: Path) -> Path:
|
|
try:
|
|
lines = self.config.nas_env_file.read_text(encoding="utf-8").splitlines()
|
|
except OSError as exc:
|
|
raise StageFailure("candidate_stack_env", str(exc)) from exc
|
|
overrides = {
|
|
"HTTP_PORT": str(self.config.candidate_http_port),
|
|
"HTTPS_PORT": str(self.config.candidate_https_port),
|
|
"FRONTEND_BASE_URL": self.config.candidate_base_url,
|
|
"FRONTEND_ORIGIN_MAP": json.dumps(
|
|
{"default": self.config.candidate_base_url}, separators=(",", ":")
|
|
),
|
|
"CORS_ORIGINS": json.dumps([self.config.candidate_base_url], separators=(",", ":")),
|
|
"AUTH_DEV_LOGIN_EXTRA_ORIGINS": json.dumps(
|
|
[self.config.candidate_base_url], separators=(",", ":")
|
|
),
|
|
"AUTH_DEV_LOGIN_ENABLED": "true",
|
|
"PUBLIC_API_BASE": "/api",
|
|
}
|
|
seen: set[str] = set()
|
|
output: list[str] = []
|
|
for line in lines:
|
|
if not line or line.lstrip().startswith("#") or "=" not in line:
|
|
output.append(line)
|
|
continue
|
|
key = line.split("=", 1)[0].strip()
|
|
if key in overrides:
|
|
output.append(f"{key}={overrides[key]}")
|
|
seen.add(key)
|
|
else:
|
|
output.append(line)
|
|
for key in sorted(overrides.keys() - seen):
|
|
output.append(f"{key}={overrides[key]}")
|
|
destination = candidate / ".release-agent/candidate.env"
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text("\n".join(output) + "\n", encoding="utf-8", newline="\n")
|
|
return destination
|
|
|
|
def _run_postdeploy_browser_review(self, candidate: Path) -> dict[str, Any]:
|
|
web = candidate / "apps/web"
|
|
env = os.environ.copy()
|
|
env["PLAYWRIGHT_BASE_URL"] = self.config.target.base_url
|
|
env["PLAYWRIGHT_SKIP_WEB_SERVER"] = "1"
|
|
result = self._run(
|
|
"postdeploy_browser_review",
|
|
[
|
|
"node.exe",
|
|
str(web / "node_modules/@playwright/test/cli.js"),
|
|
"test",
|
|
"e2e/session-persistence.spec.ts",
|
|
"--project=chromium-single-run",
|
|
"--workers=1",
|
|
"--grep",
|
|
"persists the browser SSE stream into DB-backed review",
|
|
"--reporter=line",
|
|
],
|
|
cwd=web,
|
|
env=env,
|
|
timeout=300,
|
|
)
|
|
return {
|
|
"status": "passed",
|
|
"test": "session-persistence: browser SSE stream -> DB-backed review",
|
|
"physical_microphone_used": False,
|
|
"stdout_sha256": sha256_bytes(result.stdout.encode("utf-8")),
|
|
}
|
|
|
|
def _base_report(self, desired_sha: str | None = None) -> dict[str, Any]:
|
|
return {
|
|
"schema": "vignette.outcome-os-release-agent-evidence.v1",
|
|
"captured_at": utc_now(),
|
|
"ok": False,
|
|
"mode": "execute" if self.config.execute else "dry-run",
|
|
"target": asdict(self.config.target),
|
|
"milestone": asdict(self.config.milestone) if self.config.milestone else None,
|
|
"desired_sha": desired_sha,
|
|
"stages": self.stage_evidence,
|
|
}
|
|
|
|
def _write_evidence(self, report: dict[str, Any]) -> None:
|
|
if self.config.evidence_out is None:
|
|
return
|
|
path = self.config.evidence_out
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
candidate = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
try:
|
|
candidate.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
os.replace(candidate, path)
|
|
finally:
|
|
if candidate.exists():
|
|
candidate.unlink()
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
candidate: Path | None = None
|
|
snapshot: DeploymentSnapshot | None = None
|
|
promotion_attempted = False
|
|
desired_sha: str | None = None
|
|
try:
|
|
validate_target(self.config.target)
|
|
validate_milestone(self.config.milestone)
|
|
release = self._build_release_twice()
|
|
desired_sha = release["patch_sha256"]
|
|
self._check_manifest_binding(release)
|
|
active_state = self.deployment.read_active_state()
|
|
active_sha = active_state.get("active_sha") if active_state else None
|
|
if active_sha is not None and (
|
|
not isinstance(active_sha, str) or not SHA256_RE.fullmatch(active_sha)
|
|
):
|
|
raise StageFailure("deployment_state", "active deployment SHA is invalid")
|
|
|
|
if active_sha == desired_sha:
|
|
snapshot = self.deployment.snapshot()
|
|
self._validate_snapshot_binding(active_state, snapshot)
|
|
snapshot = None
|
|
runtime = self.runtime_probe.verify()
|
|
report = self._base_report(desired_sha)
|
|
report.update(
|
|
{
|
|
"ok": True,
|
|
"status": "noop_same_sha",
|
|
"active_sha": active_sha,
|
|
"runtime": runtime,
|
|
"deployment_mutated": False,
|
|
}
|
|
)
|
|
self._write_evidence(report)
|
|
return report
|
|
|
|
if not self.config.execute:
|
|
report = self._base_report(desired_sha)
|
|
report.update(
|
|
{
|
|
"ok": True,
|
|
"status": "dry_run_changed_sha" if active_sha else "dry_run_untracked_state",
|
|
"active_sha": active_sha,
|
|
"deployment_mutated": False,
|
|
"would_promote": active_sha is not None and active_sha != desired_sha,
|
|
}
|
|
)
|
|
self._write_evidence(report)
|
|
return report
|
|
|
|
if active_sha is None:
|
|
raise StageFailure(
|
|
"deployment_state",
|
|
"remote release-agent state is untracked; adopt it only after an image/Compose rollback snapshot audit",
|
|
)
|
|
|
|
candidate = self.candidate_manager.materialize(
|
|
release["base_commit"], release["output_patch"]
|
|
)
|
|
self._run_candidate_gates(candidate, desired_sha)
|
|
snapshot = self.deployment.snapshot()
|
|
self._validate_snapshot_binding(active_state, snapshot)
|
|
promotion_attempted = True
|
|
promoted_state = self.deployment.promote(candidate, desired_sha, snapshot)
|
|
runtime = self._verify_runtime_ready()
|
|
browser_review = self._run_postdeploy_browser_review(candidate)
|
|
self.deployment.commit_active_state(promoted_state)
|
|
|
|
report = self._base_report(desired_sha)
|
|
report.update(
|
|
{
|
|
"ok": True,
|
|
"status": "deployed_ssot_sync_required",
|
|
"active_sha": desired_sha,
|
|
"previous_sha": active_sha,
|
|
"deployment": {
|
|
"desired_sha": desired_sha,
|
|
"previous_sha": active_sha,
|
|
"api_image": promoted_state.get("api_image"),
|
|
"web_image": promoted_state.get("web_image"),
|
|
"compose_project": self.config.target.compose_project,
|
|
},
|
|
"runtime": runtime,
|
|
"browser_review_proof": browser_review,
|
|
"deployment_mutated": True,
|
|
"ssot_sync": {
|
|
"status": "required",
|
|
"deployment_sha": desired_sha,
|
|
"required_paths": list(SSOT_REQUIRED_PATHS),
|
|
"required_anchor": desired_sha,
|
|
"checker_commands": [
|
|
"py -3.11 -X utf8 scripts/check-dev-dashboard-ssot.py --json",
|
|
"py -3.11 -X utf8 scripts/check-outcome-os-release-manifest.py --json",
|
|
],
|
|
},
|
|
}
|
|
)
|
|
self._write_evidence(report)
|
|
return report
|
|
except (StageFailure, ValueError, OSError) as exc:
|
|
stage = exc.stage if isinstance(exc, StageFailure) else "configuration"
|
|
rollback: dict[str, Any] = {"status": "not-required"}
|
|
if promotion_attempted and snapshot is not None:
|
|
rollback = {"status": "failed"}
|
|
try:
|
|
rollback_effect = self.deployment.rollback(snapshot)
|
|
restored_runtime = self._verify_runtime_ready()
|
|
rollback_proof = self.deployment.verify_rollback(snapshot)
|
|
if rollback_proof.get("ok") is not True:
|
|
raise StageFailure(
|
|
"rollback_proof",
|
|
f"rollback image proof mismatch: {rollback_proof}",
|
|
)
|
|
rollback = {
|
|
"status": "verified",
|
|
"effect": rollback_effect,
|
|
"runtime": restored_runtime,
|
|
"image_proof": rollback_proof,
|
|
}
|
|
except (StageFailure, ValueError, OSError) as rollback_exc:
|
|
rollback = {"status": "failed", "error": str(rollback_exc)}
|
|
report = self._base_report(desired_sha)
|
|
report.update(
|
|
{
|
|
"ok": False,
|
|
"status": "failed_closed",
|
|
"failed_stage": stage,
|
|
"error": str(exc),
|
|
"deployment_mutated": promotion_attempted,
|
|
"rollback": rollback,
|
|
}
|
|
)
|
|
self._write_evidence(report)
|
|
raise ReleaseAgentFailure(report) from exc
|
|
finally:
|
|
if candidate is not None:
|
|
self.candidate_manager.cleanup()
|
|
|
|
|
|
def load_milestone(path: Path) -> MaterialMilestone:
|
|
payload = load_json_object(path, "material_milestone")
|
|
if payload.get("schema") != "vignette.material-milestone.v1":
|
|
raise StageFailure("material_milestone", "milestone schema is invalid")
|
|
goals = payload.get("goals")
|
|
evidence_refs = payload.get("evidence_refs")
|
|
if not isinstance(goals, list) or any(not isinstance(value, str) for value in goals):
|
|
raise StageFailure("material_milestone", "goals must be strings")
|
|
if not isinstance(evidence_refs, list) or any(
|
|
not isinstance(value, str) for value in evidence_refs
|
|
):
|
|
raise StageFailure("material_milestone", "evidence_refs must be strings")
|
|
milestone_id = payload.get("milestone_id")
|
|
if not isinstance(milestone_id, str):
|
|
raise StageFailure("material_milestone", "milestone_id must be a string")
|
|
return MaterialMilestone(
|
|
milestone_id=milestone_id,
|
|
material=payload.get("material") is True,
|
|
goals=tuple(goals),
|
|
evidence_refs=tuple(evidence_refs),
|
|
)
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--milestone", type=Path, required=True)
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--hunk-map", type=Path, default=DEFAULT_HUNK_MAP)
|
|
parser.add_argument("--nas-env-file", type=Path, default=DEFAULT_NAS_ENV)
|
|
parser.add_argument("--evidence-out", type=Path)
|
|
parser.add_argument("--ssh-target", help="Required for live state inspection/execution")
|
|
parser.add_argument("--current-deployment-sha", help="Dry-run state override")
|
|
parser.add_argument(
|
|
"--execute",
|
|
action="store_true",
|
|
help="Promote only the dedicated NAS preview; default is read-only dry-run",
|
|
)
|
|
parser.add_argument(
|
|
"--run-candidate-stack",
|
|
action="store_true",
|
|
help="Start a uniquely named local Compose stack for pre-deploy session E2E",
|
|
)
|
|
parser.add_argument("--json", action="store_true")
|
|
return parser
|
|
|
|
|
|
class StaticDeploymentState:
|
|
"""Dry-run-only state provider; every mutating method is a hard failure."""
|
|
|
|
def __init__(self, active_sha: str | None):
|
|
if active_sha is not None and not SHA256_RE.fullmatch(active_sha):
|
|
raise ValueError("current deployment SHA must be 64 lowercase hex characters")
|
|
self.active_sha = active_sha
|
|
|
|
def read_active_state(self) -> dict[str, Any] | None:
|
|
if self.active_sha is None:
|
|
return None
|
|
return {"schema": "vignette.release-agent-active.v1", "active_sha": self.active_sha}
|
|
|
|
def _deny(self, *_: Any, **__: Any):
|
|
raise StageFailure("deployment_driver", "static dry-run driver cannot mutate")
|
|
|
|
snapshot = _deny
|
|
promote = _deny
|
|
commit_active_state = _deny
|
|
rollback = _deny
|
|
verify_rollback = _deny
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
milestone = load_milestone(args.milestone)
|
|
runner = SubprocessRunner()
|
|
if args.execute:
|
|
if not args.ssh_target:
|
|
raise StageFailure("configuration", "--execute requires --ssh-target")
|
|
deployment: DeploymentDriver = NasPreviewDeploymentDriver(
|
|
runner,
|
|
ssh_target=args.ssh_target,
|
|
)
|
|
elif args.ssh_target:
|
|
deployment = NasPreviewDeploymentDriver(runner, ssh_target=args.ssh_target)
|
|
else:
|
|
deployment = StaticDeploymentState(args.current_deployment_sha)
|
|
config = ReleaseAgentConfig(
|
|
repo_root=REPO_ROOT,
|
|
manifest_path=args.manifest,
|
|
hunk_map_path=args.hunk_map,
|
|
execute=args.execute,
|
|
milestone=milestone,
|
|
target=NAS_PREVIEW_TARGET,
|
|
evidence_out=args.evidence_out,
|
|
nas_env_file=args.nas_env_file,
|
|
run_candidate_stack=args.run_candidate_stack,
|
|
)
|
|
agent = ReleaseAgent(
|
|
config,
|
|
runner=runner,
|
|
deployment=deployment,
|
|
runtime_probe=HttpRuntimeProbe(NAS_PREVIEW_TARGET.base_url),
|
|
candidate_manager=CleanCandidateManager(REPO_ROOT, runner),
|
|
)
|
|
report = agent.run()
|
|
if args.json:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
|
else:
|
|
print(
|
|
f"PASS release agent: {report['status']} "
|
|
f"desired={report.get('desired_sha')} active={report.get('active_sha')}"
|
|
)
|
|
return 0
|
|
except ReleaseAgentFailure as exc:
|
|
if args.json:
|
|
print(json.dumps(exc.report, ensure_ascii=False, indent=2, sort_keys=True))
|
|
else:
|
|
print(str(exc), file=sys.stderr)
|
|
return 1
|
|
except (StageFailure, ValueError, OSError) as exc:
|
|
report = {"ok": False, "status": "failed_closed", "error": str(exc)}
|
|
if args.json:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
|
else:
|
|
print(f"FAIL release agent: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|