From ccdcfcd2f57bb7fffd3813d38c66fd999d7d0200 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Sat, 29 Aug 2026 23:58:33 +0900 Subject: [PATCH] =?UTF-8?q?=EC=95=84=EB=B0=94=ED=83=80=20=EC=A0=80?= =?UTF-8?q?=EC=9E=A5=EC=86=8C=20=EC=8A=B9=EA=B2=A9=20=EA=B3=84=EC=95=BD?= =?UTF-8?q?=EC=9D=84=20=EC=99=84=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/app/config.py | 20 + apps/api/app/db.py | 8 + apps/api/app/main.py | 119 +- apps/api/app/routes/users.py | 270 +- apps/api/app/test_engine_health_contract.py | 118 + apps/api/app/test_upload_storage_contract.py | 1230 ++++++++ apps/api/app/upload_runtime.py | 228 ++ apps/api/app/upload_storage.py | 1059 +++++++ apps/api/requirements.txt | 1 + scripts/boot-public-runtime.ps1 | 175 +- ...trap-legacy-public-runtime-upload-root.ps1 | 2670 +++++++++++++++++ .../initialize-public-runtime-upload-root.ps1 | 395 +++ .../initialize-public-runtime-upload-root.py | 1664 ++++++++++ scripts/install-public-runtime-task.ps1 | 102 +- .../probe-public-runtime-database-identity.py | 61 + scripts/probe-public-runtime-upload-root.py | 369 +++ ...public-runtime-task-definition-cutover.ps1 | 392 +++ scripts/public-runtime-task-maintenance.ps1 | 220 ++ scripts/public-runtime-upload-root.ps1 | 382 +++ scripts/public_runtime_database_identity.py | 74 + scripts/register-boot-task.ps1 | 98 +- scripts/start-public-runtime.ps1 | 1632 +++++++++- ...t_initialize_public_runtime_upload_root.py | 675 +++++ ..._legacy_public_runtime_upload_bootstrap.py | 309 ++ ...test_public_runtime_environment_handoff.py | 282 ++ .../test_public_runtime_listener_pid_probe.py | 259 ++ ..._public_runtime_task_definition_cutover.py | 285 ++ .../test_public_runtime_task_maintenance.py | 327 ++ ...st_public_runtime_upload_release_safety.py | 460 +++ scripts/test_public_runtime_upload_root.py | 290 ++ ...test_public_runtime_watchdog_provenance.py | 73 +- scripts/test_start_public_runtime_contract.py | 197 +- ...idate-public-runtime-offline-quiescence.py | 183 ++ ...validate-public-runtime-upload-manifest.py | 136 + scripts/watch-public-runtime-hidden.vbs | 2 +- scripts/watch-public-runtime.ps1 | 191 +- 36 files changed, 14734 insertions(+), 222 deletions(-) create mode 100644 apps/api/app/test_upload_storage_contract.py create mode 100644 apps/api/app/upload_runtime.py create mode 100644 apps/api/app/upload_storage.py create mode 100644 scripts/bootstrap-legacy-public-runtime-upload-root.ps1 create mode 100644 scripts/initialize-public-runtime-upload-root.ps1 create mode 100644 scripts/initialize-public-runtime-upload-root.py create mode 100644 scripts/probe-public-runtime-database-identity.py create mode 100644 scripts/probe-public-runtime-upload-root.py create mode 100644 scripts/public-runtime-task-definition-cutover.ps1 create mode 100644 scripts/public-runtime-task-maintenance.ps1 create mode 100644 scripts/public-runtime-upload-root.ps1 create mode 100644 scripts/public_runtime_database_identity.py create mode 100644 scripts/test_initialize_public_runtime_upload_root.py create mode 100644 scripts/test_legacy_public_runtime_upload_bootstrap.py create mode 100644 scripts/test_public_runtime_environment_handoff.py create mode 100644 scripts/test_public_runtime_listener_pid_probe.py create mode 100644 scripts/test_public_runtime_task_definition_cutover.py create mode 100644 scripts/test_public_runtime_task_maintenance.py create mode 100644 scripts/test_public_runtime_upload_release_safety.py create mode 100644 scripts/test_public_runtime_upload_root.py create mode 100644 scripts/validate-public-runtime-offline-quiescence.py create mode 100644 scripts/validate-public-runtime-upload-manifest.py diff --git a/apps/api/app/config.py b/apps/api/app/config.py index ed8eb32..5ac61e8 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -465,6 +465,26 @@ class Settings(BaseSettings): default="uploads", validation_alias="USER_UPLOAD_DIR", ) + user_upload_manifest_required: bool = Field( + default=False, + validation_alias="USER_UPLOAD_MANIFEST_REQUIRED", + ) + user_upload_manifest_path: str = Field( + default="", + validation_alias="USER_UPLOAD_MANIFEST_PATH", + ) + user_upload_manifest_sha256: str = Field( + default="", + validation_alias="USER_UPLOAD_MANIFEST_SHA256", + ) + user_upload_write_freeze_path: str = Field( + default="", + validation_alias="USER_UPLOAD_WRITE_FREEZE_PATH", + ) + public_runtime_db_target_sha256: str = Field( + default="", + validation_alias="PUBLIC_RUNTIME_DB_TARGET_SHA256", + ) # ── 운영 메일 알림 ───────────────────────────────────── notification_email_provider: Literal["disabled", "smtp"] = Field( diff --git a/apps/api/app/db.py b/apps/api/app/db.py index d4bd95c..1124935 100644 --- a/apps/api/app/db.py +++ b/apps/api/app/db.py @@ -15,6 +15,7 @@ from typing import Any, AsyncIterator, Optional, Sequence import asyncpg from .config import settings +from .upload_storage import connected_database_target_sha256 # 전역 풀 핸들. main.py lifespan 에서 init/close. _pool: Optional[asyncpg.Pool] = None @@ -27,6 +28,13 @@ async def _init_connection(conn: asyncpg.Connection) -> None: - vector(1024): pgvector. 런타임 인코딩은 RAG 경로에서 처리(여기선 텍스트 캐스트 허용). TODO: pgvector 바이너리 코덱 등록(register_vector) — RAG 라우터 구현 시 BGE-M3 1024d 연동. """ + if settings.user_upload_manifest_required: + actual_database_target_sha256 = await connected_database_target_sha256(conn) + if ( + actual_database_target_sha256 + != settings.public_runtime_db_target_sha256 + ): + raise RuntimeError("connected public runtime database target drift") await conn.set_type_codec( "jsonb", encoder=lambda v: json.dumps(v, ensure_ascii=False), diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 220a86b..4627f83 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -9,17 +9,16 @@ from __future__ import annotations from contextlib import asynccontextmanager import logging -from pathlib import Path +from dataclasses import asdict from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles from . import __version__ from .access_logging import install_uvicorn_access_log_redaction from .auth_sessions import ensure_runtime_tables from .config import settings -from .db import acquire, close_pool, healthcheck, init_pool +from .db import acquire, close_pool, get_pool, healthcheck, init_pool from .engine_client import engine_client from .persona_repository import materialize_seed_personas from .session_persistence import ensure_review_tables @@ -63,6 +62,13 @@ from .services import ( supervision_research_producer, ) from .services.voice import voice_service +from .upload_runtime import ( + FlatPublicAvatarStaticFiles, + public_avatar_root, + upload_manifest_proof, + upload_write_freeze_gate, + validate_runtime_upload_database_state, +) logger = logging.getLogger(__name__) @@ -83,8 +89,13 @@ async def lifespan(app: FastAPI): supervision_research_schema_ready = False continuous_improvement_schema_ready = False multimodal_alliance_schema_ready = False + app.state.upload_database_proof = None try: await init_pool() + if upload_manifest_proof is not None: + app.state.upload_database_proof = ( + await validate_runtime_upload_database_state(get_pool()) + ) await ensure_runtime_tables() await ensure_review_tables() await ensure_notification_tables() @@ -190,7 +201,7 @@ async def lifespan(app: FastAPI): await materialize_seed_personas() await admin_routes.apply_engine_config_from_store() except Exception as exc: # DB 없어도 store 폴백으로 1턴 동작 (dev/로컬) - if settings.environment != "dev": + if settings.environment != "dev" or upload_manifest_proof is not None: raise import logging @@ -245,11 +256,11 @@ app.add_middleware( # TODO: Presidio PII 마스킹 미들웨어 (외부 LLM 경로 진입 전 하드 게이트, R7/F-03) -_upload_root = Path(settings.user_upload_dir) -if not _upload_root.is_absolute(): - _upload_root = Path.cwd() / _upload_root -_upload_root.mkdir(parents=True, exist_ok=True) -app.mount("/uploads", StaticFiles(directory=str(_upload_root)), name="uploads") +app.mount( + "/uploads/profile-avatars", + FlatPublicAvatarStaticFiles(directory=str(public_avatar_root)), + name="profile-avatars", +) app.include_router(auth_routes.router) app.include_router(calibration_transfer_routes.router) @@ -280,8 +291,25 @@ async def health() -> dict[str, object]: db_ok = await healthcheck() engine = await engine_client.health_detail() engine_ok = bool(engine.get("ok")) + upload_freeze = asdict(upload_write_freeze_gate.status()) + upload_database_proof = getattr(app.state, "upload_database_proof", None) + upload_write_gate_ok = ( + not bool(upload_freeze["capable"]) or bool(upload_freeze["valid"]) + ) + upload_database_ok = ( + upload_manifest_proof is None + or ( + upload_database_proof is not None + and upload_database_proof.database_target_sha256 + == upload_manifest_proof.database_target_sha256 + ) + ) return { - "status": "ok" if db_ok and engine_ok else "degraded", + "status": ( + "ok" + if db_ok and engine_ok and upload_write_gate_ok and upload_database_ok + else "degraded" + ), "version": __version__, "environment": settings.environment, "db": db_ok, @@ -293,4 +321,75 @@ async def health() -> dict[str, object]: ), "default_engine": engine.get("default_engine"), "live_client_engine": engine.get("live_client_engine"), + "upload_write_freeze": upload_freeze, + "upload_manifest": ( + { + "required": True, + "validated": True, + "manifest_sha256": upload_manifest_proof.manifest_sha256, + "database_target_sha256": ( + upload_manifest_proof.database_target_sha256 + ), + "preserved_object_count": ( + upload_manifest_proof.preserved_object_count + ), + "preserved_total_size_bytes": ( + upload_manifest_proof.preserved_total_size_bytes + ), + "preserved_decode_valid_count": ( + upload_manifest_proof.preserved_decode_valid_count + ), + "preserved_decode_invalid_count": ( + upload_manifest_proof.preserved_decode_invalid_count + ), + "required_decode_invalid_object_count": ( + upload_manifest_proof.required_decode_invalid_object_count + ), + "required_decode_invalid_reference_count": ( + upload_manifest_proof.required_decode_invalid_reference_count + ), + "forensic_fallback_active": ( + upload_manifest_proof.required_decode_invalid_reference_count + > 0 + or ( + upload_database_proof is not None + and upload_database_proof.current_references.decode_invalid_reference_count + > 0 + ) + ), + "preserved_object_set_sha256": ( + upload_manifest_proof.preserved_object_set_sha256 + ), + "required_object_count": upload_manifest_proof.required_object_count, + "reference_set_sha256": upload_manifest_proof.reference_set_sha256, + "runtime_database_validated": upload_database_ok, + "current_object_count": ( + upload_database_proof.current_references.object_count + if upload_database_proof is not None + else None + ), + "current_reference_count": ( + upload_database_proof.current_references.reference_count + if upload_database_proof is not None + else None + ), + "current_reference_set_sha256": ( + upload_database_proof.current_references.reference_set_sha256 + if upload_database_proof is not None + else None + ), + "current_decode_invalid_object_count": ( + upload_database_proof.current_references.decode_invalid_object_count + if upload_database_proof is not None + else None + ), + "current_decode_invalid_reference_count": ( + upload_database_proof.current_references.decode_invalid_reference_count + if upload_database_proof is not None + else None + ), + } + if upload_manifest_proof is not None + else {"required": False, "validated": False} + ), } diff --git a/apps/api/app/routes/users.py b/apps/api/app/routes/users.py index 10454a5..46ccc97 100644 --- a/apps/api/app/routes/users.py +++ b/apps/api/app/routes/users.py @@ -4,8 +4,11 @@ from __future__ import annotations import secrets import time +import os +import uuid +from contextlib import contextmanager, nullcontext from pathlib import Path -from typing import Literal +from typing import Iterator, Literal import re @@ -27,12 +30,23 @@ from ..runtime_policy import require_runtime_fallback_allowed from ..services.phase3_kpi_contract import PREPOST_MEASURE_NAMES, PREPOST_TIMEPOINTS from ..services.support_tickets import support_ticket_fingerprint from ..services.voice import PRESET_RATE, PRESET_TO_OPENAI_VOICE +from ..upload_runtime import ( + assert_public_avatar_url_exists, + public_avatar_root, + upload_write_freeze_gate, +) +from ..upload_storage import ( + PUBLIC_AVATAR_MAX_BYTES, + UploadWriteFrozen, + assert_path_without_reparse, + inspect_public_avatar_image, +) router = APIRouter(prefix="/users", tags=["users"]) TERMS_VERSION = "terms-draft-2026-06-27" PRIVACY_VERSION = "privacy-draft-2026-06-27" -AVATAR_MAX_BYTES = 3 * 1024 * 1024 +AVATAR_MAX_BYTES = PUBLIC_AVATAR_MAX_BYTES AVATAR_CONTENT_TYPES = { "image/png": ("png", b"\x89PNG\r\n\x1a\n"), "image/jpeg": ("jpg", b"\xff\xd8\xff"), @@ -430,12 +444,61 @@ def _onboarding_required(managed) -> bool: def _upload_root() -> Path: - root = Path(settings.user_upload_dir) - if not root.is_absolute(): - root = Path.cwd() / root - avatar_root = root / "profile-avatars" - avatar_root.mkdir(parents=True, exist_ok=True) - return avatar_root + try: + assert_path_without_reparse(public_avatar_root, "public avatar upload root") + except ValueError as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail="profile_upload_storage_unavailable", + ) from exc + if not public_avatar_root.is_dir(): + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail="profile_upload_storage_unavailable", + ) + return public_avatar_root + + +@contextmanager +def _profile_avatar_write_lease() -> Iterator[None]: + try: + with upload_write_freeze_gate.write_lease(): + yield + except UploadWriteFrozen as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail="profile_upload_writes_frozen", + headers={"Retry-After": "30"}, + ) from exc + + +def _assert_public_avatar_url_exists(value: str | None) -> None: + if value is None or not value.startswith("/uploads/"): + return + try: + assert_public_avatar_url_exists(value) + except ValueError as exc: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail="avatar_upload_not_found", + ) from exc + + +def _write_new_avatar_atomically(target: Path, content: bytes) -> None: + assert_path_without_reparse(target.parent, "public avatar upload root") + if not target.parent.is_dir(): + raise ValueError("public avatar upload root is unavailable") + temporary = target.with_name(f".{target.name}.{secrets.token_hex(8)}.uploading") + try: + with temporary.open("xb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + # A hard-link publishes one fully flushed inode and fails rather than + # replacing an existing random-name target. + os.link(temporary, target, follow_symlinks=False) + finally: + temporary.unlink(missing_ok=True) def _validated_avatar_extension(content_type: str, content: bytes) -> str: @@ -453,9 +516,24 @@ def _validated_avatar_extension(content_type: str, content: bytes) -> str: ) elif not content.startswith(magic): raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="invalid_avatar_file") + if not inspect_public_avatar_image( + f"profile-avatars/upload.{ext}", + content, + ).valid: + raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="invalid_avatar_file") return ext +def _normalized_upload_user_id(value: str) -> str: + try: + return str(uuid.UUID(value)) + except (AttributeError, TypeError, ValueError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail="profile_upload_identity_unavailable", + ) from exc + + async def _profile_for(principal: CurrentPrincipal) -> UserProfileResponse: managed = await get_managed_user(principal.user_id) return UserProfileResponse( @@ -518,41 +596,58 @@ async def patch_me( body: UserProfilePatch, principal: CurrentPrincipal ) -> UserProfileResponse: profile = await _profile_for(principal) - updated = await update_managed_user( - principal.user_id, - ManagedUserPatch( - display_name=body.display_name - if body.display_name is not None - else profile.display_name, - affiliation=body.affiliation - if body.affiliation is not None - else profile.affiliation, - legal_name=body.legal_name - if body.legal_name is not None - else profile.legal_name, - department=body.department - if body.department is not None - else profile.department, - grade_level=body.grade_level - if body.grade_level is not None - else profile.grade_level, - phone=body.phone if body.phone is not None else profile.phone, - contact_address=( - body.contact_address - if body.contact_address is not None - else profile.contact_address + avatar_requested = body.avatar_url is not None + lease = _profile_avatar_write_lease() if avatar_requested else nullcontext() + with lease: + _assert_public_avatar_url_exists(body.avatar_url if avatar_requested else None) + updated = await update_managed_user( + principal.user_id, + ManagedUserPatch( + display_name=( + body.display_name + if body.display_name is not None + else profile.display_name + ), + affiliation=( + body.affiliation + if body.affiliation is not None + else profile.affiliation + ), + legal_name=( + body.legal_name + if body.legal_name is not None + else profile.legal_name + ), + department=( + body.department + if body.department is not None + else profile.department + ), + grade_level=( + body.grade_level + if body.grade_level is not None + else profile.grade_level + ), + phone=body.phone if body.phone is not None else profile.phone, + contact_address=( + body.contact_address + if body.contact_address is not None + else profile.contact_address + ), + nickname=( + body.nickname if body.nickname is not None else profile.nickname + ), + self_introduction=( + body.self_introduction + if body.self_introduction is not None + else profile.self_introduction + ), + # None is deliberate: update_managed_user uses COALESCE so an + # unrelated profile PATCH cannot restore the stale avatar read + # above over a concurrent avatar change. + avatar_url=body.avatar_url if avatar_requested else None, ), - nickname=body.nickname if body.nickname is not None else profile.nickname, - self_introduction=( - body.self_introduction - if body.self_introduction is not None - else profile.self_introduction - ), - avatar_url=body.avatar_url - if body.avatar_url is not None - else profile.avatar_url, - ), - ) + ) if updated is None: raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found") return await _profile_for(principal) @@ -563,34 +658,29 @@ async def upload_my_avatar( principal: CurrentPrincipal, file: UploadFile = File(...), ) -> AvatarUploadResponse: - content_type = (file.content_type or "").strip().lower() - content = await file.read(AVATAR_MAX_BYTES + 1) - await file.close() - if not content: - raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="empty_avatar_file") - if len(content) > AVATAR_MAX_BYTES: - raise HTTPException( - status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="avatar_too_large" + with _profile_avatar_write_lease(): + normalized_user_id = _normalized_upload_user_id(principal.user_id) + content_type = (file.content_type or "").strip().lower() + content = await file.read(AVATAR_MAX_BYTES + 1) + await file.close() + if not content: + raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="empty_avatar_file") + if len(content) > AVATAR_MAX_BYTES: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="avatar_too_large" + ) + ext = _validated_avatar_extension(content_type, content) + + root = _upload_root() + filename = f"{normalized_user_id}-{secrets.token_urlsafe(10)}.{ext}" + target = root / filename + _write_new_avatar_atomically(target, content) + avatar_url = f"/uploads/profile-avatars/{filename}" + return AvatarUploadResponse( + avatar_url=avatar_url, + content_type=content_type.split(";", 1)[0], + size_bytes=len(content), ) - ext = _validated_avatar_extension(content_type, content) - - root = _upload_root() - for existing in root.glob(f"{principal.user_id}-*.png"): - existing.unlink(missing_ok=True) - for existing in root.glob(f"{principal.user_id}-*.jpg"): - existing.unlink(missing_ok=True) - for existing in root.glob(f"{principal.user_id}-*.webp"): - existing.unlink(missing_ok=True) - - filename = f"{principal.user_id}-{secrets.token_urlsafe(10)}.{ext}" - target = root / filename - target.write_bytes(content) - avatar_url = f"/uploads/profile-avatars/{filename}" - return AvatarUploadResponse( - avatar_url=avatar_url, - content_type=content_type.split(";", 1)[0], - size_bytes=len(content), - ) @router.post("/me/onboarding", response_model=UserProfileResponse) @@ -604,24 +694,30 @@ async def complete_onboarding( raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="privacy_not_accepted") display_name = body.nickname.strip() - updated = await update_managed_user( - principal.user_id, - ManagedUserPatch( - display_name=display_name, - affiliation=body.affiliation, - legal_name=body.legal_name, - department=body.department, - grade_level=body.grade_level, - phone=body.phone, - contact_address=body.contact_address, - nickname=body.nickname, - self_introduction=body.self_introduction, - avatar_url=body.avatar_url, - complete_onboarding=True, - terms_version=TERMS_VERSION, - privacy_version=PRIVACY_VERSION, - ), - ) + # Onboarding always writes avatar_url (including clearing/replacing an + # existing upload), so the entire DB update participates in cutover drain. + with _profile_avatar_write_lease(): + _assert_public_avatar_url_exists( + body.avatar_url if body.avatar_url.startswith("/uploads/") else None + ) + updated = await update_managed_user( + principal.user_id, + ManagedUserPatch( + display_name=display_name, + affiliation=body.affiliation, + legal_name=body.legal_name, + department=body.department, + grade_level=body.grade_level, + phone=body.phone, + contact_address=body.contact_address, + nickname=body.nickname, + self_introduction=body.self_introduction, + avatar_url=body.avatar_url, + complete_onboarding=True, + terms_version=TERMS_VERSION, + privacy_version=PRIVACY_VERSION, + ), + ) if updated is None: raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found") diff --git a/apps/api/app/test_engine_health_contract.py b/apps/api/app/test_engine_health_contract.py index c144317..5695244 100644 --- a/apps/api/app/test_engine_health_contract.py +++ b/apps/api/app/test_engine_health_contract.py @@ -3,12 +3,130 @@ from __future__ import annotations import unittest +from types import SimpleNamespace from unittest.mock import AsyncMock, patch from . import main +from .upload_storage import UploadWriteFreezeStatus class EngineHealthContractTest(unittest.IsolatedAsyncioTestCase): + async def test_health_requires_and_exposes_lifespan_database_proof(self) -> None: + target_sha256 = "a" * 64 + manifest = SimpleNamespace( + manifest_sha256="b" * 64, + database_target_sha256=target_sha256, + preserved_object_count=93, + preserved_total_size_bytes=52973, + preserved_decode_valid_count=3, + preserved_decode_invalid_count=90, + required_decode_invalid_object_count=0, + required_decode_invalid_reference_count=0, + preserved_object_set_sha256="c" * 64, + required_object_count=8, + reference_set_sha256="d" * 64, + ) + runtime_proof = SimpleNamespace( + database_target_sha256=target_sha256, + current_references=SimpleNamespace( + object_count=8, + reference_count=8, + reference_set_sha256="e" * 64, + decode_invalid_object_count=6, + decode_invalid_reference_count=6, + ), + ) + with ( + patch.object(main, "upload_manifest_proof", manifest), + patch.object(main, "healthcheck", AsyncMock(return_value=True)), + patch.object( + main.engine_client, + "health_detail", + AsyncMock(return_value={"ok": True}), + ), + patch.object( + main.app.state, + "upload_database_proof", + runtime_proof, + create=True, + ), + ): + response = await main.health() + + self.assertEqual("ok", response["status"]) + self.assertTrue(response["upload_manifest"]["runtime_database_validated"]) + self.assertEqual(8, response["upload_manifest"]["current_object_count"]) + self.assertEqual( + 52973, + response["upload_manifest"]["preserved_total_size_bytes"], + ) + self.assertEqual(8, response["upload_manifest"]["current_reference_count"]) + self.assertTrue(response["upload_manifest"]["forensic_fallback_active"]) + self.assertEqual( + 6, + response["upload_manifest"]["current_decode_invalid_reference_count"], + ) + self.assertEqual( + "e" * 64, + response["upload_manifest"]["current_reference_set_sha256"], + ) + + async def test_health_is_degraded_for_invalid_active_upload_freeze(self) -> None: + engine_detail = {"ok": True, "detail": "ready"} + invalid_freeze = UploadWriteFreezeStatus( + capable=True, + active=True, + valid=False, + in_flight=0, + path_sha256="a" * 64, + token_sha256=None, + ) + with ( + patch.object(main, "healthcheck", AsyncMock(return_value=True)), + patch.object( + main.engine_client, + "health_detail", + AsyncMock(return_value=engine_detail), + ), + patch.object( + main.upload_write_freeze_gate, + "status", + return_value=invalid_freeze, + ), + ): + response = await main.health() + + self.assertEqual("degraded", response["status"]) + self.assertFalse(response["upload_write_freeze"]["valid"]) + + async def test_health_stays_ok_for_valid_active_maintenance_freeze(self) -> None: + engine_detail = {"ok": True, "detail": "ready"} + valid_freeze = UploadWriteFreezeStatus( + capable=True, + active=True, + valid=True, + in_flight=0, + path_sha256="a" * 64, + token_sha256="b" * 64, + ) + with ( + patch.object(main, "healthcheck", AsyncMock(return_value=True)), + patch.object( + main.engine_client, + "health_detail", + AsyncMock(return_value=engine_detail), + ), + patch.object( + main.upload_write_freeze_gate, + "status", + return_value=valid_freeze, + ), + ): + response = await main.health() + + self.assertEqual("ok", response["status"]) + self.assertTrue(response["upload_write_freeze"]["active"]) + async def test_health_is_degraded_when_live_client_lane_is_unready(self) -> None: engine_detail = { "ok": False, diff --git a/apps/api/app/test_upload_storage_contract.py b/apps/api/app/test_upload_storage_contract.py new file mode 100644 index 0000000..5e8e950 --- /dev/null +++ b/apps/api/app/test_upload_storage_contract.py @@ -0,0 +1,1230 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +import tempfile +import threading +import unittest +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +from PIL import Image +from starlette.datastructures import Headers, UploadFile + +from . import upload_storage as upload_storage_module +from .upload_storage import ( + UploadWriteFreezeGate, + UploadWriteFrozen, + assert_path_without_reparse, + build_privacy_safe_manifest, + database_target_sha256, + inspect_public_avatar_image, + is_runtime_generated_public_avatar_relative_path, + normalize_public_avatar_relative_path, + public_avatar_relative_path_sha256, + validate_current_avatar_references, + validate_upload_manifest, +) + +DATABASE_TARGET_SHA256 = hashlib.sha256(b"connected-database-target").hexdigest() + + +def _valid_png_bytes() -> bytes: + buffer = io.BytesIO() + Image.new("RGBA", (1, 1), (0, 0, 0, 0)).save(buffer, format="PNG") + return buffer.getvalue() + + +class _AsyncValueContext: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, traceback): + return False + + +class _RuntimeDatabaseConnection: + def __init__(self, *, avatar_urls: list[str], database_name: str = "vignette"): + self.avatar_urls = avatar_urls + self.database_name = database_name + self.transaction_options: list[dict[str, object]] = [] + self.executed: list[str] = [] + self.codec_names: list[str] = [] + + def transaction(self, **options): + self.transaction_options.append(options) + return _AsyncValueContext(self) + + async def fetchrow(self, _query: str): + return { + "database_name": self.database_name, + "database_role": "vignette_app", + "server_address": "10.0.0.5", + "server_port": 5432, + } + + async def execute(self, query: str): + self.executed.append(query) + + async def set_type_codec(self, name: str, **_options): + self.codec_names.append(name) + + async def fetch(self, _query: str): + return [{"avatar_url": value} for value in self.avatar_urls] + + +class _RuntimeDatabasePool: + def __init__(self, connection: _RuntimeDatabaseConnection): + self.connection = connection + self.acquire_count = 0 + + def acquire(self): + self.acquire_count += 1 + return _AsyncValueContext(self.connection) + + +class UploadStorageContractTest(unittest.TestCase): + def test_broken_png_crc_is_decode_invalid_not_an_initializer_error(self) -> None: + content = bytearray(_valid_png_bytes()) + idat = content.index(b"IDAT") + 4 + content[idat] ^= 0x01 + proof = inspect_public_avatar_image( + "profile-avatars/broken.png", + bytes(content), + ) + self.assertFalse(proof.valid) + + def test_api_and_release_probe_share_connected_database_identity_digest(self) -> None: + helper_path = ( + Path(__file__).resolve().parents[3] + / "scripts" + / "public_runtime_database_identity.py" + ) + spec = importlib.util.spec_from_file_location( + "public_runtime_database_identity_contract_test", + helper_path, + ) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader if spec is not None else None) + module = importlib.util.module_from_spec(spec) + assert spec is not None and spec.loader is not None + spec.loader.exec_module(module) + release_database_target_sha256 = module.database_target_sha256 + + identity = { + "database_name": "vignette", + "database_role": "vignette_app", + "server_address": "10.0.0.5", + "server_port": 5432, + } + self.assertEqual( + release_database_target_sha256(**identity), + database_target_sha256(**identity), + ) + + def test_manifest_proves_required_object_without_recording_its_path(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-manifest-") as raw: + root = Path(raw) / "uploads" + object_path = root / "profile-avatars" / "private-object.jpg" + object_path.parent.mkdir(parents=True) + object_path.write_bytes(b"avatar-bytes") + token_sha = hashlib.sha256(b"freeze-token").hexdigest() + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/private-object.jpg", "profile_avatar")], + preserved_paths=["profile-avatars/private-object.jpg"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=token_sha, + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn("private-object", serialized) + self.assertNotIn('"relative_path":', serialized) + + manifest_dir = Path(raw) / "private-state" / "manifests" + manifest_dir.mkdir(parents=True) + manifest = manifest_dir / "manifest.json" + manifest.write_text(serialized, encoding="utf-8") + expected_sha = hashlib.sha256(manifest.read_bytes()).hexdigest() + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=expected_sha, + expected_write_freeze_path=Path(raw) / "upload-write.freeze", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + self.assertEqual(1, proof.required_object_count) + public_manifest = root / "manifest.json" + public_manifest.write_text(serialized, encoding="utf-8") + public_manifest_sha = hashlib.sha256( + public_manifest.read_bytes() + ).hexdigest() + with self.assertRaisesRegex(ValueError, "outside the public upload root"): + validate_upload_manifest( + upload_root=root, + manifest_path=public_manifest, + expected_manifest_sha256=public_manifest_sha, + expected_write_freeze_path=Path(raw) / "upload-write.freeze", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + object_path.write_bytes(b"drift") + with self.assertRaisesRegex(ValueError, "content hash"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=expected_sha, + expected_write_freeze_path=Path(raw) / "upload-write.freeze", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + + def test_duplicate_db_references_are_aggregated_without_raw_paths(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-duplicates-") as raw: + root = Path(raw) / "uploads" + object_path = root / "profile-avatars" / "shared-avatar.png" + object_path.parent.mkdir(parents=True) + object_path.write_bytes(b"shared-avatar") + token_sha = hashlib.sha256(b"freeze-token").hexdigest() + payload = build_privacy_safe_manifest( + upload_root=root, + references=[ + ("profile-avatars/shared-avatar.png", "profile_avatar"), + ("profile-avatars/shared-avatar.png", "profile_avatar"), + ], + preserved_paths=["profile-avatars/shared-avatar.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=token_sha, + ) + self.assertEqual(1, payload["required_object_count"]) + self.assertEqual(2, payload["required_reference_count"]) + self.assertEqual( + 2, payload["preserved_objects"][0]["reference_count"] + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn("shared-avatar", serialized) + + manifest_dir = Path(raw) / "private-state" + manifest_dir.mkdir() + manifest = manifest_dir / "manifest.json" + manifest.write_text(serialized, encoding="utf-8") + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=hashlib.sha256( + manifest.read_bytes() + ).hexdigest(), + expected_write_freeze_path=Path(raw) / "write-freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + current = validate_current_avatar_references( + upload_root=root, + avatar_urls=[ + "/uploads/profile-avatars/shared-avatar.png", + "/uploads/profile-avatars/shared-avatar.png", + ], + initial_manifest=proof, + expected_database_target_sha256=DATABASE_TARGET_SHA256, + ) + self.assertEqual(1, current.object_count) + self.assertEqual(2, current.reference_count) + self.assertEqual(proof.reference_set_sha256, current.reference_set_sha256) + + def test_runtime_generated_current_references_redecode_and_count_corruption( + self, + ) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-current-decode-") as raw: + root = Path(raw) / "uploads" + avatar_root = root / "profile-avatars" + avatar_root.mkdir(parents=True) + preserved = avatar_root / "preserved.png" + preserved.write_bytes(_valid_png_bytes()) + payload = build_privacy_safe_manifest( + upload_root=root, + references=[], + preserved_paths=["profile-avatars/preserved.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=hashlib.sha256(b"freeze-token").hexdigest(), + ) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=hashlib.sha256( + manifest.read_bytes() + ).hexdigest(), + expected_write_freeze_path=private / "write-freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + + valid_name = ( + "8da1b8fc-3bdd-4e72-86f7-d17fc315c4cb-" + "AbCdEf_12345.png" + ) + corrupt_name = ( + "1265087b-d3d0-4e85-82bc-499d993a3f89-" + "QrStUv_67890.png" + ) + (avatar_root / valid_name).write_bytes(_valid_png_bytes()) + (avatar_root / corrupt_name).write_bytes(b"not-a-png") + + current = validate_current_avatar_references( + upload_root=root, + avatar_urls=[ + f"/uploads/profile-avatars/{valid_name}", + f"/uploads/profile-avatars/{corrupt_name}", + f"/uploads/profile-avatars/{corrupt_name}", + ], + initial_manifest=proof, + expected_database_target_sha256=DATABASE_TARGET_SHA256, + ) + + self.assertEqual(2, current.object_count) + self.assertEqual(3, current.reference_count) + self.assertEqual(1, current.decode_invalid_object_count) + self.assertEqual(2, current.decode_invalid_reference_count) + + arbitrary = avatar_root / "unbound.png" + arbitrary.write_bytes(_valid_png_bytes()) + with self.assertRaisesRegex(ValueError, "not authorized"): + validate_current_avatar_references( + upload_root=root, + avatar_urls=["/uploads/profile-avatars/unbound.png"], + initial_manifest=proof, + expected_database_target_sha256=DATABASE_TARGET_SHA256, + ) + + def test_manifest_v3_preserves_zero_ref_upload_and_binds_connected_db(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-preserved-v2-") as raw: + root = Path(raw) / "uploads" + avatar_root = root / "profile-avatars" + avatar_root.mkdir(parents=True) + referenced = avatar_root / "referenced.png" + pending = avatar_root / "pending.webp" + referenced.write_bytes(b"referenced") + pending.write_bytes(b"pending") + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/referenced.png", "profile_avatar")], + preserved_paths=[ + "profile-avatars/referenced.png", + "profile-avatars/pending.webp", + ], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=hashlib.sha256(b"freeze").hexdigest(), + ) + self.assertEqual( + "vignette.public-avatar-upload-manifest.v3", + payload["schema_version"], + ) + self.assertEqual(2, payload["preserved_object_count"]) + self.assertEqual(0, payload["preserved_decode_valid_count"]) + self.assertEqual(2, payload["preserved_decode_invalid_count"]) + self.assertEqual(1, payload["required_decode_invalid_object_count"]) + self.assertEqual( + 1, + payload["required_decode_invalid_reference_count"], + ) + self.assertEqual( + len(b"referenced") + len(b"pending"), + payload["preserved_total_size_bytes"], + ) + self.assertEqual(1, payload["required_object_count"]) + self.assertEqual( + [0, 1], + sorted( + int(record["reference_count"]) + for record in payload["preserved_objects"] + ), + ) + self.assertEqual( + 1, + payload["database_reference_contract"]["reference_count"], + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn("referenced.png", serialized) + self.assertNotIn("pending.webp", serialized) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(serialized, encoding="utf-8") + manifest_sha = hashlib.sha256(manifest.read_bytes()).hexdigest() + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + self.assertEqual(2, proof.preserved_object_count) + self.assertEqual(0, len(proof.preserved_records)) + self.assertEqual(2, proof.preserved_decode_invalid_count) + self.assertEqual( + len(b"referenced") + len(b"pending"), + proof.preserved_total_size_bytes, + ) + self.assertEqual(DATABASE_TARGET_SHA256, proof.database_target_sha256) + with self.assertRaisesRegex(ValueError, "database target binding"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=hashlib.sha256( + b"other-database" + ).hexdigest(), + require_freeze_path_binding=False, + ) + + (avatar_root / "unknown.txt").write_bytes(b"must-not-be-served") + with self.assertRaisesRegex(ValueError, "invalid extra"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + + def test_manifest_hash_and_json_parse_share_one_byte_read(self) -> None: + source = Path(__file__).with_name("upload_storage.py").read_text( + encoding="utf-8" + ) + validator = source[ + source.index("def validate_upload_manifest(") : + source.index("class CurrentAvatarReferenceProof") + ] + self.assertIn("manifest_bytes = manifest.read_bytes()", validator) + self.assertIn("_sha256_bytes(manifest_bytes)", validator) + self.assertIn('json.loads(manifest_bytes.decode("utf-8"))', validator) + self.assertNotIn("manifest.read_text", validator) + + def test_routine_manifest_validation_rehashes_preserved_and_allows_only_generated_extras( + self, + ) -> None: + with tempfile.TemporaryDirectory( + prefix="vignette-upload-routine-validation-" + ) as raw: + root = Path(raw) / "uploads" + avatar_root = root / "profile-avatars" + avatar_root.mkdir(parents=True) + preserved = avatar_root / "legacy-avatar.png" + preserved.write_bytes(_valid_png_bytes()) + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/legacy-avatar.png", "profile_avatar")], + preserved_paths=["profile-avatars/legacy-avatar.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=hashlib.sha256(b"freeze").hexdigest(), + ) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + manifest_sha256 = hashlib.sha256(manifest.read_bytes()).hexdigest() + generated_name = ( + "8da1b8fc-3bdd-4e72-86f7-d17fc315c4cb-" + "AbCdEf_12345.webp" + ) + (avatar_root / generated_name).write_bytes(b"new-runtime-upload") + + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha256, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + verify_preserved_objects=True, + reject_unbound_extras=False, + ) + self.assertEqual(1, len(proof.preserved_records)) + self.assertTrue( + is_runtime_generated_public_avatar_relative_path( + f"profile-avatars/{generated_name}" + ) + ) + + with self.assertRaisesRegex(ValueError, "unbound extra"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha256, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + verify_preserved_objects=True, + reject_unbound_extras=True, + ) + + arbitrary = avatar_root / "looks-valid-but-unbound.png" + arbitrary.write_bytes(b"unauthorized") + with self.assertRaisesRegex(ValueError, "unauthorized extra"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha256, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + verify_preserved_objects=True, + reject_unbound_extras=False, + ) + arbitrary.unlink() + preserved.write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "content hash"): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=manifest_sha256, + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + verify_preserved_objects=True, + reject_unbound_extras=False, + ) + + def test_preserved_immutable_cache_has_a_fail_closed_total_size_limit(self) -> None: + from . import upload_storage + + with tempfile.TemporaryDirectory(prefix="vignette-avatar-cache-cap-") as raw: + root = Path(raw) / "uploads" + avatar = root / "profile-avatars" / "legacy.png" + avatar.parent.mkdir(parents=True) + avatar.write_bytes(_valid_png_bytes()) + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/legacy.png", "profile_avatar")], + preserved_paths=["profile-avatars/legacy.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=hashlib.sha256(b"freeze").hexdigest(), + ) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + with ( + patch.object(upload_storage, "MAX_PRESERVED_AVATAR_CACHE_BYTES", 1), + self.assertRaisesRegex(ValueError, "cache size limit"), + ): + validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=hashlib.sha256( + manifest.read_bytes() + ).hexdigest(), + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + def test_public_avatar_normalizer_rejects_encoded_control_and_non_image_paths( + self, + ) -> None: + for value in ( + "profile-avatars/encoded%2fpath.png", + "profile-avatars/control\x01.png", + "profile-avatars/not-an-image.txt", + "profile-avatars/nested/file.png", + "profile-avatars/.hidden.png", + ): + with self.subTest(value=value), self.assertRaises(ValueError): + normalize_public_avatar_relative_path(value) + + def test_current_inventory_cannot_be_empty_after_nonempty_initialization( + self, + ) -> None: + with tempfile.TemporaryDirectory( + prefix="vignette-upload-empty-current-" + ) as raw: + root = Path(raw) / "uploads" + object_path = root / "profile-avatars" / "initial.png" + object_path.parent.mkdir(parents=True) + object_path.write_bytes(b"initial") + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/initial.png", "profile_avatar")], + preserved_paths=["profile-avatars/initial.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=hashlib.sha256(b"token").hexdigest(), + ) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=hashlib.sha256( + manifest.read_bytes() + ).hexdigest(), + expected_write_freeze_path=Path(raw) / "freeze.json", + expected_database_target_sha256=DATABASE_TARGET_SHA256, + require_freeze_path_binding=False, + ) + with self.assertRaisesRegex(ValueError, "explicit reset receipt"): + validate_current_avatar_references( + upload_root=root, + avatar_urls=[], + initial_manifest=proof, + expected_database_target_sha256=DATABASE_TARGET_SHA256, + ) + + def test_manifest_and_freeze_state_are_rejected_inside_public_root(self) -> None: + with tempfile.TemporaryDirectory( + prefix="vignette-upload-private-state-" + ) as raw: + root = Path(raw) / "uploads" + object_path = root / "profile-avatars" / "avatar.png" + object_path.parent.mkdir(parents=True) + object_path.write_bytes(b"avatar") + token_sha = hashlib.sha256(b"token").hexdigest() + with self.assertRaisesRegex(ValueError, "outside the public upload root"): + build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/avatar.png", "profile_avatar")], + preserved_paths=["profile-avatars/avatar.png"], + database_target_sha256=DATABASE_TARGET_SHA256, + write_freeze_token_sha256=token_sha, + write_freeze_path=root / ".private-freeze.json", + ) + + def test_static_mount_exposes_only_profile_avatar_directory(self) -> None: + source = Path(__file__).with_name("main.py").read_text(encoding="utf-8") + runtime_source = Path(__file__).with_name("upload_runtime.py").read_text( + encoding="utf-8" + ) + self.assertIn('app.mount(\n "/uploads/profile-avatars"', source) + self.assertIn("FlatPublicAvatarStaticFiles(directory=", source) + self.assertNotIn("\n StaticFiles(directory=", source) + self.assertNotIn('app.mount("/uploads",', source) + self.assertNotIn('name="uploads"', source) + self.assertIn( + "assert_path_without_reparse(\n public_avatar_root,", + runtime_source, + ) + + def test_public_avatar_root_reparse_is_rejected(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-avatar-reparse-") as raw: + root = Path(raw) / "uploads" + avatar_root = root / "profile-avatars" + avatar_root.mkdir(parents=True) + with patch.object( + upload_storage_module, + "_is_reparse_or_symlink", + side_effect=lambda candidate: candidate == avatar_root, + ): + with self.assertRaisesRegex(ValueError, "reparse point"): + assert_path_without_reparse( + avatar_root, + "initialized public avatar directory", + ) + + def test_write_gate_blocks_new_writes_and_counts_inflight_leases(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-freeze-") as raw: + freeze = Path(raw) / "upload-write.freeze" + gate = UploadWriteFreezeGate(freeze) + with gate.write_lease(): + self.assertEqual(1, gate.status().in_flight) + self.assertEqual(0, gate.status().in_flight) + freeze.write_text( + json.dumps( + { + "schema_version": "vignette.public-upload-write-freeze.v1", + "token": "opaque-token-0123456789abcdef0123456789", + } + ), + encoding="utf-8", + ) + with self.assertRaises(UploadWriteFrozen): + with gate.write_lease(): + pass + status = gate.status() + self.assertTrue(status.active) + self.assertTrue(status.valid) + self.assertEqual(0, status.in_flight) + + def test_write_gate_status_reads_sentinel_and_counter_under_one_lock(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-freeze-race-") as raw: + freeze = Path(raw) / "upload-write.freeze" + gate = UploadWriteFreezeGate(freeze) + status_read_started = threading.Event() + allow_status_read = threading.Event() + lease_entered = threading.Event() + statuses = [] + def blocking_entry_probe(candidate: Path) -> bool: + if candidate == freeze: + status_read_started.set() + if not allow_status_read.wait(2): + raise AssertionError("status sentinel read did not resume") + return False + return candidate.exists() + + def read_status() -> None: + statuses.append(gate.status()) + + def enter_lease() -> None: + with gate.write_lease(): + lease_entered.set() + + with patch.object( + upload_storage_module, + "_path_entry_exists_without_following", + side_effect=blocking_entry_probe, + ): + status_thread = threading.Thread(target=read_status) + status_thread.start() + self.assertTrue(status_read_started.wait(1)) + lease_thread = threading.Thread(target=enter_lease) + lease_thread.start() + self.assertFalse(lease_entered.wait(0.1)) + allow_status_read.set() + status_thread.join(2) + lease_thread.join(2) + + self.assertFalse(status_thread.is_alive()) + self.assertFalse(lease_thread.is_alive()) + self.assertEqual(1, len(statuses)) + self.assertEqual(0, statuses[0].in_flight) + self.assertTrue(lease_entered.is_set()) + + def test_unreadable_or_dangling_freeze_entry_blocks_writes(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-freeze-entry-") as raw: + freeze = Path(raw) / "dangling-freeze" + gate = UploadWriteFreezeGate(freeze) + with patch.object( + upload_storage_module, + "_path_entry_exists_without_following", + return_value=True, + ): + with self.assertRaises(UploadWriteFrozen): + with gate.write_lease(): + pass + status = gate.status() + self.assertTrue(status.active) + self.assertFalse(status.valid) + + +class RuntimeUploadDatabaseContractTest(unittest.IsolatedAsyncioTestCase): + @staticmethod + def _manifest_proof(raw: str): + root = Path(raw) / "uploads" + avatar = root / "profile-avatars" / "initial.png" + avatar.parent.mkdir(parents=True) + avatar.write_bytes(b"initial") + target_sha256 = database_target_sha256( + database_name="vignette", + database_role="vignette_app", + server_address="10.0.0.5", + server_port=5432, + ) + payload = build_privacy_safe_manifest( + upload_root=root, + references=[("profile-avatars/initial.png", "profile_avatar")], + preserved_paths=["profile-avatars/initial.png"], + database_target_sha256=target_sha256, + write_freeze_token_sha256=hashlib.sha256(b"freeze").hexdigest(), + ) + private = Path(raw) / "private" + private.mkdir() + manifest = private / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + proof = validate_upload_manifest( + upload_root=root, + manifest_path=manifest, + expected_manifest_sha256=hashlib.sha256(manifest.read_bytes()).hexdigest(), + expected_write_freeze_path=private / "freeze.json", + expected_database_target_sha256=target_sha256, + require_freeze_path_binding=False, + ) + return root, proof, target_sha256 + + async def test_lifespan_database_proof_uses_one_repeatable_read_pool_snapshot( + self, + ) -> None: + from . import upload_runtime + + with tempfile.TemporaryDirectory(prefix="vignette-runtime-db-proof-") as raw: + root, proof, target_sha256 = self._manifest_proof(raw) + connection = _RuntimeDatabaseConnection( + avatar_urls=["/uploads/profile-avatars/initial.png"] + ) + pool = _RuntimeDatabasePool(connection) + with ( + patch.object(upload_runtime, "upload_root", root), + patch.object(upload_runtime, "upload_manifest_proof", proof), + patch.object( + upload_runtime.settings, + "public_runtime_db_target_sha256", + target_sha256, + ), + ): + runtime_proof = ( + await upload_runtime.validate_runtime_upload_database_state(pool) + ) + + self.assertEqual(1, pool.acquire_count) + self.assertEqual( + [{"isolation": "repeatable_read", "readonly": True}], + connection.transaction_options, + ) + self.assertEqual(target_sha256, runtime_proof.database_target_sha256) + self.assertEqual(1, runtime_proof.current_references.object_count) + self.assertEqual(1, runtime_proof.current_references.reference_count) + self.assertTrue(connection.executed) + + main_source = Path(__file__).with_name("main.py").read_text(encoding="utf-8") + init_index = main_source.index("await init_pool()") + proof_index = main_source.index("await validate_runtime_upload_database_state") + first_mutation_index = main_source.index("await ensure_runtime_tables()") + self.assertLess(init_index, proof_index) + self.assertLess(proof_index, first_mutation_index) + + async def test_lifespan_database_proof_rejects_connected_target_drift(self) -> None: + from . import upload_runtime + + with tempfile.TemporaryDirectory(prefix="vignette-runtime-db-drift-") as raw: + root, proof, target_sha256 = self._manifest_proof(raw) + connection = _RuntimeDatabaseConnection( + avatar_urls=["/uploads/profile-avatars/initial.png"], + database_name="other_database", + ) + pool = _RuntimeDatabasePool(connection) + with ( + patch.object(upload_runtime, "upload_root", root), + patch.object(upload_runtime, "upload_manifest_proof", proof), + patch.object( + upload_runtime.settings, + "public_runtime_db_target_sha256", + target_sha256, + ), + ): + with self.assertRaisesRegex(RuntimeError, "database target drift"): + await upload_runtime.validate_runtime_upload_database_state(pool) + + async def test_every_new_pool_connection_is_bound_before_codec_setup(self) -> None: + from . import db + + target_sha256 = database_target_sha256( + database_name="vignette", + database_role="vignette_app", + server_address="10.0.0.5", + server_port=5432, + ) + connection = _RuntimeDatabaseConnection(avatar_urls=[]) + with ( + patch.object(db.settings, "user_upload_manifest_required", True), + patch.object( + db.settings, + "public_runtime_db_target_sha256", + target_sha256, + ), + ): + await db._init_connection(connection) + self.assertEqual(["jsonb", "json"], connection.codec_names) + + drifted = _RuntimeDatabaseConnection( + avatar_urls=[], database_name="other_database" + ) + with ( + patch.object(db.settings, "user_upload_manifest_required", True), + patch.object( + db.settings, + "public_runtime_db_target_sha256", + target_sha256, + ), + ): + with self.assertRaisesRegex(RuntimeError, "database target drift"): + await db._init_connection(drifted) + self.assertEqual([], drifted.codec_names) + + +class AvatarWriteSafetyTest(unittest.IsolatedAsyncioTestCase): + @staticmethod + def _profile(users, *, avatar_url: str): + return users.UserProfileResponse( + user_id="user-1", + email="user@example.test", + display_name="사용자", + role="learner", + cohort_ids=[], + affiliation="한신대학교", + avatar_url=avatar_url, + ) + + async def test_static_avatar_handler_rejects_nested_hidden_and_unknown_files( + self, + ) -> None: + from starlette.exceptions import HTTPException as StarletteHTTPException + + from . import upload_runtime + + with tempfile.TemporaryDirectory(prefix="vignette-avatar-static-") as raw: + upload_root = Path(raw) / "uploads" + root = upload_root / "profile-avatars" + root.mkdir(parents=True) + generated_name = ( + "8da1b8fc-3bdd-4e72-86f7-d17fc315c4cb-" + "AbCdEf_12345.png" + ) + valid = root / generated_name + valid_bytes = _valid_png_bytes() + valid.write_bytes(valid_bytes) + arbitrary = root / "avatar.png" + arbitrary.write_bytes(b"unbound") + nested = root / "nested" + nested.mkdir() + (nested / "avatar.png").write_bytes(b"nested") + (root / ".hidden.png").write_bytes(b"hidden") + (root / "notes.txt").write_bytes(b"private") + files = upload_runtime.FlatPublicAvatarStaticFiles(directory=str(root)) + scope = {"method": "GET", "headers": []} + with patch.object(upload_runtime, "upload_manifest_proof", None): + response = await files.get_response(generated_name, scope) + self.assertEqual(200, response.status_code) + self.assertEqual(valid_bytes, response.body) + self.assertEqual( + f'"{hashlib.sha256(valid_bytes).hexdigest()}"', + response.headers["etag"], + ) + with patch.object(upload_runtime, "upload_manifest_proof", None): + head = await files.get_response( + generated_name, + {"method": "HEAD", "headers": []}, + ) + self.assertEqual(b"", head.body) + self.assertEqual(str(len(valid_bytes)), head.headers["content-length"]) + with ( + patch.object(upload_runtime, "upload_root", upload_root), + patch.object(upload_runtime, "upload_manifest_proof", None), + ): + upload_runtime.assert_public_avatar_url_exists( + f"/uploads/profile-avatars/{generated_name}" + ) + with self.assertRaisesRegex(ValueError, "not authorized"): + upload_runtime.assert_public_avatar_url_exists( + "/uploads/profile-avatars/avatar.png" + ) + + valid.write_bytes(b"damaged-after-upload") + with patch.object(upload_runtime, "upload_manifest_proof", None): + with self.assertRaises(StarletteHTTPException) as damaged: + await files.get_response(generated_name, scope) + self.assertEqual(404, damaged.exception.status_code) + + corrupt_name = ( + "1265087b-d3d0-4e85-82bc-499d993a3f89-" + "QrStUv_67890.png" + ) + (root / corrupt_name).write_bytes(b"not-a-png") + with patch.object(upload_runtime, "upload_manifest_proof", None): + with self.assertRaises(StarletteHTTPException) as corrupt: + await files.get_response(corrupt_name, scope) + self.assertEqual(404, corrupt.exception.status_code) + + mismatch_name = ( + "02f01847-1d0c-48c8-9559-4ba791d00275-" + "MnOpQr_24680.png" + ) + mismatch_buffer = io.BytesIO() + Image.new("RGB", (1, 1), (255, 255, 255)).save( + mismatch_buffer, + format="JPEG", + ) + (root / mismatch_name).write_bytes(mismatch_buffer.getvalue()) + with patch.object(upload_runtime, "upload_manifest_proof", None): + with self.assertRaises(StarletteHTTPException) as mismatch: + await files.get_response(mismatch_name, scope) + self.assertEqual(404, mismatch.exception.status_code) + for invalid in ( + "avatar.png", + "nested/avatar.png", + ".hidden.png", + "notes.txt", + ): + with self.subTest(path=invalid): + with patch.object(upload_runtime, "upload_manifest_proof", None): + with self.assertRaises(StarletteHTTPException) as raised: + await files.get_response(invalid, scope) + self.assertEqual(404, raised.exception.status_code) + + legacy_name = "legacy-avatar.png" + legacy = root / legacy_name + legacy.write_bytes(b"legacy") + path_sha256 = public_avatar_relative_path_sha256( + f"profile-avatars/{legacy_name}" + ) + proof = SimpleNamespace( + preserved_records=( + ( + path_sha256, + hashlib.sha256(b"legacy").hexdigest(), + len(b"legacy"), + "image/png", + b"legacy", + ), + ), + preserved_path_decode_records=((path_sha256, True),), + ) + with patch.object(upload_runtime, "upload_manifest_proof", proof): + response = await files.get_response(legacy_name, scope) + self.assertEqual(200, response.status_code) + self.assertEqual(b"legacy", response.body) + self.assertEqual("6", response.headers["content-length"]) + self.assertEqual("image/png", response.headers["content-type"]) + self.assertEqual( + f'"{hashlib.sha256(b"legacy").hexdigest()}"', + response.headers["etag"], + ) + legacy.write_bytes(b"drift") + cached = await files.get_response(legacy_name, scope) + self.assertEqual(b"legacy", cached.body) + head = await files.get_response( + legacy_name, + {"method": "HEAD", "headers": []}, + ) + self.assertEqual(b"", head.body) + self.assertEqual("6", head.headers["content-length"]) + + forensic_name = "forensic-avatar.png" + forensic_path_sha256 = public_avatar_relative_path_sha256( + f"profile-avatars/{forensic_name}" + ) + (root / forensic_name).write_bytes(b"\x89PNG\r\n\x1a\ncorrupt") + forensic_proof = SimpleNamespace( + preserved_records=(), + preserved_path_decode_records=( + (forensic_path_sha256, False), + ), + ) + with patch.object( + upload_runtime, + "upload_manifest_proof", + forensic_proof, + ): + with self.assertRaises(StarletteHTTPException) as raised: + await files.get_response(forensic_name, scope) + self.assertEqual(404, raised.exception.status_code) + + async def test_upload_root_is_revalidated_immediately_before_write(self) -> None: + from fastapi import HTTPException + + from .routes import users + + with tempfile.TemporaryDirectory(prefix="vignette-avatar-root-drift-") as raw: + root = Path(raw) + with ( + patch.object(users, "public_avatar_root", root), + patch.object( + users, + "assert_path_without_reparse", + side_effect=ValueError("reparse point"), + ), + ): + with self.assertRaises(HTTPException) as raised: + users._upload_root() + self.assertEqual(503, raised.exception.status_code) + + target = root / "avatar.png" + with patch.object( + users, + "assert_path_without_reparse", + side_effect=ValueError("reparse point"), + ): + with self.assertRaisesRegex(ValueError, "reparse point"): + users._write_new_avatar_atomically(target, b"avatar") + self.assertFalse(target.exists()) + + async def test_unrelated_profile_patch_does_not_restore_stale_avatar(self) -> None: + from .deps import Principal, Role + from .routes import users + + principal = Principal( + user_id="user-1", + role=Role.LEARNER, + email="user@example.test", + display_name="사용자", + ) + stale_profile = self._profile( + users, avatar_url="/uploads/profile-avatars/stale.png" + ) + update = AsyncMock(return_value=object()) + with ( + patch.object(users, "_profile_for", AsyncMock(return_value=stale_profile)), + patch.object(users, "update_managed_user", update), + patch.object( + users, + "_profile_avatar_write_lease", + side_effect=AssertionError("avatar lease must not be needed"), + ), + ): + await users.patch_me(users.UserProfilePatch(display_name="새 이름"), principal) + + managed_patch = update.await_args.args[1] + self.assertIsNone(managed_patch.avatar_url) + + async def test_explicit_same_avatar_patch_still_uses_write_lease(self) -> None: + from .deps import Principal, Role + from .routes import users + + principal = Principal( + user_id="user-1", + role=Role.LEARNER, + email="user@example.test", + display_name="사용자", + ) + avatar_url = "/uploads/profile-avatars/current.png" + profile = self._profile(users, avatar_url=avatar_url) + lease = patch.object( + users, "_profile_avatar_write_lease", return_value=nullcontext() + ) + with ( + patch.object(users, "_profile_for", AsyncMock(return_value=profile)), + patch.object(users, "update_managed_user", AsyncMock(return_value=object())), + patch.object(users, "assert_public_avatar_url_exists", return_value=None), + lease as lease_mock, + ): + await users.patch_me(users.UserProfilePatch(avatar_url=avatar_url), principal) + + lease_mock.assert_called_once_with() + + async def test_onboarding_always_participates_in_avatar_write_freeze(self) -> None: + from .deps import Principal, Role + from .routes import users + + principal = Principal( + user_id="user-1", + role=Role.LEARNER, + email="user@example.test", + display_name="사용자", + consent_at=1.0, + ) + profile = self._profile(users, avatar_url="") + updated = SimpleNamespace( + display_name="학습자", profile_completed_at=1.0 + ) + lease = patch.object( + users, "_profile_avatar_write_lease", return_value=nullcontext() + ) + body = users.OnboardingRequest( + legal_name="학습자", + affiliation="한신대학교", + department="상담심리학과", + grade_level="3학년", + phone="010-1234-5678", + contact_address="경기도 오산시", + nickname="학습자", + self_introduction="소개", + avatar_url="", + terms_accepted=True, + privacy_accepted=True, + ) + with ( + patch.object(users, "_profile_for", AsyncMock(return_value=profile)), + patch.object(users, "update_managed_user", AsyncMock(return_value=updated)), + lease as lease_mock, + ): + await users.complete_onboarding(body, principal) + + lease_mock.assert_called_once_with() + + async def test_patch_failure_preserves_old_db_referenced_avatar(self) -> None: + from .deps import Principal, Role + from .routes import users + + with tempfile.TemporaryDirectory(prefix="vignette-avatar-write-") as raw: + root = Path(raw) + old = root / "user-1-old.png" + old.write_bytes(b"old-avatar") + user_id = "8da1b8fc-3bdd-4e72-86f7-d17fc315c4cb" + principal = Principal( + user_id=user_id, + role=Role.LEARNER, + email="user@example.test", + display_name="사용자", + ) + upload = UploadFile( + io.BytesIO(_valid_png_bytes()), + filename="avatar.png", + headers=Headers({"content-type": "image/png"}), + ) + with patch.object(users, "_upload_root", return_value=root): + uploaded = await users.upload_my_avatar(principal, upload) + + new_name = uploaded.avatar_url.rsplit("/", 1)[-1] + new_path = root / new_name + self.assertTrue(new_path.is_file()) + self.assertEqual(b"old-avatar", old.read_bytes()) + + profile = users.UserProfileResponse( + user_id="user-1", + email="user@example.test", + display_name="사용자", + role="learner", + cohort_ids=[], + affiliation="한신대학교", + avatar_url="/uploads/profile-avatars/user-1-old.png", + ) + with ( + patch.object(users, "_profile_for", AsyncMock(return_value=profile)), + patch.object( + users, + "update_managed_user", + AsyncMock(side_effect=RuntimeError("DB write failed")), + ), + patch.object( + users, "assert_public_avatar_url_exists", return_value=None + ), + ): + with self.assertRaisesRegex(RuntimeError, "DB write failed"): + await users.patch_me( + users.UserProfilePatch(avatar_url=uploaded.avatar_url), + principal, + ) + + self.assertEqual(b"old-avatar", old.read_bytes()) + self.assertTrue(new_path.is_file()) + + async def test_avatar_upload_rejects_non_uuid_principal_before_file_creation( + self, + ) -> None: + from fastapi import HTTPException + + from .deps import Principal, Role + from .routes import users + + with tempfile.TemporaryDirectory(prefix="vignette-avatar-user-id-") as raw: + root = Path(raw) + principal = Principal( + user_id="not-a-uuid", + role=Role.LEARNER, + email="user@example.test", + display_name="사용자", + ) + upload = UploadFile( + io.BytesIO(_valid_png_bytes()), + filename="avatar.png", + headers=Headers({"content-type": "image/png"}), + ) + with patch.object(users, "_upload_root", return_value=root): + with self.assertRaises(HTTPException) as raised: + await users.upload_my_avatar(principal, upload) + self.assertEqual(503, raised.exception.status_code) + self.assertEqual([], list(root.iterdir())) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/api/app/upload_runtime.py b/apps/api/app/upload_runtime.py new file mode 100644 index 0000000..cd9b705 --- /dev/null +++ b/apps/api/app/upload_runtime.py @@ -0,0 +1,228 @@ +"""Configured public-avatar storage boundary used by API routes and health.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.responses import Response +from starlette.staticfiles import StaticFiles + +from .config import settings +from .upload_storage import ( + CurrentAvatarReferenceProof, + PUBLIC_AVATAR_DIRECTORY, + UploadManifestProof, + UploadWriteFreezeGate, + assert_path_without_reparse, + assert_regular_file_without_reparse, + connected_database_target_sha256, + is_runtime_generated_public_avatar_relative_path, + public_avatar_url_to_relative_path, + public_avatar_relative_path_sha256, + read_decodable_public_avatar_object, + validate_current_avatar_references, + validate_upload_manifest, +) + + +def _configured_upload_root() -> Path: + configured = Path(settings.user_upload_dir) + if not configured.is_absolute(): + if settings.user_upload_manifest_required: + raise RuntimeError("public USER_UPLOAD_DIR must be absolute") + configured = Path.cwd() / configured + if settings.user_upload_manifest_required: + try: + assert_path_without_reparse(configured, "public USER_UPLOAD_DIR") + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + return configured.resolve(strict=False) + + +upload_root = _configured_upload_root() +upload_manifest_proof: UploadManifestProof | None = None + +if settings.user_upload_manifest_required: + if not upload_root.is_dir(): + raise RuntimeError("initialized public USER_UPLOAD_DIR does not exist") + if not settings.user_upload_manifest_path: + raise RuntimeError("USER_UPLOAD_MANIFEST_PATH is required") + if len(settings.user_upload_manifest_sha256) != 64: + raise RuntimeError("USER_UPLOAD_MANIFEST_SHA256 is required") + if not settings.user_upload_write_freeze_path: + raise RuntimeError("USER_UPLOAD_WRITE_FREEZE_PATH is required") + freeze_path = Path(settings.user_upload_write_freeze_path) + if not freeze_path.is_absolute(): + raise RuntimeError("USER_UPLOAD_WRITE_FREEZE_PATH must be absolute") + upload_manifest_proof = validate_upload_manifest( + upload_root=upload_root, + manifest_path=Path(settings.user_upload_manifest_path), + expected_manifest_sha256=settings.user_upload_manifest_sha256, + expected_write_freeze_path=freeze_path, + expected_database_target_sha256=settings.public_runtime_db_target_sha256, + verify_preserved_objects=True, + reject_unbound_extras=False, + ) +else: + upload_root.mkdir(parents=True, exist_ok=True) + +public_avatar_root = upload_root / PUBLIC_AVATAR_DIRECTORY +if settings.user_upload_manifest_required: + if not public_avatar_root.is_dir(): + raise RuntimeError("initialized public avatar directory does not exist") + try: + assert_path_without_reparse( + public_avatar_root, + "initialized public avatar directory", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc +else: + public_avatar_root.mkdir(parents=True, exist_ok=True) + +configured_freeze_path = ( + Path(settings.user_upload_write_freeze_path) + if settings.user_upload_write_freeze_path + else None +) +upload_write_freeze_gate = UploadWriteFreezeGate(configured_freeze_path) + + +@dataclass(frozen=True, slots=True) +class RuntimeUploadDatabaseProof: + database_target_sha256: str + current_references: CurrentAvatarReferenceProof + + +async def validate_runtime_upload_database_state( + pool: Any, +) -> RuntimeUploadDatabaseProof: + """Bind readiness to one repeatable-read snapshot from this API's pool.""" + + if upload_manifest_proof is None: + raise RuntimeError("public upload manifest proof is unavailable") + async with pool.acquire() as connection: + async with connection.transaction(isolation="repeatable_read", readonly=True): + database_target = await connected_database_target_sha256(connection) + if database_target != settings.public_runtime_db_target_sha256: + raise RuntimeError("connected public runtime database target drift") + await connection.execute( + "SELECT set_config('app.ai_context', '', true), " + "set_config('app.current_role', 'admin', true)" + ) + rows = await connection.fetch( + """ + SELECT avatar_url + FROM app.app_user + WHERE avatar_url IS NOT NULL + AND btrim(avatar_url) LIKE '/uploads/%' + ORDER BY avatar_url + """ + ) + current_references = validate_current_avatar_references( + upload_root=upload_root, + avatar_urls=[str(row["avatar_url"]) for row in rows], + initial_manifest=upload_manifest_proof, + expected_database_target_sha256=database_target, + ) + return RuntimeUploadDatabaseProof( + database_target_sha256=database_target, + current_references=current_references, + ) + + +def assert_public_avatar_url_exists(value: str) -> None: + if not value.startswith("/uploads/"): + return + relative_path = public_avatar_url_to_relative_path(value) + path_sha256 = public_avatar_relative_path_sha256(relative_path) + preserved_decode_by_path = dict( + upload_manifest_proof.preserved_path_decode_records + if upload_manifest_proof is not None + else () + ) + decode_valid = preserved_decode_by_path.get(path_sha256) + if decode_valid is False: + raise ValueError("public avatar URL object is forensic-only") + candidate = upload_root / Path(*relative_path.split("/")) + if decode_valid is True: + assert_regular_file_without_reparse( + candidate, + "public avatar URL object", + ) + return + if not is_runtime_generated_public_avatar_relative_path(relative_path): + raise ValueError("public avatar URL object is not authorized") + read_decodable_public_avatar_object( + relative_path=relative_path, + path=candidate, + label="public avatar URL object", + ) + + +class FlatPublicAvatarStaticFiles(StaticFiles): + """Serve only one validated public avatar image, never nested/hidden extras.""" + + async def get_response(self, path: str, scope): + method = str(scope.get("method", "GET")).upper() + if method not in {"GET", "HEAD"}: + raise StarletteHTTPException(status_code=405) + try: + normalized = public_avatar_url_to_relative_path( + f"/uploads/profile-avatars/{path}" + ) + filename = normalized.split("/", 1)[1] + path_sha256 = public_avatar_relative_path_sha256(normalized) + preserved_decode_by_path = dict( + upload_manifest_proof.preserved_path_decode_records + if upload_manifest_proof is not None + else () + ) + preserved_public_cache = { + record[0]: (record[1], record[2], record[3], record[4]) + for record in ( + upload_manifest_proof.preserved_records + if upload_manifest_proof is not None + else () + ) + } + decode_valid = preserved_decode_by_path.get(path_sha256) + if decode_valid is False: + raise ValueError("public avatar object is forensic-only") + preserved = preserved_public_cache.get(path_sha256) + if preserved is not None: + content_sha256, size_bytes, media_type, content = preserved + body = content if method == "GET" else b"" + return Response( + content=body, + media_type=media_type, + headers={ + "Content-Length": str(size_bytes), + "ETag": f'"{content_sha256}"', + "Cache-Control": "public, max-age=3600", + }, + ) + if decode_valid is True: + raise ValueError("validated public avatar cache is unavailable") + if not is_runtime_generated_public_avatar_relative_path(normalized): + raise ValueError("public avatar object is not authorized") + candidate = Path(self.directory) / filename + runtime_object = read_decodable_public_avatar_object( + relative_path=normalized, + path=candidate, + label="public avatar static object", + ) + except (OSError, ValueError) as exc: + raise StarletteHTTPException(status_code=404) from exc + return Response( + content=runtime_object.content if method == "GET" else b"", + media_type=runtime_object.media_type, + headers={ + "Content-Length": str(runtime_object.size_bytes), + "ETag": f'"{runtime_object.content_sha256}"', + "Cache-Control": "public, max-age=3600", + }, + ) diff --git a/apps/api/app/upload_storage.py b/apps/api/app/upload_storage.py new file mode 100644 index 0000000..89b5214 --- /dev/null +++ b/apps/api/app/upload_storage.py @@ -0,0 +1,1059 @@ +"""Public avatar storage manifest and upload-write freeze contracts. + +The public surface owns only ``profile-avatars/``. Private multimodal +audio is deliberately excluded from both manifests and static serving. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import stat +import threading +import warnings +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Iterator, Sequence + +from PIL import Image, UnidentifiedImageError, __version__ as PILLOW_VERSION + +MANIFEST_SCHEMA_VERSION = "vignette.public-avatar-upload-manifest.v3" +WRITE_FREEZE_SCHEMA_VERSION = "vignette.public-upload-write-freeze.v1" +PUBLIC_AVATAR_DIRECTORY = "profile-avatars" +PUBLIC_AVATAR_MAX_BYTES = 3 * 1024 * 1024 +MAX_PRESERVED_AVATAR_CACHE_BYTES = 64 * 1024 * 1024 +IMAGE_DECODE_CONTRACT_VERSION = "vignette.public-avatar-image-decode.v1" +_SHA256_LENGTH = 64 +_PUBLIC_AVATAR_BASENAME = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9._-]{0,239}\.(?:png|jpe?g|webp)$", + re.IGNORECASE, +) +_RUNTIME_GENERATED_AVATAR_BASENAME = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + r"-[A-Za-z0-9_-]{8,32}\.(?:png|jpg|jpeg|webp)$" +) +_IMAGE_FORMAT_BY_SUFFIX = { + ".png": "PNG", + ".jpg": "JPEG", + ".jpeg": "JPEG", + ".webp": "WEBP", +} +DATABASE_IDENTITY_QUERY = """ +SELECT + current_database()::text AS database_name, + current_user::text AS database_role, + COALESCE(inet_server_addr()::text, 'local') AS server_address, + inet_server_port() AS server_port +""" + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256_text(value: str) -> str: + return _sha256_bytes(value.encode("utf-8")) + + +def database_target_sha256( + *, + database_name: str, + database_role: str, + server_address: str, + server_port: int, +) -> str: + """Hash the same credential-free connected-DB identity as release probes.""" + + if ( + not isinstance(database_name, str) + or not database_name.strip() + or not isinstance(database_role, str) + or not database_role.strip() + or not isinstance(server_address, str) + or not server_address.strip() + ): + raise ValueError("database target identity is incomplete") + if ( + not isinstance(server_port, int) + or isinstance(server_port, bool) + or server_port < 1 + or server_port > 65535 + ): + raise ValueError("database target port is invalid") + canonical = json.dumps( + { + "database": database_name, + "database_role": database_role, + "server_address": server_address, + "server_port": server_port, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + return _sha256_text(canonical) + + +async def connected_database_target_sha256(connection: Any) -> str: + target = await connection.fetchrow(DATABASE_IDENTITY_QUERY) + if target is None: + raise ValueError("database target identity is unavailable") + return database_target_sha256( + database_name=str(target["database_name"]), + database_role=str(target["database_role"]), + server_address=str(target["server_address"]), + server_port=int(target["server_port"]), + ) + + +def sha256_file(path: Path) -> str: + assert_regular_file_without_reparse(path, "public avatar object") + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class PublicAvatarImageInvalid(ValueError): + """The authorized file exists, but cannot be served as a public avatar.""" + + +def _read_regular_file_bytes( + path: Path, + label: str, + *, + max_bytes: int | None = None, +) -> bytes: + """Read one already-authorized inode once for size/hash/cache binding.""" + + assert_regular_file_without_reparse(path, label) + try: + with path.open("rb") as stream: + metadata = os.fstat(stream.fileno()) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} must be a regular non-reparse file") + if max_bytes is not None and metadata.st_size > max_bytes: + raise PublicAvatarImageInvalid(f"{label} exceeds the size limit") + content = stream.read(max_bytes + 1 if max_bytes is not None else -1) + if max_bytes is not None and len(content) > max_bytes: + raise PublicAvatarImageInvalid(f"{label} exceeds the size limit") + return content + except OSError as exc: + raise ValueError(f"{label} could not be read") from exc + + +def _public_avatar_media_type(relative_path: str) -> str: + suffix = PurePosixPath(relative_path).suffix.casefold() + return { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", + }[suffix] + + +def canonical_path_identity(path: Path) -> str: + return str(path.resolve(strict=False)).replace("\\", "/").casefold() + + +def canonical_path_sha256(path: Path) -> str: + return _sha256_text(canonical_path_identity(path)) + + +def _assert_outside_root(path: Path, root: Path, label: str) -> None: + try: + path.resolve(strict=False).relative_to(root.resolve(strict=True)) + except ValueError: + return + raise ValueError(f"{label} must be outside the public upload root") + + +def _is_reparse_or_symlink(path: Path) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + return False + if stat.S_ISLNK(metadata.st_mode): + return True + reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(getattr(metadata, "st_file_attributes", 0) & reparse_attribute) + + +def _path_entry_exists_without_following(path: Path) -> bool: + try: + path.lstat() + except FileNotFoundError: + return False + except OSError: + # An unreadable directory entry is not equivalent to a proven absence. + return True + return True + + +def assert_path_without_reparse( + path: Path, + label: str, + *, + require_absolute: bool = True, +) -> None: + if require_absolute and not path.is_absolute(): + raise ValueError(f"{label} must be absolute") + absolute = Path(os.path.abspath(path)) + for candidate in (absolute, *absolute.parents): + try: + candidate.lstat() + except FileNotFoundError: + continue + except OSError as exc: + raise ValueError(f"{label} path identity is unreadable") from exc + if _is_reparse_or_symlink(candidate): + raise ValueError(f"{label} cannot traverse a symlink or reparse point") + + +def assert_regular_file_without_reparse(path: Path, label: str) -> None: + assert_path_without_reparse(path, label) + try: + metadata = path.lstat() + except FileNotFoundError as exc: + raise ValueError(f"{label} is missing") from exc + if _is_reparse_or_symlink(path) or not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} must be a regular non-reparse file") + + +def _is_lower_hex_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == _SHA256_LENGTH + and all(character in "0123456789abcdef" for character in value) + ) + + +def normalize_public_avatar_relative_path(value: str) -> str: + if not isinstance(value, str): + raise ValueError("public avatar reference must be text") + if "%" in value or any( + ord(character) < 32 or ord(character) == 127 for character in value + ): + raise ValueError("public avatar reference contains forbidden characters") + normalized = value.replace("\\", "/").strip("/") + path = PurePosixPath(normalized) + if ( + path.is_absolute() + or len(path.parts) != 2 + or path.parts[0] != PUBLIC_AVATAR_DIRECTORY + or path.parts[1] in {"", ".", ".."} + or any(part in {"", ".", ".."} for part in path.parts) + or not _PUBLIC_AVATAR_BASENAME.fullmatch(path.parts[1]) + ): + raise ValueError( + "public avatar reference must be a flat profile-avatars image path" + ) + return path.as_posix() + + +def public_avatar_relative_path_sha256(value: str) -> str: + return _sha256_text(normalize_public_avatar_relative_path(value).casefold()) + + +def public_avatar_url_to_relative_path(value: str) -> str: + prefix = "/uploads/" + if not value.startswith(prefix) or "?" in value or "#" in value: + raise ValueError("public avatar URL must be an exact /uploads/ path") + return normalize_public_avatar_relative_path(value[len(prefix) :]) + + +@dataclass(frozen=True, slots=True) +class AvatarImageDecodeProof: + valid: bool + image_format: str | None + + +def inspect_public_avatar_image( + relative_path: str, + content: bytes, +) -> AvatarImageDecodeProof: + """Fully decode one public avatar and bind its format to the suffix. + + Legacy bytes stay preservable even when invalid. Callers use this proof to + keep those bytes forensic-only instead of returning them as public images. + """ + + normalized = normalize_public_avatar_relative_path(relative_path) + expected_format = _IMAGE_FORMAT_BY_SUFFIX[PurePosixPath(normalized).suffix.lower()] + if not isinstance(content, bytes) or not content: + return AvatarImageDecodeProof(valid=False, image_format=None) + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(content)) as candidate: + verified_format = candidate.format + candidate.verify() + with Image.open(io.BytesIO(content)) as candidate: + loaded_format = candidate.format + candidate.load() + image_format = str(loaded_format or verified_format or "").upper() or None + return AvatarImageDecodeProof( + valid=( + image_format == expected_format + and str(verified_format or "").upper() == expected_format + ), + image_format=image_format, + ) + except ( + OSError, + SyntaxError, + UnidentifiedImageError, + ValueError, + Warning, + Image.DecompressionBombError, + ): + return AvatarImageDecodeProof(valid=False, image_format=None) + + +@dataclass(frozen=True, slots=True) +class PublicAvatarObjectProof: + content: bytes + content_sha256: str + size_bytes: int + media_type: str + + +def read_decodable_public_avatar_object( + *, + relative_path: str, + path: Path, + label: str, +) -> PublicAvatarObjectProof: + """Bind one regular file read to its full image decode and response metadata.""" + + normalized = normalize_public_avatar_relative_path(relative_path) + content = _read_regular_file_bytes( + path, + label, + max_bytes=PUBLIC_AVATAR_MAX_BYTES, + ) + if not inspect_public_avatar_image(normalized, content).valid: + raise PublicAvatarImageInvalid(f"{label} is not a decodable image") + return PublicAvatarObjectProof( + content=content, + content_sha256=_sha256_bytes(content), + size_bytes=len(content), + media_type=_public_avatar_media_type(normalized), + ) + + +def is_runtime_generated_public_avatar_relative_path(value: str) -> bool: + """Return true only for the server's UUID + random-token filename shape.""" + + try: + normalized = normalize_public_avatar_relative_path(value) + except ValueError: + return False + return _RUNTIME_GENERATED_AVATAR_BASENAME.fullmatch( + PurePosixPath(normalized).name + ) is not None + + +def _object_record( + upload_root: Path, + relative_path: str, + kind: str, + reference_count: int, +) -> dict[str, object]: + normalized = normalize_public_avatar_relative_path(relative_path) + target = upload_root / Path(*PurePosixPath(normalized).parts) + assert_regular_file_without_reparse(target, "required public avatar object") + if kind != "profile_avatar": + raise ValueError("manifest can contain only public profile avatars") + content = _read_regular_file_bytes(target, "required public avatar object") + decode_proof = inspect_public_avatar_image(normalized, content) + return { + "path_sha256": public_avatar_relative_path_sha256(normalized), + "content_sha256": _sha256_bytes(content), + "size_bytes": len(content), + "kind": kind, + "reference_count": reference_count, + "decode_valid": decode_proof.valid, + } + + +def _aggregate_references( + references: Sequence[tuple[str, str]], +) -> list[tuple[str, str, int]]: + counts: dict[str, int] = {} + canonical: dict[str, str] = {} + for relative_path, kind in references: + if kind != "profile_avatar": + raise ValueError("manifest can contain only public profile avatars") + normalized = normalize_public_avatar_relative_path(relative_path) + key = normalized.casefold() + canonical.setdefault(key, normalized) + counts[key] = counts.get(key, 0) + 1 + return sorted( + ((canonical[key], "profile_avatar", counts[key]) for key in counts), + key=lambda item: public_avatar_relative_path_sha256(item[0]), + ) + + +def _reference_records_from_objects( + records: Sequence[dict[str, object]], +) -> list[dict[str, object]]: + return sorted( + ( + { + "path_sha256": str(record["path_sha256"]), + "reference_count": int(record["reference_count"]), + } + for record in records + ), + key=lambda item: str(item["path_sha256"]), + ) + + +def _reference_set_sha256(records: Sequence[dict[str, object]]) -> str: + serialized = json.dumps(list(records), sort_keys=True, separators=(",", ":")) + return _sha256_text(serialized) + + +def _preserved_object_set_sha256(records: Sequence[dict[str, object]]) -> str: + identity_records = [ + { + "path_sha256": str(record["path_sha256"]), + "content_sha256": str(record["content_sha256"]), + "size_bytes": int(record["size_bytes"]), + "kind": str(record["kind"]), + } + for record in records + ] + serialized = json.dumps( + identity_records, + sort_keys=True, + separators=(",", ":"), + ) + return _sha256_text(serialized) + + +def build_privacy_safe_manifest( + *, + upload_root: Path, + references: Sequence[tuple[str, str]], + preserved_paths: Sequence[str], + database_target_sha256: str, + write_freeze_token_sha256: str, + write_freeze_path: Path | None = None, +) -> dict[str, object]: + assert_path_without_reparse(upload_root, "public upload root") + root = upload_root.resolve(strict=True) + aggregated = _aggregate_references(references) + reference_counts = { + relative_path.casefold(): reference_count + for relative_path, _kind, reference_count in aggregated + } + canonical_preserved: dict[str, str] = {} + for raw_path in preserved_paths: + normalized = normalize_public_avatar_relative_path(raw_path) + key = normalized.casefold() + if key in canonical_preserved: + raise ValueError("preserved public avatar path is duplicated") + canonical_preserved[key] = normalized + if not canonical_preserved: + raise ValueError("preserved public avatar inventory cannot be empty") + if not set(reference_counts).issubset(canonical_preserved): + raise ValueError("database avatar reference is absent from preserved inventory") + records = sorted( + ( + _object_record( + root, + relative_path, + "profile_avatar", + reference_counts.get(key, 0), + ) + for key, relative_path in canonical_preserved.items() + ), + key=lambda record: str(record["path_sha256"]), + ) + database_records = _reference_records_from_objects( + [record for record in records if int(record["reference_count"]) > 0] + ) + preserved_decode_valid_count = sum( + 1 for record in records if bool(record["decode_valid"]) + ) + required_decode_invalid_records = [ + record + for record in records + if int(record["reference_count"]) > 0 + and not bool(record["decode_valid"]) + ] + if not _is_lower_hex_sha256(write_freeze_token_sha256): + raise ValueError("write freeze token SHA256 is invalid") + if not _is_lower_hex_sha256(database_target_sha256): + raise ValueError("database target SHA256 is invalid") + freeze: dict[str, object] = {"token_sha256": write_freeze_token_sha256} + if write_freeze_path is not None: + if not write_freeze_path.is_absolute(): + raise ValueError("upload write-freeze path must be absolute") + assert_path_without_reparse(write_freeze_path, "upload write-freeze path") + _assert_outside_root(write_freeze_path, root, "upload write-freeze path") + freeze["path_sha256"] = canonical_path_sha256(write_freeze_path) + return { + "schema_version": MANIFEST_SCHEMA_VERSION, + "status": "initialized_complete", + "root_path_sha256": canonical_path_sha256(root), + "database_target_sha256": database_target_sha256, + "preserved_object_count": len(records), + "preserved_total_size_bytes": sum( + int(record["size_bytes"]) for record in records + ), + "image_decode_contract": { + "schema_version": IMAGE_DECODE_CONTRACT_VERSION, + "decoder": "Pillow", + "decoder_version": PILLOW_VERSION, + "mode": "verify_load_extension_match", + }, + "preserved_decode_valid_count": preserved_decode_valid_count, + "preserved_decode_invalid_count": len(records) + - preserved_decode_valid_count, + "required_decode_invalid_object_count": len( + required_decode_invalid_records + ), + "required_decode_invalid_reference_count": sum( + int(record["reference_count"]) + for record in required_decode_invalid_records + ), + "preserved_object_set_sha256": _preserved_object_set_sha256(records), + "preserved_objects": records, + "required_object_count": len(database_records), + "required_reference_count": sum( + int(record["reference_count"]) for record in records + ), + "reference_set_sha256": _reference_set_sha256(database_records), + "database_reference_contract": { + "reference_count": sum( + int(record["reference_count"]) for record in records + ), + "unique_object_count": len(database_records), + "reference_set_sha256": _reference_set_sha256(database_records), + "objects": database_records, + }, + "write_freeze": freeze, + "privacy": { + "raw_relative_paths_recorded": False, + "raw_user_ids_recorded": False, + "raw_filenames_recorded": False, + "emails_recorded": False, + }, + } + + +@dataclass(frozen=True, slots=True) +class UploadManifestProof: + manifest_sha256: str + root_path_sha256: str + database_target_sha256: str + preserved_object_count: int + preserved_total_size_bytes: int + preserved_decode_valid_count: int + preserved_decode_invalid_count: int + required_decode_invalid_object_count: int + required_decode_invalid_reference_count: int + preserved_object_set_sha256: str + required_object_count: int + required_reference_count: int + reference_set_sha256: str + write_freeze_token_sha256: str + preserved_records: tuple[tuple[str, str, int, str, bytes], ...] + preserved_path_decode_records: tuple[tuple[str, bool], ...] + reference_records: tuple[tuple[str, int], ...] + + +def _iter_keys(value: object) -> Iterable[str]: + if isinstance(value, dict): + for key, child in value.items(): + yield str(key) + yield from _iter_keys(child) + elif isinstance(value, list): + for child in value: + yield from _iter_keys(child) + + +def validate_upload_manifest( + *, + upload_root: Path, + manifest_path: Path, + expected_manifest_sha256: str, + expected_write_freeze_path: Path, + expected_database_target_sha256: str, + require_freeze_path_binding: bool = True, + verify_preserved_objects: bool = True, + reject_unbound_extras: bool = True, +) -> UploadManifestProof: + if not upload_root.is_absolute(): + raise ValueError("public upload root must be absolute") + if not manifest_path.is_absolute(): + raise ValueError("upload manifest path must be absolute") + if not expected_write_freeze_path.is_absolute(): + raise ValueError("upload write-freeze path must be absolute") + assert_path_without_reparse(upload_root, "public upload root") + assert_path_without_reparse(manifest_path, "upload manifest path") + assert_path_without_reparse( + expected_write_freeze_path, + "upload write-freeze path", + ) + root = upload_root.resolve(strict=True) + manifest = manifest_path.resolve(strict=True) + assert_regular_file_without_reparse(manifest, "upload manifest") + _assert_outside_root(manifest, root, "upload manifest") + _assert_outside_root( + expected_write_freeze_path, + root, + "upload write-freeze path", + ) + try: + manifest_bytes = manifest.read_bytes() + except OSError as exc: + raise ValueError("upload manifest could not be read") from exc + actual_manifest_sha256 = _sha256_bytes(manifest_bytes) + if not _is_lower_hex_sha256(expected_manifest_sha256): + raise ValueError("expected upload manifest SHA256 is invalid") + if actual_manifest_sha256 != expected_manifest_sha256: + raise ValueError("upload manifest SHA256 drift") + try: + payload = json.loads(manifest_bytes.decode("utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("upload manifest is not valid UTF-8 JSON") from exc + if not isinstance(payload, dict): + raise ValueError("upload manifest must be a JSON object") + forbidden_keys = { + "relative_path", + "source_path", + "avatar_url", + "user_id", + "email", + "filename", + } + if forbidden_keys.intersection(key.casefold() for key in _iter_keys(payload)): + raise ValueError("upload manifest contains a privacy-sensitive key") + if payload.get("schema_version") != MANIFEST_SCHEMA_VERSION: + raise ValueError("upload manifest schema version mismatch") + if payload.get("status") != "initialized_complete": + raise ValueError("upload manifest is not initialized and complete") + root_sha256 = canonical_path_sha256(root) + if payload.get("root_path_sha256") != root_sha256: + raise ValueError("upload manifest root binding mismatch") + if ( + not _is_lower_hex_sha256(expected_database_target_sha256) + or payload.get("database_target_sha256") + != expected_database_target_sha256 + ): + raise ValueError("upload manifest database target binding mismatch") + if "objects" in payload: + raise ValueError("legacy upload manifest object contract is rejected") + preserved_objects = payload.get("preserved_objects") + if not isinstance(preserved_objects, list): + raise ValueError("upload manifest preserved objects must be an array") + preserved_count = payload.get("preserved_object_count") + if ( + not isinstance(preserved_count, int) + or preserved_count <= 0 + or preserved_count != len(preserved_objects) + ): + raise ValueError("upload manifest preserved object count mismatch") + preserved_total_size_bytes = payload.get("preserved_total_size_bytes") + if ( + not isinstance(preserved_total_size_bytes, int) + or isinstance(preserved_total_size_bytes, bool) + or preserved_total_size_bytes < 0 + ): + raise ValueError("upload manifest preserved total size is invalid") + expected_decode_contract = { + "schema_version": IMAGE_DECODE_CONTRACT_VERSION, + "decoder": "Pillow", + "decoder_version": PILLOW_VERSION, + "mode": "verify_load_extension_match", + } + if payload.get("image_decode_contract") != expected_decode_contract: + raise ValueError("upload manifest image decode contract mismatch") + decode_count_names = ( + "preserved_decode_valid_count", + "preserved_decode_invalid_count", + "required_decode_invalid_object_count", + "required_decode_invalid_reference_count", + ) + decode_counts: dict[str, int] = {} + for name in decode_count_names: + value = payload.get(name) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError("upload manifest image decode count is invalid") + decode_counts[name] = value + required_count = payload.get("required_object_count") + if ( + not isinstance(required_count, int) + or required_count < 0 + or required_count > preserved_count + ): + raise ValueError("upload manifest database object count mismatch") + required_reference_count = payload.get("required_reference_count") + if ( + not isinstance(required_reference_count, int) + or required_reference_count < 0 + or required_reference_count < required_count + ): + raise ValueError("upload manifest reference count is invalid") + freeze = payload.get("write_freeze") + if not isinstance(freeze, dict): + raise ValueError("upload manifest write-freeze binding is missing") + token_sha256 = freeze.get("token_sha256") + if not _is_lower_hex_sha256(token_sha256): + raise ValueError("upload manifest write-freeze token binding is invalid") + if require_freeze_path_binding: + if freeze.get("path_sha256") != canonical_path_sha256( + expected_write_freeze_path + ): + raise ValueError("upload manifest write-freeze path binding mismatch") + + available: dict[str, tuple[str, Path]] = {} + avatar_root = root / PUBLIC_AVATAR_DIRECTORY + if verify_preserved_objects or reject_unbound_extras: + if not avatar_root.is_dir() or _is_reparse_or_symlink(avatar_root): + raise ValueError("public avatar directory must be a non-reparse directory") + for candidate in avatar_root.iterdir(): + if _is_reparse_or_symlink(candidate) or not candidate.is_file(): + raise ValueError("public avatar directory contains an unsafe extra") + try: + relative = normalize_public_avatar_relative_path( + f"{PUBLIC_AVATAR_DIRECTORY}/{candidate.name}" + ) + except ValueError as exc: + raise ValueError( + "public avatar directory contains an invalid extra" + ) from exc + path_digest = public_avatar_relative_path_sha256(relative) + if path_digest in available: + raise ValueError("public avatar directory contains a duplicate path") + available[path_digest] = (relative, candidate) + + normalized_records: list[dict[str, object]] = [] + preserved_cache: list[tuple[str, str, int, str, bytes]] = [] + preserved_cache_bytes = 0 + seen: set[str] = set() + for item in preserved_objects: + if not isinstance(item, dict): + raise ValueError("upload manifest object entry must be an object") + path_sha256 = item.get("path_sha256") + content_sha256 = item.get("content_sha256") + size_bytes = item.get("size_bytes") + reference_count = item.get("reference_count") + decode_valid = item.get("decode_valid") + if not _is_lower_hex_sha256(path_sha256) or path_sha256 in seen: + raise ValueError("upload manifest path hash is invalid or duplicated") + if not _is_lower_hex_sha256(content_sha256): + raise ValueError("upload manifest content hash is invalid") + if not isinstance(size_bytes, int) or size_bytes < 0: + raise ValueError("upload manifest object size is invalid") + if not isinstance(reference_count, int) or reference_count < 0: + raise ValueError("upload manifest object reference count is invalid") + if not isinstance(decode_valid, bool): + raise ValueError("upload manifest object decode status is invalid") + if item.get("kind") != "profile_avatar": + raise ValueError("upload manifest contains a non-public object kind") + if verify_preserved_objects: + available_record = available.get(path_sha256) + if available_record is None: + raise ValueError("preserved public avatar object is missing") + _relative_path, target = available_record + assert_regular_file_without_reparse( + target, + "preserved public avatar object", + ) + content = _read_regular_file_bytes( + target, + "preserved public avatar object", + ) + if len(content) != size_bytes or _sha256_bytes(content) != content_sha256: + raise ValueError( + "preserved public avatar object content hash or size drift" + ) + actual_decode_valid = inspect_public_avatar_image( + _relative_path, + content, + ).valid + if actual_decode_valid is not decode_valid: + raise ValueError("preserved public avatar decode status drift") + if decode_valid: + preserved_cache_bytes += len(content) + if preserved_cache_bytes > MAX_PRESERVED_AVATAR_CACHE_BYTES: + raise ValueError( + "preserved public avatar cache size limit exceeded" + ) + preserved_cache.append( + ( + str(path_sha256), + str(content_sha256), + int(size_bytes), + _public_avatar_media_type(_relative_path), + content, + ) + ) + seen.add(path_sha256) + normalized_records.append( + { + "path_sha256": path_sha256, + "content_sha256": content_sha256, + "size_bytes": size_bytes, + "kind": "profile_avatar", + "reference_count": reference_count, + "decode_valid": decode_valid, + } + ) + normalized_records.sort(key=lambda item: str(item["path_sha256"])) + if sum(int(record["size_bytes"]) for record in normalized_records) != ( + preserved_total_size_bytes + ): + raise ValueError("upload manifest preserved total size mismatch") + preserved_decode_valid_count = sum( + 1 for record in normalized_records if bool(record["decode_valid"]) + ) + preserved_decode_invalid_count = len(normalized_records) - ( + preserved_decode_valid_count + ) + required_decode_invalid_records = [ + record + for record in normalized_records + if int(record["reference_count"]) > 0 + and not bool(record["decode_valid"]) + ] + if ( + decode_counts["preserved_decode_valid_count"] + != preserved_decode_valid_count + or decode_counts["preserved_decode_invalid_count"] + != preserved_decode_invalid_count + or decode_counts["required_decode_invalid_object_count"] + != len(required_decode_invalid_records) + or decode_counts["required_decode_invalid_reference_count"] + != sum( + int(record["reference_count"]) + for record in required_decode_invalid_records + ) + ): + raise ValueError("upload manifest image decode count mismatch") + unbound = set(available).difference(seen) + if reject_unbound_extras and unbound: + raise ValueError("public avatar directory contains an unbound extra") + if any( + not is_runtime_generated_public_avatar_relative_path(available[digest][0]) + for digest in unbound + ): + raise ValueError("public avatar directory contains an unauthorized extra") + preserved_set_sha256 = _preserved_object_set_sha256(normalized_records) + if payload.get("preserved_object_set_sha256") != preserved_set_sha256: + raise ValueError("upload manifest preserved-object set hash mismatch") + referenced_records = [ + record + for record in normalized_records + if int(record["reference_count"]) > 0 + ] + reference_records = _reference_records_from_objects(referenced_records) + reference_set_sha256 = _reference_set_sha256(reference_records) + if payload.get("reference_set_sha256") != reference_set_sha256: + raise ValueError("upload manifest reference-set hash mismatch") + if ( + len(reference_records) != required_count + or sum(int(record["reference_count"]) for record in referenced_records) + != required_reference_count + ): + raise ValueError("upload manifest database reference count mismatch") + database_contract = payload.get("database_reference_contract") + if not isinstance(database_contract, dict): + raise ValueError("upload manifest database reference contract is invalid") + if ( + database_contract.get("reference_count") != required_reference_count + or database_contract.get("unique_object_count") != required_count + or database_contract.get("reference_set_sha256") != reference_set_sha256 + or database_contract.get("objects") != reference_records + ): + raise ValueError("upload manifest database reference contract drift") + return UploadManifestProof( + manifest_sha256=actual_manifest_sha256, + root_path_sha256=root_sha256, + database_target_sha256=expected_database_target_sha256, + preserved_object_count=preserved_count, + preserved_total_size_bytes=preserved_total_size_bytes, + preserved_decode_valid_count=preserved_decode_valid_count, + preserved_decode_invalid_count=preserved_decode_invalid_count, + required_decode_invalid_object_count=len( + required_decode_invalid_records + ), + required_decode_invalid_reference_count=sum( + int(record["reference_count"]) + for record in required_decode_invalid_records + ), + preserved_object_set_sha256=preserved_set_sha256, + required_object_count=required_count, + required_reference_count=required_reference_count, + reference_set_sha256=reference_set_sha256, + write_freeze_token_sha256=token_sha256, + preserved_records=tuple(preserved_cache), + preserved_path_decode_records=tuple( + (str(record["path_sha256"]), bool(record["decode_valid"])) + for record in normalized_records + ), + reference_records=tuple( + (str(record["path_sha256"]), int(record["reference_count"])) + for record in reference_records + ), + ) + + +@dataclass(frozen=True, slots=True) +class CurrentAvatarReferenceProof: + object_count: int + reference_count: int + reference_set_sha256: str + decode_invalid_object_count: int + decode_invalid_reference_count: int + + +def validate_current_avatar_references( + *, + upload_root: Path, + avatar_urls: Sequence[str], + initial_manifest: UploadManifestProof, + expected_database_target_sha256: str, +) -> CurrentAvatarReferenceProof: + if ( + not _is_lower_hex_sha256(expected_database_target_sha256) + or initial_manifest.database_target_sha256 + != expected_database_target_sha256 + ): + raise ValueError("current avatar inventory database target is unverified") + references = [ + (public_avatar_url_to_relative_path(value), "profile_avatar") + for value in avatar_urls + if isinstance(value, str) and value.strip() + ] + aggregated = _aggregate_references(references) + if initial_manifest.required_reference_count > 0 and not aggregated: + raise ValueError( + "current public avatar inventory is empty without an explicit reset receipt" + ) + assert_path_without_reparse(upload_root, "public upload root") + root = upload_root.resolve(strict=True) + records: list[dict[str, object]] = [] + preserved_decode_status = dict(initial_manifest.preserved_path_decode_records) + decode_invalid_object_count = 0 + decode_invalid_reference_count = 0 + for relative_path, _kind, reference_count in aggregated: + target = root / Path(*PurePosixPath(relative_path).parts) + path_sha256 = public_avatar_relative_path_sha256(relative_path) + preserved_decode_valid = preserved_decode_status.get(path_sha256) + if preserved_decode_valid is None: + if not is_runtime_generated_public_avatar_relative_path(relative_path): + raise ValueError( + "current DB-referenced avatar object is not authorized" + ) + try: + read_decodable_public_avatar_object( + relative_path=relative_path, + path=target, + label="current DB-referenced avatar object", + ) + except PublicAvatarImageInvalid: + decode_invalid_object_count += 1 + decode_invalid_reference_count += reference_count + else: + sha256_file(target) + if preserved_decode_valid is False: + decode_invalid_object_count += 1 + decode_invalid_reference_count += reference_count + records.append( + {"path_sha256": path_sha256, "reference_count": reference_count} + ) + records.sort(key=lambda item: str(item["path_sha256"])) + return CurrentAvatarReferenceProof( + object_count=len(records), + reference_count=sum(int(item["reference_count"]) for item in records), + reference_set_sha256=_reference_set_sha256(records), + decode_invalid_object_count=decode_invalid_object_count, + decode_invalid_reference_count=decode_invalid_reference_count, + ) + + +class UploadWriteFrozen(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class UploadWriteFreezeStatus: + capable: bool + active: bool + valid: bool + in_flight: int + path_sha256: str | None + token_sha256: str | None + + +class UploadWriteFreezeGate: + def __init__(self, freeze_path: Path | None): + self._freeze_path = freeze_path + self._lock = threading.Lock() + self._in_flight = 0 + + def status(self) -> UploadWriteFreezeStatus: + path = self._freeze_path + with self._lock: + in_flight = self._in_flight + if path is None: + return UploadWriteFreezeStatus( + False, False, False, in_flight, None, None + ) + path_sha256 = canonical_path_sha256(path) + try: + assert_path_without_reparse(path, "upload write-freeze path") + except ValueError: + return UploadWriteFreezeStatus( + True, True, False, in_flight, path_sha256, None + ) + if not _path_entry_exists_without_following(path): + return UploadWriteFreezeStatus( + True, False, True, in_flight, path_sha256, None + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + token = payload.get("token") if isinstance(payload, dict) else None + valid = ( + isinstance(payload, dict) + and payload.get("schema_version") == WRITE_FREEZE_SCHEMA_VERSION + and isinstance(token, str) + and len(token) >= 32 + ) + token_sha256 = _sha256_text(token) if valid else None + except (OSError, UnicodeError, json.JSONDecodeError): + valid = False + token_sha256 = None + return UploadWriteFreezeStatus( + True, + True, + valid, + in_flight, + path_sha256, + token_sha256, + ) + + @contextmanager + def write_lease(self) -> Iterator[None]: + with self._lock: + if self._freeze_path is not None and _path_entry_exists_without_following( + self._freeze_path + ): + raise UploadWriteFrozen("public upload writes are temporarily frozen") + self._in_flight += 1 + try: + yield + finally: + with self._lock: + self._in_flight -= 1 diff --git a/apps/api/requirements.txt b/apps/api/requirements.txt index d7aae2d..4a8332e 100644 --- a/apps/api/requirements.txt +++ b/apps/api/requirements.txt @@ -6,6 +6,7 @@ asyncpg==0.31.0 pydantic==2.9.2 pydantic-settings==2.12.0 python-multipart==0.0.18 +Pillow==12.2.0 httpx==0.28.1 websockets==14.2 sse-starlette==3.0.3 diff --git a/scripts/boot-public-runtime.ps1 b/scripts/boot-public-runtime.ps1 index c5776d2..d160984 100644 --- a/scripts/boot-public-runtime.ps1 +++ b/scripts/boot-public-runtime.ps1 @@ -23,6 +23,12 @@ param( [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{64}$")] [string]$ExpectedStartScriptSha256, + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedPythonSha256 = "", + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedCloudflaredSha256 = "", + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedCloudflaredConfigSha256 = "", [string]$DockerDesktop = "C:\Program Files\Docker\Docker\Docker Desktop.exe", [int]$DaemonTimeoutSec = 360, [int]$DbTimeoutSec = 90, @@ -33,6 +39,17 @@ param( [int]$WhisperPort = 9882, [int]$MeloTtsPort = 9883, [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", + [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", + [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, [string]$BootLog = "" ) @@ -42,6 +59,10 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" +$uploadRootContract = Join-Path $resolvedSourceRoot "scripts\public-runtime-upload-root.ps1" +$uploadRootProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-runtime-upload-root.py" +$uploadManifestProbe = Join-Path $resolvedSourceRoot "scripts\validate-public-runtime-upload-manifest.py" +$databaseIdentityHelper = Join-Path $resolvedSourceRoot "scripts\public_runtime_database_identity.py" function Invoke-GitText { param([string[]]$Arguments) @@ -54,7 +75,15 @@ function Invoke-GitText { } function Assert-StableSourceProvenance { - foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) { + foreach ($requiredScript in @( + $expectedBootScript, + $startScript, + $voiceSidecarProbe, + $uploadRootContract, + $uploadRootProbe, + $uploadManifestProbe, + $databaseIdentityHelper + )) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Pinned public runtime script not found at $requiredScript" } @@ -104,7 +133,13 @@ function Assert-StableSourceProvenance { foreach ($relativePath in @( "scripts/boot-public-runtime.ps1", "scripts/start-public-runtime.ps1", - "scripts/probe-public-voice-sidecars.py" + "scripts/probe-public-voice-sidecars.py", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -117,10 +152,40 @@ function Assert-StableSourceProvenance { if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) { throw "Pinned start script SHA256 drift" } + foreach ($pin in @( + [pscustomobject]@{ Path = $Python; Expected = $ExpectedPythonSha256; Role = "Python" }, + [pscustomobject]@{ Path = $Cloudflared; Expected = $ExpectedCloudflaredSha256; Role = "cloudflared" }, + [pscustomobject]@{ Path = $CloudflaredConfig; Expected = $ExpectedCloudflaredConfigSha256; Role = "cloudflared config" } + )) { + if ([string]::IsNullOrWhiteSpace([string]$pin.Expected)) { + continue + } + if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) { + throw "Pinned $($pin.Role) is unavailable" + } + $actual = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -cne ([string]$pin.Expected).ToLowerInvariant()) { + throw "Pinned $($pin.Role) SHA256 drift" + } + } } # Docker/DB/process mutation보다 먼저 stable source를 매 실행 재검증한다. Assert-StableSourceProvenance +. $uploadRootContract +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $UserUploadDir ` + -ProbeWritable +$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadManifestPath ` + -RequireFile +$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadWriteFreezePath if (!$BootLog) { $BootLog = Join-Path $resolvedSourceRoot "boot-public-runtime.log" @@ -152,6 +217,28 @@ function Test-Tcp([string]$Host_, [int]$Port) { } catch { return $false } } +function Get-LocalApiHealthSnapshot { + try { + $request = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/health") + $request.Timeout = 5000 + $request.ReadWriteTimeout = 5000 + $request.Proxy = $null + $response = $request.GetResponse() + try { + $reader = New-Object System.IO.StreamReader($response.GetResponseStream()) + try { + return ($reader.ReadToEnd() | ConvertFrom-Json) + } finally { + $reader.Dispose() + } + } finally { + $response.Dispose() + } + } catch { + return $null + } +} + function Test-ApiControlPlaneHealthy { # HttpWebRequest + Proxy=$null: WININET/시스템 프록시에 영향받지 않는 가장 직결적인 검사. # 비대화형 스케줄러 컨텍스트에서도 127.0.0.1 로 직접 연결한다. 3회 재시도. @@ -166,7 +253,14 @@ function Test-ApiControlPlaneHealthy { $body = $reader.ReadToEnd() $reader.Close(); $resp.Close() $h = $body | ConvertFrom-Json - if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true } + if ( + $h.environment -eq "prod" -and + $h.db -eq $true -and + $h.engine -eq $true -and + $h.upload_write_freeze.capable -eq $true -and + $h.upload_write_freeze.active -eq $false -and + $h.upload_write_freeze.valid -eq $true + ) { return $true } Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body) return $false } catch { @@ -304,22 +398,81 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) { exit 1 } Write-BootLog "postgres 127.0.0.1:$DbPort up" +$uploadManifestHealthy = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $uploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +if (-not $uploadManifestHealthy.Ok) { + Write-BootLog "ERROR: public upload migration receipt or current DB inventory is invalid" + exit 1 +} +$expectedDatabaseTargetSha256 = [string]$uploadManifestHealthy.Payload.database_target_sha256 +if ($expectedDatabaseTargetSha256 -notmatch "^[0-9a-f]{64}$") { + Write-BootLog "ERROR: public upload inventory proof did not return a valid database target identity" + exit 1 +} +if (Test-Path -LiteralPath $resolvedUserUploadWriteFreezePath -PathType Leaf) { + $promotionHealth = Get-LocalApiHealthSnapshot + $promotionFreeze = $null + if ($null -ne $promotionHealth) { + $promotionFreeze = $promotionHealth.upload_write_freeze + } + if ( + $null -ne $promotionFreeze -and + $promotionFreeze.capable -eq $true -and + $promotionFreeze.active -eq $true -and + $promotionFreeze.valid -eq $true -and + [int]$promotionFreeze.in_flight -eq 0 -and + [string]$promotionFreeze.token_sha256 -ceq + [string]$uploadManifestHealthy.Payload.write_freeze_token_sha256 + ) { + Write-BootLog "promotion-in-progress: valid drained upload freeze is active; skipping runtime mutation" + exit 0 + } + Write-BootLog "ERROR: upload freeze sentinel exists without exact active/drained API proof; refusing runtime mutation" + exit 1 +} # 3) 엔진/API/web/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지) # web preview는 살아 있을 때만 -SkipWebRestart 한다. 무조건 스킵하면 재부팅 직후처럼 # vite가 죽은 상태에서 boot 경로로는 web이 영영 복구되지 않는다. $webHealthy = Test-WebPreviewHealthy -if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack) -and $webHealthy) { +if ( + (Test-ApiControlPlaneHealthy) -and + (Test-EngineHealthy) -and + (Test-VoiceApiHealthy) -and + (Test-VoiceSidecarStack) -and + (Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort).Ok -and + $webHealthy +) { Write-BootLog "control plane, engine, exact local voice API, sidecars, and web preview already healthy; skipping runtime restart" } else { $startArgs = @( "-Workspace", $resolvedSourceRoot, "-Python", $Python, + "-Cloudflared", $Cloudflared, + "-CloudflaredConfig", $CloudflaredConfig, "-ApiPort", $ApiPort, "-WebPort", $WebPort, "-EnginePort", $EnginePort, "-WhisperPort", $WhisperPort, - "-MeloTtsPort", $MeloTtsPort + "-MeloTtsPort", $MeloTtsPort, + "-UserUploadDir", $resolvedUserUploadDir, + "-UserUploadManifestPath", $resolvedUserUploadManifestPath, + "-ExpectedUserUploadManifestSha256", $ExpectedUserUploadManifestSha256, + "-UserUploadWriteFreezePath", $resolvedUserUploadWriteFreezePath ) if ($webHealthy) { $startArgs += "-SkipWebRestart" @@ -342,7 +495,17 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiH # 4) 최종 확인 if (Test-ApiControlPlaneHealthy) { - if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) { + $apiUploadRootHealthy = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort + if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack) -and $apiUploadRootHealthy.Ok) { Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy" exit 0 } else { diff --git a/scripts/bootstrap-legacy-public-runtime-upload-root.ps1 b/scripts/bootstrap-legacy-public-runtime-upload-root.ps1 new file mode 100644 index 0000000..c36492c --- /dev/null +++ b/scripts/bootstrap-legacy-public-runtime-upload-root.ps1 @@ -0,0 +1,2670 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$StableSourceRoot, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{40}$")] + [string]$ExpectedSourceCommit, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{40}$")] + [string]$ExpectedSourceTree, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{40}$")] + [string]$ExpectedLegacySourceCommit, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{40}$")] + [string]$ExpectedLegacySourceTree, + [Parameter(Mandatory = $true)] + [string]$PythonPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedPythonSha256, + [Parameter(Mandatory = $true)] + [string]$CloudflaredPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedCloudflaredSha256, + [Parameter(Mandatory = $true)] + [string]$CloudflaredConfigPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedCloudflaredConfigSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$ManifestStateDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, + [Parameter(Mandatory = $true)] + [ValidateRange(0, 2147483647)] + [int]$ExpectedReferenceCount, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedPreservedObjectCount, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 9223372036854775807)] + [long]$ExpectedPreservedTotalSizeBytes, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedPreservedInventorySha256, + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string[]]$AllowedLegacySourceUploadDir, + [Parameter(Mandatory = $true)] + [string]$CutoverReceiptPath, + [Parameter(Mandatory = $true)] + [string]$TaskRecoveryReceiptPath, + [int]$ApiPort = 8001, + [int]$WhisperPort = 9882, + [int]$MeloTtsPort = 9883, + [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health", + [string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock", + [ValidateRange(10, 300)] + [int]$HealthTimeoutSeconds = 60, + [ValidateRange(5, 120)] + [int]$ProcessStopTimeoutSeconds = 30, + [string[]]$CoordinatedTaskNames = @( + "VignettePublicRuntimeWatchdog", + "VignettePublicRuntime" + ) +) + +$ErrorActionPreference = "Stop" +[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false) +$OutputEncoding = [System.Text.UTF8Encoding]::new($false) + +function Get-Utf8Sha256 { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $bytes = [System.Text.UTF8Encoding]::new($false).GetBytes($Value) + $hasher = [System.Security.Cryptography.SHA256]::Create() + try { + return ([BitConverter]::ToString($hasher.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $hasher.Dispose() + } +} + +function Get-RequiredPrivacySafeCount { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][string]$Role + ) + + $property = $Payload.PSObject.Properties[$Name] + if ($null -eq $property -or $null -eq $property.Value) { + throw "$Role did not return privacy-safe decode count $Name" + } + $typeCode = [System.Type]::GetTypeCode($property.Value.GetType()) + if (@( + [System.TypeCode]::Byte, + [System.TypeCode]::SByte, + [System.TypeCode]::Int16, + [System.TypeCode]::UInt16, + [System.TypeCode]::Int32, + [System.TypeCode]::UInt32, + [System.TypeCode]::Int64, + [System.TypeCode]::UInt64 + ) -notcontains $typeCode) { + throw "$Role returned non-integer privacy-safe decode count $Name" + } + $value = [decimal]$property.Value + if ($value -lt 0 -or $value -gt [int]::MaxValue) { + throw "$Role returned out-of-range privacy-safe decode count $Name" + } + return [int]$value +} + +function Get-PreservedDecodeCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$PreservedObjectCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $validCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "preserved_decode_valid_count" ` + -Role $Role + $invalidCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "preserved_decode_invalid_count" ` + -Role $Role + if (($validCount + $invalidCount) -ne $PreservedObjectCount) { + throw "$Role preserved decode counts do not cover the exact object inventory" + } + return [pscustomobject]@{ + ValidCount = $validCount + InvalidCount = $invalidCount + } +} + +function Get-RequiredDecodeInvalidCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$RequiredObjectCount, + [Parameter(Mandatory = $true)][int]$RequiredReferenceCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $objectCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "required_decode_invalid_object_count" ` + -Role $Role + $referenceCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "required_decode_invalid_reference_count" ` + -Role $Role + if ( + $objectCount -gt $RequiredObjectCount -or + $referenceCount -gt $RequiredReferenceCount + ) { + throw "$Role required decode-invalid counts exceed the bound DB inventory" + } + return [pscustomobject]@{ + ObjectCount = $objectCount + ReferenceCount = $referenceCount + } +} + +function Get-CurrentDecodeInvalidCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$CurrentObjectCount, + [Parameter(Mandatory = $true)][int]$CurrentReferenceCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $objectCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "current_decode_invalid_object_count" ` + -Role $Role + $referenceCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "current_decode_invalid_reference_count" ` + -Role $Role + if ( + $objectCount -gt $CurrentObjectCount -or + $referenceCount -gt $CurrentReferenceCount + ) { + throw "$Role current decode-invalid counts exceed the bound DB inventory" + } + return [pscustomobject]@{ + ObjectCount = $objectCount + ReferenceCount = $referenceCount + } +} + +function Get-CanonicalPathSha256 { + param([Parameter(Mandatory = $true)][string]$Path) + + $resolved = (Resolve-Path -LiteralPath $Path).Path + $identity = $resolved.Replace("\", "/").ToLowerInvariant() + return Get-Utf8Sha256 -Value $identity +} + +function Invoke-PinnedGitText { + param( + [string]$Root, + [string[]]$Arguments + ) + + $output = @(& git.exe -C $Root @Arguments) + if ($LASTEXITCODE -ne 0) { + throw "Stable source Git proof failed" + } + return ((@($output) -join [Environment]::NewLine).Trim()) +} + +function Assert-BootstrapSourceProvenance { + param([string]$Root) + + $gitRoot = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--show-toplevel") + if (-not [string]::Equals( + (Resolve-Path -LiteralPath $gitRoot).Path, + $Root, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Bootstrap source is not its Git toplevel" + } + $symbolicHead = @(& git.exe -C $Root symbolic-ref --quiet HEAD) + $symbolicExit = $LASTEXITCODE + if ($symbolicExit -eq 0 -or $symbolicHead.Count -gt 0) { + throw "Bootstrap source must use detached HEAD" + } + if ($symbolicExit -ne 1) { + throw "Bootstrap detached HEAD proof failed" + } + $actualCommit = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--verify", "HEAD") + $actualTree = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--verify", "HEAD^{tree}") + if ($actualCommit -cne $ExpectedSourceCommit -or $actualTree -cne $ExpectedSourceTree) { + throw "Bootstrap source commit or tree drift" + } + $dirty = Invoke-PinnedGitText -Root $Root -Arguments @( + "status", "--porcelain=v1", "--untracked-files=normal" + ) + if ($dirty) { + throw "Bootstrap source must be clean" + } + foreach ($relativePath in @( + "scripts/bootstrap-legacy-public-runtime-upload-root.ps1", + "scripts/initialize-public-runtime-upload-root.ps1", + "scripts/initialize-public-runtime-upload-root.py", + "scripts/validate-public-runtime-offline-quiescence.py", + "scripts/start-public-runtime.ps1", + "scripts/public-runtime-task-maintenance.ps1", + "scripts/public-runtime-task-definition-cutover.ps1", + "scripts/register-boot-task.ps1", + "scripts/install-public-runtime-task.ps1", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "scripts/probe-public-runtime-database-identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" + )) { + $null = Invoke-PinnedGitText ` + -Root $Root ` + -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) + } +} + +function Assert-LegacyApiSourceProvenance { + param([System.Collections.IDictionary]$ApiIdentity) + + $cwd = (Resolve-Path -LiteralPath ([string]$ApiIdentity.cwd)).Path + $legacyGitRoot = Invoke-PinnedGitText ` + -Root $cwd ` + -Arguments @("rev-parse", "--show-toplevel") + $resolvedLegacyGitRoot = (Resolve-Path -LiteralPath $legacyGitRoot).Path + $symbolicHead = @(& git.exe -C $resolvedLegacyGitRoot symbolic-ref --quiet HEAD) + $symbolicExit = $LASTEXITCODE + if ($symbolicExit -eq 0 -or $symbolicHead.Count -gt 0 -or $symbolicExit -ne 1) { + throw "Legacy API source must use detached HEAD" + } + $commit = Invoke-PinnedGitText ` + -Root $resolvedLegacyGitRoot ` + -Arguments @("rev-parse", "--verify", "HEAD") + $tree = Invoke-PinnedGitText ` + -Root $resolvedLegacyGitRoot ` + -Arguments @("rev-parse", "--verify", "HEAD^{tree}") + if ($commit -cne $ExpectedLegacySourceCommit -or $tree -cne $ExpectedLegacySourceTree) { + throw "Legacy API source commit or tree drift" + } + $dirty = Invoke-PinnedGitText ` + -Root $resolvedLegacyGitRoot ` + -Arguments @("status", "--porcelain=v1", "--untracked-files=normal") + if ($dirty) { + throw "Legacy API source must be clean" + } + $expectedApiCwd = Join-Path $resolvedLegacyGitRoot "apps\api" + if (-not [string]::Equals( + $cwd, + [System.IO.Path]::GetFullPath($expectedApiCwd), + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Legacy API cwd is not bound to its Git root" + } + return [ordered]@{ + commit = $commit + tree = $tree + } +} + +function Assert-FileSha256 { + param( + [string]$Path, + [string]$ExpectedSha256, + [string]$Role + ) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "$Role is unavailable" + } + $actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -cne $ExpectedSha256) { + throw "$Role SHA256 drift" + } +} + +function Resolve-PrivateBootstrapPath { + param( + [string]$Path, + [string]$StateRoot, + [switch]$RequireFile + ) + + if (-not [System.IO.Path]::IsPathRooted($Path)) { + throw "Bootstrap private state path must be absolute" + } + $full = Get-PublicRuntimeCanonicalPath -Path $Path + $parent = [System.IO.Path]::GetDirectoryName($full) + if (-not [string]::Equals( + $parent, + $StateRoot, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Bootstrap receipt and freeze files must be direct private-state children" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $full + if ($RequireFile) { + if (-not (Test-Path -LiteralPath $full -PathType Leaf)) { + throw "Bootstrap private state file is unavailable" + } + return (Resolve-Path -LiteralPath $full).Path + } + return $full +} + +function Resolve-PrivateBootstrapStateDirectory { + param( + [string]$Path, + [string]$SourceRoot, + [string]$UploadRoot + ) + + if (-not [System.IO.Path]::IsPathRooted($Path)) { + throw "Bootstrap private state directory must be absolute" + } + $full = Get-PublicRuntimeCanonicalPath -Path $Path + foreach ($boundary in @($SourceRoot, $UploadRoot)) { + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $full -Parent $boundary) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $boundary -Parent $full) + ) { + throw "Bootstrap private state must be disjoint from source and public roots" + } + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $full + if (-not (Test-Path -LiteralPath $full)) { + [System.IO.Directory]::CreateDirectory($full) | Out-Null + } + if (-not (Test-Path -LiteralPath $full -PathType Container)) { + throw "Bootstrap private state directory is unavailable" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $full + return (Resolve-Path -LiteralPath $full).Path +} + +function Get-ExactLoopbackListenerPid { + param([int]$Port) + + $listeners = @( + Get-NetTCPConnection ` + -State Listen ` + -LocalPort $Port ` + -ErrorAction SilentlyContinue + ) + if ($listeners.Count -eq 0) { + throw "Expected API listener is absent" + } + foreach ($listener in $listeners) { + if ([string]$listener.LocalAddress -cne "127.0.0.1") { + throw "API port has a non-loopback listener" + } + } + $processIds = @($listeners | ForEach-Object { [int]$_.OwningProcess } | Sort-Object -Unique) + if ($processIds.Count -ne 1) { + throw "API listener does not have one exact owner" + } + return [int]$processIds[0] +} + +function Assert-LoopbackListenerAbsent { + param( + [int]$Port, + [int]$TimeoutSec + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $listeners = @( + Get-NetTCPConnection ` + -State Listen ` + -LocalPort $Port ` + -ErrorAction SilentlyContinue + ) + if ($listeners.Count -eq 0) { + return + } + Start-Sleep -Milliseconds 200 + } while ((Get-Date) -lt $deadline) + throw "API listener absence was not proven" +} + +function Wait-ProcessIdentity { + param( + [int]$ProcessId, + [string]$Role, + [string]$ExpectedCwd = "", + [int]$TimeoutSec = 15 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $process = Get-CimInstance Win32_Process ` + -Filter "ProcessId = $ProcessId" ` + -ErrorAction SilentlyContinue + if ($null -ne $process -and $process.ExecutablePath -and $process.CommandLine) { + $probeArgs = @( + "-X", "utf8", "-c", + "import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00','Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8',errors='strict')).hexdigest()); print(json.dumps(argv[1:],ensure_ascii=True,separators=(',',':'))); print(json.dumps(p.environ(),ensure_ascii=True,separators=(',',':')))", + "$ProcessId" + ) + $probe = @(& $resolvedPythonPath @probeArgs) + if ($LASTEXITCODE -ne 0 -or $probe.Count -ne 5) { + throw "$Role identity probe failed" + } + $cwd = $probe[0].Trim() + $startedAtUtc = $probe[1].Trim() + $commandLineSha256 = $probe[2].Trim().ToLowerInvariant() + $argumentList = @($probe[3] | ConvertFrom-Json) + $environmentObject = $probe[4] | ConvertFrom-Json + $environment = [ordered]@{} + foreach ($property in $environmentObject.PSObject.Properties) { + $environment[$property.Name] = [string]$property.Value + } + if (-not $cwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") { + throw "$Role identity is incomplete" + } + if ($ExpectedCwd -and -not [string]::Equals( + [System.IO.Path]::GetFullPath($cwd), + [System.IO.Path]::GetFullPath($ExpectedCwd), + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "$Role working directory drift" + } + return [ordered]@{ + pid = [int]$process.ProcessId + started_at_utc = $startedAtUtc + executable_path = [string]$process.ExecutablePath + executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant() + command_line_sha256 = $commandLineSha256 + argument_list = $argumentList + environment = $environment + cwd = $cwd + } + } + Start-Sleep -Milliseconds 200 + } while ((Get-Date) -lt $deadline) + throw "$Role identity timed out" +} + +function Test-ArgumentPair { + param( + [object[]]$Arguments, + [string]$Name, + [string]$Value + ) + + $nameCount = 0 + $matchingValueCount = 0 + for ($index = 0; $index -lt $Arguments.Count; $index++) { + if ([string]$Arguments[$index] -cne $Name) { + continue + } + $nameCount++ + if ( + $index -lt ($Arguments.Count - 1) -and + [string]::Equals( + [string]$Arguments[$index + 1], + $Value, + [System.StringComparison]::OrdinalIgnoreCase + ) + ) { + $matchingValueCount++ + } + } + return $nameCount -eq 1 -and $matchingValueCount -eq 1 +} + +function Assert-ApiIdentityContract { + param([System.Collections.IDictionary]$Identity) + + $arguments = @($Identity.argument_list) + if ( + $arguments -notcontains "uvicorn" -or + $arguments -notcontains "app.main:app" -or + -not (Test-ArgumentPair -Arguments $arguments -Name "--port" -Value "$ApiPort") -or + -not (Test-ArgumentPair -Arguments $arguments -Name "--workers" -Value "1") + ) { + throw "API listener command contract drift" + } +} + +function Get-TunnelIdentitiesForConfig { + param([string]$ConfigPath) + + $candidateProcesses = @( + Get-CimInstance Win32_Process | + Where-Object { + $_.Name -eq "cloudflared.exe" -and + $_.CommandLine -and + $_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 + } + ) + $identities = New-Object System.Collections.Generic.List[object] + foreach ($candidate in $candidateProcesses) { + $identity = Wait-ProcessIdentity ` + -ProcessId ([int]$candidate.ProcessId) ` + -Role "cloudflared" ` + -TimeoutSec $ProcessStopTimeoutSeconds + $arguments = @($identity.argument_list) + if ( + (Test-ArgumentPair -Arguments $arguments -Name "--config" -Value $ConfigPath) -and + $arguments -contains "tunnel" -and + $arguments -contains "run" + ) { + $identities.Add($identity) + } + } + return @($identities) +} + +function Get-ExactTunnelIdentity { + param([string]$ConfigPath) + + $identities = @( + Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath + ) + if ($identities.Count -ne 1) { + throw "Expected exactly one tunnel identity" + } + return $identities[0] +} + +function Assert-TunnelAbsent { + param( + [string]$ConfigPath, + [int]$TimeoutSec + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $matches = @( + Get-CimInstance Win32_Process | + Where-Object { + $_.Name -eq "cloudflared.exe" -and + $_.CommandLine -and + $_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 + } + ) + if ($matches.Count -eq 0) { + return + } + Start-Sleep -Milliseconds 200 + } while ((Get-Date) -lt $deadline) + throw "Tunnel absence was not proven" +} + +function Assert-CurrentTunnelIdentity { + param( + [string]$ConfigPath, + [System.Collections.IDictionary]$Expected + ) + + $current = @(Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath) + if ( + $current.Count -ne 1 -or + -not (Test-IdentityExactlyMatches -Expected $Expected -Actual $current[0]) + ) { + throw "Current tunnel identity is not the unique launched tunnel" + } +} + +function Assert-IdentityUnchanged { + param( + [System.Collections.IDictionary]$Expected, + [System.Collections.IDictionary]$Actual, + [string]$Role + ) + + foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) { + if ($Expected[$field].ToString() -cne $Actual[$field].ToString()) { + throw "$Role identity drift" + } + } +} + +function Test-IdentityExactlyMatches { + param( + [System.Collections.IDictionary]$Expected, + [System.Collections.IDictionary]$Actual + ) + + if ($null -eq $Expected -or $null -eq $Actual) { + return $false + } + foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) { + if ($Expected[$field].ToString() -cne $Actual[$field].ToString()) { + return $false + } + } + return $true +} + +function Stop-VerifiedIdentity { + param( + [System.Collections.IDictionary]$Identity, + [string]$Role + ) + + $current = Wait-ProcessIdentity ` + -ProcessId ([int]$Identity.pid) ` + -Role $Role ` + -ExpectedCwd ([string]$Identity.cwd) ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-IdentityUnchanged -Expected $Identity -Actual $current -Role $Role + Stop-Process -Id ([int]$Identity.pid) -Force -ErrorAction Stop + $deadline = (Get-Date).AddSeconds($ProcessStopTimeoutSeconds) + do { + if ($null -eq (Get-Process -Id ([int]$Identity.pid) -ErrorAction SilentlyContinue)) { + return + } + Start-Sleep -Milliseconds 200 + } while ((Get-Date) -lt $deadline) + throw "$Role did not stop within the bounded interval" +} + +function ConvertTo-SafeProcessIdentity { + param([System.Collections.IDictionary]$Identity) + + return [ordered]@{ + pid = [int]$Identity.pid + started_at_utc = [string]$Identity.started_at_utc + executable_sha256 = [string]$Identity.executable_sha256 + command_line_sha256 = [string]$Identity.command_line_sha256 + cwd_sha256 = Get-CanonicalPathSha256 -Path ([string]$Identity.cwd) + } +} + +function Save-CompleteProcessEnvironment { + $snapshot = [ordered]@{} + $environment = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + foreach ($name in $environment.Keys) { + $snapshot[[string]$name] = [string]$environment[$name] + } + return $snapshot +} + +function Get-IdentityEnvironmentValue { + param( + [System.Collections.IDictionary]$Environment, + [string]$Name, + [switch]$AllowEmpty + ) + + foreach ($key in $Environment.Keys) { + if ([string]$key -ieq $Name) { + $value = [string]$Environment[$key] + if (-not $AllowEmpty -and [string]::IsNullOrWhiteSpace($value)) { + throw "Required API environment value is empty" + } + return $value + } + } + throw "Required API environment key is missing" +} + +function Invoke-EffectiveApiSettingsProbe { + param( + [string]$PythonPath, + [string]$ApiCwd, + [System.Collections.IDictionary]$Environment + ) + + $code = "import json; from app.config import settings as s; j=lambda v:json.dumps(v,ensure_ascii=True,separators=(',',':')); d={'DATABASE_URL':str(s.database_url),'SESSION_SECRET':s.session_secret,'ENGINE_URL':str(s.engine_url),'ENGINE_GATEWAY_SHARED_SECRET':s.engine_gateway_shared_secret.get_secret_value(),'OAUTH_GOOGLE_CLIENT_ID':s.oauth_google_client_id,'OAUTH_GOOGLE_CLIENT_SECRET':s.oauth_google_client_secret,'OAUTH_REDIRECT_URI':s.oauth_redirect_uri,'AUTH_ALLOWED_EMAIL_DOMAINS':j(s.auth_allowed_email_domains),'AUTH_TEACHER_EMAILS':j(s.auth_teacher_emails),'AUTH_ADMIN_EMAILS':j(s.auth_admin_emails),'AUTH_SUPER_ADMIN_EMAILS':j(s.auth_super_admin_emails),'AUTH_APPROVED_EMAILS':j(s.auth_approved_emails),'AUTH_NEW_USER_DEFAULT_STATUS':s.auth_new_user_default_status,'AUTH_EMAIL_COHORT_MAP':j(s.auth_email_cohort_map),'AUTH_DOMAIN_COHORT_MAP':j(s.auth_domain_cohort_map),'DEFAULT_AFFILIATION':s.default_affiliation}; print(json.dumps(d,ensure_ascii=True,separators=(',',':')))" + $callerEnvironment = Save-CompleteProcessEnvironment + try { + Set-CompleteProcessEnvironment -Environment $Environment + Push-Location $ApiCwd + try { + $probeOutput = @(& $PythonPath @("-X", "utf8", "-c", $code) 2>$null) + $probeExit = $LASTEXITCODE + } finally { + Pop-Location + } + } finally { + Set-CompleteProcessEnvironment -Environment $callerEnvironment + } + if ($probeExit -ne 0 -or $probeOutput.Count -ne 1) { + throw "API effective settings could not be loaded" + } + $effective = $probeOutput[0] | ConvertFrom-Json + $values = [ordered]@{} + foreach ($property in $effective.PSObject.Properties) { + $name = [string]$property.Name + $value = [string]$property.Value + $values[$name] = $value + } + return $values +} + +function Set-RequiredApiEnvironmentFromIdentity { + param( + [System.Collections.IDictionary]$Identity, + [string]$ExpectedEnvironmentFileSha256 + ) + + $priorEnvPath = Join-Path ([string]$Identity.cwd) ".env" + Assert-EnvironmentFilePinned ` + -Path $priorEnvPath ` + -ExpectedSha256 $ExpectedEnvironmentFileSha256 ` + -Role "prior API" + $values = Invoke-EffectiveApiSettingsProbe ` + -PythonPath ([string]$Identity.executable_path) ` + -ApiCwd ([string]$Identity.cwd) ` + -Environment $Identity.environment + Assert-EnvironmentFilePinned ` + -Path $priorEnvPath ` + -ExpectedSha256 $ExpectedEnvironmentFileSha256 ` + -Role "prior API after settings load" + foreach ($name in @( + "DATABASE_URL", "SESSION_SECRET", "OAUTH_GOOGLE_CLIENT_ID", + "OAUTH_GOOGLE_CLIENT_SECRET", "OAUTH_REDIRECT_URI" + )) { + if (-not $values.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$values[$name])) { + throw "Prior API effective authentication/database setting is empty" + } + } + foreach ($name in $values.Keys) { + [System.Environment]::SetEnvironmentVariable( + [string]$name, + [string]$values[$name], + [System.EnvironmentVariableTarget]::Process + ) + } + return $values +} + +function Ensure-ReleaseEnvironmentFile { + param( + [System.Collections.IDictionary]$PriorApiIdentity, + [string]$ExpectedSourceSha256 + ) + + $source = (Resolve-Path -LiteralPath (Join-Path $PriorApiIdentity.cwd ".env")).Path + $target = Join-Path $resolvedStableSourceRoot "apps\api\.env" + Assert-PublicRuntimePathHasNoReparsePoint -Path $source + Assert-PublicRuntimePathHasNoReparsePoint -Path $target + Assert-EnvironmentFilePinned ` + -Path $source ` + -ExpectedSha256 $ExpectedSourceSha256 ` + -Role "prior API before environment copy" + $ignored = @( + & git.exe -C $resolvedStableSourceRoot check-ignore --quiet -- "apps/api/.env" + ) + if ($LASTEXITCODE -ne 0 -or $ignored.Count -gt 0) { + throw "New release private environment file is not ignored" + } + if ([string]::Equals( + $source, + [System.IO.Path]::GetFullPath($target), + [System.StringComparison]::OrdinalIgnoreCase + )) { + return + } + $sourceSha256 = $ExpectedSourceSha256 + if (Test-Path -LiteralPath $target) { + if (-not (Test-Path -LiteralPath $target -PathType Leaf)) { + throw "New release private environment target is not a regular file" + } + $targetSha256 = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant() + if ($targetSha256 -cne $sourceSha256) { + throw "New release private environment target differs from the prior runtime" + } + Assert-EnvironmentFilePinned ` + -Path $source ` + -ExpectedSha256 $ExpectedSourceSha256 ` + -Role "prior API after existing environment verification" + return + } + $input = $null + $output = $null + try { + $input = [System.IO.File]::Open( + $source, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::Read + ) + $output = [System.IO.File]::Open( + $target, + [System.IO.FileMode]::CreateNew, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None + ) + $input.CopyTo($output) + $output.Flush($true) + } finally { + if ($null -ne $output) { + $output.Dispose() + } + if ($null -ne $input) { + $input.Dispose() + } + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $target + $targetSha256 = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant() + if ($targetSha256 -cne $sourceSha256) { + throw "New release private environment copy verification failed" + } + Assert-EnvironmentFilePinned ` + -Path $source ` + -ExpectedSha256 $ExpectedSourceSha256 ` + -Role "prior API after environment copy" +} + +function Get-FutureLauncherRequiredApiSettings { + param( + [System.Collections.IDictionary]$PriorApiIdentity, + [string[]]$RequiredNames, + [int]$EnginePort + ) + + # Scheduled boot/watchdog launches do not inherit bootstrap-only secret + # overrides. Rebuild the exact persistent input: prior non-required process + # environment + start-public-runtime's fixed production flags + target .env. + $futureEnvironment = [ordered]@{} + foreach ($key in $PriorApiIdentity.environment.Keys) { + $isRequired = $false + foreach ($requiredName in $RequiredNames) { + if ([string]$key -ieq $requiredName) { + $isRequired = $true + break + } + } + if (-not $isRequired) { + $futureEnvironment[[string]$key] = [string]$PriorApiIdentity.environment[$key] + } + } + $futureEnvironment["ENVIRONMENT"] = "prod" + $futureEnvironment["ENGINE_URL"] = "http://127.0.0.1:$EnginePort" + $futureEnvironment["ENGINE_MODE"] = "claude_cli" + $futureEnvironment["VIGNETTE_LIVE_CLIENT_PROVIDER"] = "claude_cli" + $futureEnvironment["AUTH_DEV_LOGIN_ENABLED"] = "false" + $futureEnvironment["AUTO_SEED_PERSONAS"] = "false" + $futureEnvironment["ALLOW_SEED_PERSONA_FALLBACK"] = "false" + $futureEnvironment["VIGNETTE_VOICE_POC_SAMPLE_TTS"] = "false" + $futureEnvironment["FRONTEND_BASE_URL"] = "https://vignette.chanpaca.net" + $futureEnvironment["CORS_ORIGINS"] = '["https://vignette.chanpaca.net","https://vnet.18ka.net","https://vignette-b1q.pages.dev"]' + $futureEnvironment["FRONTEND_ORIGIN_MAP"] = '{"api-vignette.chanpaca.net":"https://vignette.chanpaca.net","api-vnet.18ka.net":"https://vnet.18ka.net"}' + + return Invoke-EffectiveApiSettingsProbe ` + -PythonPath $resolvedPythonPath ` + -ApiCwd (Join-Path $resolvedStableSourceRoot "apps\api") ` + -Environment $futureEnvironment +} + +function Assert-RequiredApiSettingsEqual { + param( + [System.Collections.IDictionary]$Expected, + [System.Collections.IDictionary]$Actual, + [string]$Role + ) + + if ($Expected.Count -ne $Actual.Count) { + throw "$Role effective settings key count drift" + } + foreach ($name in $Expected.Keys) { + if (-not $Actual.Contains($name) -or [string]$Expected[$name] -cne [string]$Actual[$name]) { + throw "$Role effective settings digest drift" + } + } +} + +function Assert-EnvironmentFilePinned { + param( + [string]$Path, + [string]$ExpectedSha256, + [string]$Role + ) + + Assert-PublicRuntimePathHasNoReparsePoint -Path $Path + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "$Role environment file is unavailable" + } + $actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -cne $ExpectedSha256) { + throw "$Role environment file SHA256 drift" + } +} + +function Save-SelectedProcessEnvironment { + param([string[]]$Names) + + $snapshot = [ordered]@{} + foreach ($name in $Names) { + $value = [System.Environment]::GetEnvironmentVariable( + $name, + [System.EnvironmentVariableTarget]::Process + ) + $snapshot[$name] = [ordered]@{ + present = $null -ne $value + value = $value + } + } + return $snapshot +} + +function Restore-SelectedProcessEnvironment { + param([System.Collections.IDictionary]$Snapshot) + + foreach ($name in $Snapshot.Keys) { + $entry = $Snapshot[$name] + $value = $null + if ([bool]$entry.present) { + $value = [string]$entry.value + } + [System.Environment]::SetEnvironmentVariable( + [string]$name, + $value, + [System.EnvironmentVariableTarget]::Process + ) + } +} + +function Assert-NewApiEnvironmentMatches { + param( + [System.Collections.IDictionary]$Expected, + [System.Collections.IDictionary]$ActualIdentity + ) + + foreach ($name in $Expected.Keys) { + $actual = Get-IdentityEnvironmentValue ` + -Environment $ActualIdentity.environment ` + -Name ([string]$name) ` + -AllowEmpty + if ([string]$Expected[$name] -cne $actual) { + throw "New API required environment digest drift" + } + } +} + +function Get-RequiredEnvironmentDigest { + param([System.Collections.IDictionary]$Environment) + + $parts = @() + foreach ($name in @($Environment.Keys | Sort-Object)) { + $parts += ([string]$name + "=" + (Get-Utf8Sha256 -Value ([string]$Environment[$name]))) + } + return Get-Utf8Sha256 -Value (@($parts) -join "`n") +} + +function Get-ConnectedDatabaseTargetSha256 { + param([string]$ProbePath) + + $probeOutput = @(& $resolvedPythonPath @("-X", "utf8", $ProbePath)) + $probeExit = $LASTEXITCODE + if ($probeExit -ne 0 -or $probeOutput.Count -ne 1) { + throw "Connected database target identity could not be proven" + } + try { + $payload = $probeOutput[0] | ConvertFrom-Json + } catch { + throw "Connected database target identity proof is invalid" + } + $digest = [string]$payload.database_target_sha256 + if ($payload.status -ne "passed" -or $digest -notmatch "^[0-9a-f]{64}$") { + throw "Connected database target identity proof failed" + } + return $digest +} + +function Set-CompleteProcessEnvironment { + param([System.Collections.IDictionary]$Environment) + + foreach ($name in @("SystemRoot", "windir", "SystemDrive", "ComSpec")) { + $missing = -not $Environment.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$Environment[$name]) + if ($missing) { + $machineValue = [System.Environment]::GetEnvironmentVariable($name, "Machine") + if ($machineValue) { + $Environment[$name] = $machineValue + } + } + } + $current = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + foreach ($name in @($current.Keys)) { + [System.Environment]::SetEnvironmentVariable( + [string]$name, + $null, + [System.EnvironmentVariableTarget]::Process + ) + } + foreach ($name in $Environment.Keys) { + [System.Environment]::SetEnvironmentVariable( + [string]$name, + [string]$Environment[$name], + [System.EnvironmentVariableTarget]::Process + ) + } +} + +function ConvertTo-WindowsCommandLineArgument { + param([AllowEmptyString()][string]$Argument) + + if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') { + return $Argument + } + $builder = New-Object System.Text.StringBuilder + $null = $builder.Append('"') + $backslashes = 0 + foreach ($character in $Argument.ToCharArray()) { + if ($character -eq '\') { + $backslashes++ + continue + } + if ($character -eq '"') { + $null = $builder.Append(('\' * (($backslashes * 2) + 1))) + $null = $builder.Append('"') + $backslashes = 0 + continue + } + if ($backslashes -gt 0) { + $null = $builder.Append(('\' * $backslashes)) + $backslashes = 0 + } + $null = $builder.Append($character) + } + if ($backslashes -gt 0) { + $null = $builder.Append(('\' * ($backslashes * 2))) + } + $null = $builder.Append('"') + return $builder.ToString() +} + +function Join-WindowsArgumentList { + param([object[]]$ArgumentList) + + return (@( + foreach ($argument in $ArgumentList) { + ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument) + } + ) -join " ") +} + +function Start-PinnedPriorProcess { + param( + [System.Collections.IDictionary]$Identity, + [string]$Role, + [string]$LogDirectory + ) + + Assert-FileSha256 ` + -Path ([string]$Identity.executable_path) ` + -ExpectedSha256 ([string]$Identity.executable_sha256) ` + -Role $Role + if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) { + throw "$Role working directory is unavailable" + } + if (@($Identity.argument_list).Count -eq 0 -or $Identity.environment.Count -eq 0) { + throw "$Role restart inputs are incomplete" + } + $callerEnvironment = Save-CompleteProcessEnvironment + $suffix = [Guid]::NewGuid().ToString("N") + try { + Set-CompleteProcessEnvironment -Environment $Identity.environment + $argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list) + return Start-Process ` + -WindowStyle Hidden ` + -FilePath $Identity.executable_path ` + -ArgumentList $argumentString ` + -WorkingDirectory $Identity.cwd ` + -RedirectStandardOutput (Join-Path $LogDirectory "$Role-rollback-$suffix.out.log") ` + -RedirectStandardError (Join-Path $LogDirectory "$Role-rollback-$suffix.err.log") ` + -PassThru + } finally { + Set-CompleteProcessEnvironment -Environment $callerEnvironment + } +} + +function Get-JsonHealth { + param([string]$Uri) + + try { + return Invoke-RestMethod -Uri $Uri -Method Get -UseBasicParsing -TimeoutSec 5 + } catch { + return $null + } +} + +function Wait-ApiHealth { + param( + [string]$Uri, + [int]$TimeoutSec, + [string]$ExpectedManifestSha256 = "", + [Nullable[bool]]$ExpectedFreezeActive = $null, + [string]$ExpectedFreezeTokenSha256 = "" + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $health = Get-JsonHealth -Uri $Uri + $healthy = ( + $null -ne $health -and + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + ) + if ($healthy -and $ExpectedManifestSha256) { + $manifest = $health.upload_manifest + $healthy = ( + $null -ne $manifest -and + $manifest.required -eq $true -and + $manifest.validated -eq $true -and + [string]$manifest.manifest_sha256 -ceq $ExpectedManifestSha256 + ) + } + if ($healthy -and $null -ne $ExpectedFreezeActive) { + $freeze = $health.upload_write_freeze + $healthy = ( + $null -ne $freeze -and + $freeze.capable -eq $true -and + [bool]$freeze.active -eq [bool]$ExpectedFreezeActive -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 + ) + if ($healthy -and [bool]$ExpectedFreezeActive) { + $healthy = [string]$freeze.token_sha256 -ceq $ExpectedFreezeTokenSha256 + } + } + if ($healthy) { + return $health + } + Start-Sleep -Milliseconds 500 + } while ((Get-Date) -lt $deadline) + throw "Runtime health contract was not proven" +} + +function Wait-GoogleAuthContract { + param( + [string]$Uri, + [int]$TimeoutSec + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $config = Get-JsonHealth -Uri $Uri + if ($null -ne $config -and $config.google_oauth_configured -eq $true) { + $enabledProviders = @( + $config.providers | + Where-Object { $_.enabled -eq $true } | + ForEach-Object { [string]$_.provider } + ) + $google = @( + $config.providers | + Where-Object { [string]$_.provider -eq "google" } + ) + if ( + $google.Count -eq 1 -and + $google[0].configured -eq $true -and + $google[0].enabled -eq $true -and + $enabledProviders.Count -eq 1 -and + $enabledProviders[0] -eq "google" -and + $config.dev_login_enabled -eq $false + ) { + return + } + } + Start-Sleep -Milliseconds 500 + } while ((Get-Date) -lt $deadline) + throw "Google-only authentication contract was not proven" +} + +function Write-PrivacySafeReceiptCreateOnly { + param( + [string]$Path, + [System.Collections.IDictionary]$Payload + ) + + if ([System.IO.File]::Exists($Path)) { + throw "Bootstrap receipt target already exists" + } + $json = ConvertTo-Json -InputObject $Payload -Depth 10 -Compress + $bytes = [System.Text.UTF8Encoding]::new($false).GetBytes( + $json + [Environment]::NewLine + ) + $stream = $null + try { + $stream = [System.IO.File]::Open( + $Path, + [System.IO.FileMode]::CreateNew, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None + ) + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush($true) + } finally { + if ($null -ne $stream) { + $stream.Dispose() + } + } + $actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -notmatch "^[0-9a-f]{64}$") { + throw "Bootstrap receipt verification failed" + } + return $actual +} + +function Enter-InheritedBootstrapRecoveryLock { + param( + [string]$Path, + [string]$SourceRoot, + [string]$UploadRoot + ) + + if (-not [System.IO.Path]::IsPathRooted($Path)) { + throw "Bootstrap recovery lock path must be absolute" + } + $full = Get-PublicRuntimeCanonicalPath -Path $Path + foreach ($boundary in @($SourceRoot, $UploadRoot)) { + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $full -Parent $boundary) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $boundary -Parent $full) + ) { + throw "Bootstrap recovery lock must be private and disjoint" + } + } + $parent = [System.IO.Path]::GetDirectoryName($full) + [System.IO.Directory]::CreateDirectory($parent) | Out-Null + Assert-PublicRuntimePathHasNoReparsePoint -Path $full + $stream = [System.IO.File]::Open( + $full, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::Read + ) + try { + $nonce = [Guid]::NewGuid().ToString("N") + [Guid]::NewGuid().ToString("N") + $ownerStartedAtUtc = ( + Get-Process -Id $PID -ErrorAction Stop + ).StartTime.ToUniversalTime().ToString("o") + $payload = [ordered]@{ + schema_version = "vignette.public-runtime-inherited-lock.v1" + status = "held" + owner_pid = $PID + owner_started_at_utc = $ownerStartedAtUtc + nonce_sha256 = Get-Utf8Sha256 -Value $nonce + source_commit = $ExpectedSourceCommit + source_tree = $ExpectedSourceTree + } + $json = ConvertTo-Json -InputObject $payload -Compress + $bytes = [System.Text.UTF8Encoding]::new($false).GetBytes( + $json + [Environment]::NewLine + ) + $stream.SetLength(0) + $stream.Position = 0 + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush($true) + $receiptSha256 = Get-Utf8Sha256 -Value ($json + [Environment]::NewLine) + return [pscustomobject]@{ + Stream = $stream + Path = $full + ReceiptSha256 = $receiptSha256 + } + } catch { + $stream.Dispose() + throw + } +} + +function Get-PublicRuntimeTaskTruth { + param( + [object[]]$Snapshot, + [switch]$ExpectedRestored + ) + + $enabledCount = 0 + $runningCount = 0 + $restored = $true + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists) { + continue + } + $task = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -ErrorAction Stop + $enabled = [bool]$task.Settings.Enabled + if ($enabled) { + $enabledCount++ + } + if ([string]$task.State -eq "Running") { + $runningCount++ + } + if ($ExpectedRestored -and $enabled -ne [bool]$entry.was_enabled) { + $restored = $false + } + } + return [ordered]@{ + enabled_count = $enabledCount + running_count = $runningCount + all_disabled_and_idle = ($enabledCount -eq 0 -and $runningCount -eq 0) + expected_state_restored = $restored + } +} + +function Assert-OfflinedRuntimeRaceGate { + param( + [object[]]$TaskSnapshot, + [string]$ConfigPath + ) + + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $TaskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-LoopbackListenerAbsent ` + -Port $ApiPort ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-TunnelAbsent ` + -ConfigPath $ConfigPath ` + -TimeoutSec $ProcessStopTimeoutSeconds +} + +function Assert-ListenerIdentityUnchanged { + param( + [int]$Port, + [System.Collections.IDictionary]$Expected, + [string]$Role + ) + + $listenerProcessId = Get-ExactLoopbackListenerPid -Port $Port + if ($listenerProcessId -ne [int]$Expected.pid) { + throw "$Role listener PID drift" + } + $actual = Wait-ProcessIdentity ` + -ProcessId $listenerProcessId ` + -Role $Role ` + -ExpectedCwd ([string]$Expected.cwd) ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-IdentityUnchanged -Expected $Expected -Actual $actual -Role $Role +} + +function Get-OwnedFreezeTokenSha256 { + param([string]$Path) + + Assert-PublicRuntimePathHasNoReparsePoint -Path $Path + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Owned bootstrap write-freeze is unavailable" + } + $payload = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json + if ( + $null -eq $payload -or + $payload.schema_version -ne "vignette.public-upload-write-freeze.v1" -or + [string]::IsNullOrWhiteSpace([string]$payload.token) + ) { + throw "Owned bootstrap write-freeze is invalid" + } + return Get-Utf8Sha256 -Value ([string]$payload.token) +} + +function Remove-OwnedFreezeByHash { + param( + [string]$Path, + [string]$ExpectedTokenSha256 + ) + + if (-not [System.IO.File]::Exists($Path)) { + return + } + $payload = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json + if ( + $null -eq $payload -or + $payload.schema_version -ne "vignette.public-upload-write-freeze.v1" -or + [string]::IsNullOrWhiteSpace([string]$payload.token) -or + (Get-Utf8Sha256 -Value ([string]$payload.token)) -cne $ExpectedTokenSha256 + ) { + throw "Bootstrap write-freeze ownership proof failed" + } + [System.IO.File]::Delete($Path) + if ([System.IO.File]::Exists($Path)) { + throw "Bootstrap write-freeze deletion failed" + } +} + +function Get-ValidatedExplicitLegacySourceRoots { + param([string[]]$AllowedRoots) + + if ($AllowedRoots.Count -ne 3) { + throw "Legacy bootstrap requires exactly three explicit source roots" + } + $resolvedAllowedRoots = @() + foreach ($allowed in $AllowedRoots) { + if (-not [System.IO.Path]::IsPathRooted($allowed)) { + throw "Allowed legacy upload root is not absolute" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $allowed + if (-not (Test-Path -LiteralPath $allowed -PathType Container)) { + throw "Allowed legacy upload root is unavailable" + } + $resolvedAllowed = (Resolve-Path -LiteralPath $allowed).Path + foreach ($existingAllowed in $resolvedAllowedRoots) { + if ([string]::Equals( + $existingAllowed, + $resolvedAllowed, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Allowed legacy upload roots contain a duplicate" + } + } + $resolvedAllowedRoots += $resolvedAllowed + } + return $resolvedAllowedRoots +} + +function Get-ValidatedLegacySourceRoots { + param( + [System.Collections.IDictionary]$ApiIdentity, + [string[]]$AllowedRoots + ) + + $declared = $null + $declaredPresent = $false + foreach ($entry in $ApiIdentity.environment.Keys) { + if ([string]$entry -ieq "USER_UPLOAD_DIR") { + $declaredPresent = $true + $candidate = [string]$ApiIdentity.environment[$entry] + if ([string]::IsNullOrWhiteSpace($candidate)) { + throw "Legacy USER_UPLOAD_DIR is present but empty" + } + $declared = $candidate + break + } + } + if (-not $declaredPresent) { + $declared = Join-Path ([string]$ApiIdentity.cwd) "uploads" + } + if (-not [System.IO.Path]::IsPathRooted($declared)) { + throw "Legacy upload source is not absolute" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $declared + if (-not (Test-Path -LiteralPath $declared -PathType Container)) { + throw "Legacy upload source is unavailable" + } + $resolvedDeclared = (Resolve-Path -LiteralPath $declared).Path + $resolvedAllowedRoots = @( + Get-ValidatedExplicitLegacySourceRoots -AllowedRoots $AllowedRoots + ) + $matches = @() + foreach ($resolvedAllowed in $resolvedAllowedRoots) { + if ([string]::Equals( + $resolvedAllowed, + $resolvedDeclared, + [System.StringComparison]::OrdinalIgnoreCase + )) { + $matches += $resolvedAllowed + } + } + if ($matches.Count -ne 1) { + throw "Legacy upload source is not one explicitly allowed root" + } + return $resolvedAllowedRoots +} + +function Restore-LegacyRuntime { + param( + [System.Collections.IDictionary]$PriorApi, + [System.Collections.IDictionary]$PriorTunnel, + [string]$ConfigPath, + [string]$FreezePath, + [string]$FreezeTokenSha256, + [string]$PrivateStateRoot, + [string]$PriorEnvironmentPath, + [string]$PriorEnvironmentSha256, + [System.Collections.IDictionary]$OwnedNewApi, + [System.Collections.IDictionary]$OwnedNewTunnel + ) + + Assert-EnvironmentFilePinned ` + -Path $PriorEnvironmentPath ` + -ExpectedSha256 $PriorEnvironmentSha256 ` + -Role "prior API rollback" + # 공개 ingress를 먼저 닫아 rollback 중 교체 API가 외부 요청을 받지 않게 한다. + $currentTunnels = @(Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath) + if ($currentTunnels.Count -gt 1) { + throw "Rollback found ambiguous tunnel identities" + } + $restoredTunnel = $null + if ($currentTunnels.Count -eq 1) { + if (Test-IdentityExactlyMatches -Expected $PriorTunnel -Actual $currentTunnels[0]) { + $restoredTunnel = $currentTunnels[0] + } elseif (Test-IdentityExactlyMatches -Expected $OwnedNewTunnel -Actual $currentTunnels[0]) { + Stop-VerifiedIdentity -Identity $OwnedNewTunnel -Role "owned new tunnel" + } else { + throw "Rollback refuses to stop an unowned tunnel" + } + } + if ($null -eq $restoredTunnel) { + Assert-TunnelAbsent -ConfigPath $ConfigPath -TimeoutSec $ProcessStopTimeoutSeconds + } + + $listeners = @( + Get-NetTCPConnection -State Listen -LocalPort $ApiPort -ErrorAction SilentlyContinue + ) + $listenerPids = @($listeners | ForEach-Object { [int]$_.OwningProcess } | Sort-Object -Unique) + if ($listenerPids.Count -gt 1) { + throw "Rollback found ambiguous API listeners" + } + $restoredApi = $null + if ($listenerPids.Count -eq 1) { + $currentApi = Wait-ProcessIdentity ` + -ProcessId ([int]$listenerPids[0]) ` + -Role "rollback API listener" ` + -TimeoutSec $ProcessStopTimeoutSeconds + if (Test-IdentityExactlyMatches -Expected $PriorApi -Actual $currentApi) { + $restoredApi = $currentApi + } elseif (Test-IdentityExactlyMatches -Expected $OwnedNewApi -Actual $currentApi) { + Stop-VerifiedIdentity -Identity $OwnedNewApi -Role "owned new API" + } else { + throw "Rollback refuses to stop an unowned API listener" + } + } + if ($null -eq $restoredApi) { + Assert-LoopbackListenerAbsent -Port $ApiPort -TimeoutSec $ProcessStopTimeoutSeconds + } + + if ([System.IO.File]::Exists($FreezePath)) { + if ($FreezeTokenSha256 -notmatch "^[0-9a-f]{64}$") { + throw "Rollback cannot prove write-freeze ownership" + } + Remove-OwnedFreezeByHash ` + -Path $FreezePath ` + -ExpectedTokenSha256 $FreezeTokenSha256 + } + + if ($null -eq $restoredApi) { + $apiProcess = Start-PinnedPriorProcess ` + -Identity $PriorApi ` + -Role "legacy-api" ` + -LogDirectory $PrivateStateRoot + $restoredApi = Wait-ProcessIdentity ` + -ProcessId $apiProcess.Id ` + -Role "restored legacy API" ` + -ExpectedCwd ([string]$PriorApi.cwd) ` + -TimeoutSec $ProcessStopTimeoutSeconds + } + foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) { + if ($PriorApi[$field].ToString() -cne $restoredApi[$field].ToString()) { + throw "Restored legacy API identity drift" + } + } + $null = Wait-ApiHealth ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec $HealthTimeoutSeconds + Wait-GoogleAuthContract ` + -Uri "http://127.0.0.1:$ApiPort/auth/config" ` + -TimeoutSec $HealthTimeoutSeconds + + if ($null -eq $restoredTunnel) { + $tunnelProcess = Start-PinnedPriorProcess ` + -Identity $PriorTunnel ` + -Role "legacy-tunnel" ` + -LogDirectory $PrivateStateRoot + $restoredTunnel = Wait-ProcessIdentity ` + -ProcessId $tunnelProcess.Id ` + -Role "restored legacy tunnel" ` + -ExpectedCwd ([string]$PriorTunnel.cwd) ` + -TimeoutSec $ProcessStopTimeoutSeconds + } + foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) { + if ($PriorTunnel[$field].ToString() -cne $restoredTunnel[$field].ToString()) { + throw "Restored legacy tunnel identity drift" + } + } + $null = Wait-ApiHealth -Uri $PublicHealthUrl -TimeoutSec $HealthTimeoutSeconds + Wait-GoogleAuthContract ` + -Uri $publicAuthConfigUrl ` + -TimeoutSec $HealthTimeoutSeconds + if ([System.IO.File]::Exists($FreezePath)) { + throw "Legacy rollback left the write-freeze sentinel present" + } +} + +$resolvedStableSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path +if ($PublicHealthUrl -cne "https://api-vignette.chanpaca.net/health") { + throw "Legacy bootstrap public health URL must be canonical" +} +$uploadContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-upload-root.ps1" +$taskContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-task-maintenance.ps1" +$taskDefinitionContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-task-definition-cutover.ps1" +$bootTaskInstaller = Join-Path $resolvedStableSourceRoot "scripts\register-boot-task.ps1" +$watchdogTaskInstaller = Join-Path $resolvedStableSourceRoot "scripts\install-public-runtime-task.ps1" +$initializer = Join-Path $resolvedStableSourceRoot "scripts\initialize-public-runtime-upload-root.ps1" +$initializerWorker = Join-Path $resolvedStableSourceRoot "scripts\initialize-public-runtime-upload-root.py" +$startScript = Join-Path $resolvedStableSourceRoot "scripts\start-public-runtime.ps1" +$manifestProbe = Join-Path $resolvedStableSourceRoot "scripts\validate-public-runtime-upload-manifest.py" +$uploadProbe = Join-Path $resolvedStableSourceRoot "scripts\probe-public-runtime-upload-root.py" +$databaseIdentityProbe = Join-Path $resolvedStableSourceRoot "scripts\probe-public-runtime-database-identity.py" +foreach ($required in @( + $uploadContract, + $taskContract, + $taskDefinitionContract, + $bootTaskInstaller, + $watchdogTaskInstaller, + $initializer, + $initializerWorker, + $startScript, + $manifestProbe, + $uploadProbe, + $databaseIdentityProbe, + $PythonPath, + $CloudflaredPath, + $CloudflaredConfigPath +)) { + if (-not (Test-Path -LiteralPath $required -PathType Leaf)) { + throw "Legacy bootstrap prerequisite is unavailable" + } +} +. $uploadContract +. $taskContract +. $taskDefinitionContract + +Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot +$resolvedPythonPath = (Resolve-Path -LiteralPath $PythonPath).Path +$resolvedCloudflaredPath = (Resolve-Path -LiteralPath $CloudflaredPath).Path +$resolvedCloudflaredConfigPath = (Resolve-Path -LiteralPath $CloudflaredConfigPath).Path +Assert-FileSha256 -Path $resolvedPythonPath -ExpectedSha256 $ExpectedPythonSha256 -Role "Python" +Assert-FileSha256 -Path $resolvedCloudflaredPath -ExpectedSha256 $ExpectedCloudflaredSha256 -Role "cloudflared" +Assert-FileSha256 ` + -Path $resolvedCloudflaredConfigPath ` + -ExpectedSha256 $ExpectedCloudflaredConfigSha256 ` + -Role "cloudflared config" + +$targetCanonical = Get-PublicRuntimeCanonicalPath -Path $UserUploadDir +$resolvedPrivateStateDir = Resolve-PrivateBootstrapStateDirectory ` + -Path $ManifestStateDir ` + -SourceRoot $resolvedStableSourceRoot ` + -UploadRoot $targetCanonical +$resolvedFreezePath = Resolve-PrivateBootstrapPath ` + -Path $UserUploadWriteFreezePath ` + -StateRoot $resolvedPrivateStateDir +$resolvedCutoverReceiptPath = Resolve-PrivateBootstrapPath ` + -Path $CutoverReceiptPath ` + -StateRoot $resolvedPrivateStateDir +$resolvedTaskRecoveryReceiptPath = Resolve-PrivateBootstrapPath ` + -Path $TaskRecoveryReceiptPath ` + -StateRoot $resolvedPrivateStateDir +foreach ($absentTarget in @( + $resolvedFreezePath, + $resolvedCutoverReceiptPath, + $resolvedTaskRecoveryReceiptPath +)) { + if (Test-Path -LiteralPath $absentTarget) { + throw "Legacy bootstrap fixed private-state target must be absent at preflight" + } +} +$fixedPrivateTargets = @( + $resolvedFreezePath, + $resolvedCutoverReceiptPath, + $resolvedTaskRecoveryReceiptPath, + (Get-PublicRuntimeCanonicalPath -Path $RecoveryLockPath) +) +if (@($fixedPrivateTargets | Sort-Object -Unique).Count -ne $fixedPrivateTargets.Count) { + throw "Legacy bootstrap private-state paths must be pairwise distinct" +} +$preflightLegacySourceRoots = @( + Get-ValidatedExplicitLegacySourceRoots ` + -AllowedRoots $AllowedLegacySourceUploadDir +) +foreach ($sourceRoot in $preflightLegacySourceRoots) { + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $sourceRoot -Parent $targetCanonical) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $targetCanonical -Parent $sourceRoot) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $sourceRoot -Parent $resolvedPrivateStateDir) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedPrivateStateDir -Parent $sourceRoot) + ) { + throw "Legacy source roots must be disjoint from public and private roots" + } +} +$preservedProbeArgs = @( + "-X", "utf8", "-B", $initializerWorker, + "probe-preserved-inventory", + "--expected-preserved-object-count", $ExpectedPreservedObjectCount.ToString(), + "--expected-preserved-total-size-bytes", $ExpectedPreservedTotalSizeBytes.ToString(), + "--expected-preserved-inventory-sha256", $ExpectedPreservedInventorySha256 +) +foreach ($sourceRoot in $preflightLegacySourceRoots) { + $preservedProbeArgs += @("--source-root", $sourceRoot) +} +$preservedProbeOutput = @(& $resolvedPythonPath @preservedProbeArgs) +$preservedProbeExit = $LASTEXITCODE +$preservedProbePayload = $null +if ($preservedProbeExit -eq 0) { + try { + $preservedProbePayload = (@($preservedProbeOutput) -join "").Trim() | + ConvertFrom-Json + } catch { + $preservedProbePayload = $null + } +} +if ( + $preservedProbeExit -ne 0 -or + $null -eq $preservedProbePayload -or + $preservedProbePayload.status -cne "verified" -or + [int]$preservedProbePayload.preserved_object_count -ne $ExpectedPreservedObjectCount -or + [long]$preservedProbePayload.preserved_total_size_bytes -ne + $ExpectedPreservedTotalSizeBytes -or + [string]$preservedProbePayload.preserved_inventory_sha256 -cne $ExpectedPreservedInventorySha256 +) { + throw "Legacy preserved source inventory preflight failed" +} +$preservedProbeDecodeCounts = Get-PreservedDecodeCountProof ` + -Payload $preservedProbePayload ` + -PreservedObjectCount $ExpectedPreservedObjectCount ` + -Role "Legacy preserved source inventory preflight" +$bootstrapLock = Enter-InheritedBootstrapRecoveryLock ` + -Path $RecoveryLockPath ` + -SourceRoot $resolvedStableSourceRoot ` + -UploadRoot $targetCanonical + +$taskSnapshot = @() +$taskMaintenanceEntered = $false +$originalTaskDefinitionSnapshot = $null +$disabledOriginalTaskDefinitionSnapshot = $null +$newDisabledTaskDefinitionSnapshot = $null +$operationalTaskDefinitionSnapshot = $null +$runtimeMutationStarted = $false +$noRollback = $false +$failureStage = "preflight" +$priorApiIdentity = $null +$priorTunnelIdentity = $null +$whisperIdentity = $null +$meloTtsIdentity = $null +$newApiIdentity = $null +$newTunnelIdentity = $null +$newApiLaunchAttempted = $false +$newTunnelProcess = $null +$freezeAbsentBeforeInitializer = $false +$requiredApiEnvironment = $null +$callerRequiredEnvironmentSnapshot = $null +$priorDatabaseTargetSha256 = "" +$requiredEnvironmentDigest = "" +$priorEnvironmentFilePath = "" +$priorEnvironmentFileSha256 = "" +$stableEnvironmentFilePath = Join-Path $resolvedStableSourceRoot "apps\api\.env" +$stableEnvironmentFileSha256 = "" +$publicAuthConfigUrl = $PublicHealthUrl -replace '/health(?:\?.*)?$', '/auth/config' +$freezeTokenSha256 = "" +$cutoverReceiptSha256 = "" +$taskRecoveryReceiptSha256 = "" + +try { + # LEGACY_BOOTSTRAP_STAGE:task_maintenance_enter + $failureStage = "task_maintenance_enter" + Assert-PublicRuntimeCoordinatedTaskNamesExact ` + -TaskNames $CoordinatedTaskNames + $originalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot ` + -RequireEnabled + $taskSnapshot = @( + Enter-PublicRuntimeTaskMaintenance ` + -TaskNames $CoordinatedTaskNames ` + -TimeoutSec $ProcessStopTimeoutSeconds + ) + $taskMaintenanceEntered = $true + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $taskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + $disabledOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot + foreach ($taskDefinition in @($disabledOriginalTaskDefinitionSnapshot.entries)) { + if ([bool]$taskDefinition.enabled) { + throw "Original public runtime task definition remained enabled during maintenance" + } + } + + # LEGACY_BOOTSTRAP_STAGE:exact_runtime_capture + $failureStage = "exact_runtime_capture" + $priorApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort + $priorApiIdentity = Wait-ProcessIdentity ` + -ProcessId $priorApiPid ` + -Role "legacy API" ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-ApiIdentityContract -Identity $priorApiIdentity + $legacySourceProof = Assert-LegacyApiSourceProvenance ` + -ApiIdentity $priorApiIdentity + $requiredApiEnvironmentNames = @( + "DATABASE_URL", "SESSION_SECRET", "ENGINE_URL", "ENGINE_GATEWAY_SHARED_SECRET", + "OAUTH_GOOGLE_CLIENT_ID", "OAUTH_GOOGLE_CLIENT_SECRET", "OAUTH_REDIRECT_URI", + "AUTH_ALLOWED_EMAIL_DOMAINS", "AUTH_TEACHER_EMAILS", "AUTH_ADMIN_EMAILS", + "AUTH_SUPER_ADMIN_EMAILS", "AUTH_APPROVED_EMAILS", "AUTH_NEW_USER_DEFAULT_STATUS", + "AUTH_EMAIL_COHORT_MAP", "AUTH_DOMAIN_COHORT_MAP", "DEFAULT_AFFILIATION" + ) + $callerRequiredEnvironmentSnapshot = Save-SelectedProcessEnvironment ` + -Names $requiredApiEnvironmentNames + $priorEnvironmentFilePath = (Resolve-Path -LiteralPath ( + Join-Path $priorApiIdentity.cwd ".env" + )).Path + Assert-PublicRuntimePathHasNoReparsePoint -Path $priorEnvironmentFilePath + $priorEnvironmentFileSha256 = ( + Get-FileHash -LiteralPath $priorEnvironmentFilePath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + $requiredApiEnvironment = Set-RequiredApiEnvironmentFromIdentity ` + -Identity $priorApiIdentity ` + -ExpectedEnvironmentFileSha256 $priorEnvironmentFileSha256 + Ensure-ReleaseEnvironmentFile ` + -PriorApiIdentity $priorApiIdentity ` + -ExpectedSourceSha256 $priorEnvironmentFileSha256 + Assert-EnvironmentFilePinned ` + -Path $priorEnvironmentFilePath ` + -ExpectedSha256 $priorEnvironmentFileSha256 ` + -Role "prior API after release environment preparation" + $stableEnvironmentFilePath = (Resolve-Path -LiteralPath $stableEnvironmentFilePath).Path + $stableEnvironmentFileSha256 = ( + Get-FileHash -LiteralPath $stableEnvironmentFilePath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + if ($stableEnvironmentFileSha256 -cne $priorEnvironmentFileSha256) { + throw "New release environment file is not exact to the prior runtime" + } + $futureRequiredApiEnvironment = Get-FutureLauncherRequiredApiSettings ` + -PriorApiIdentity $priorApiIdentity ` + -RequiredNames $requiredApiEnvironmentNames ` + -EnginePort $EnginePort + Assert-RequiredApiSettingsEqual ` + -Expected $requiredApiEnvironment ` + -Actual $futureRequiredApiEnvironment ` + -Role "future boot/watchdog launcher" + $priorDatabaseTargetSha256 = Get-ConnectedDatabaseTargetSha256 ` + -ProbePath $databaseIdentityProbe + $requiredEnvironmentDigest = Get-RequiredEnvironmentDigest ` + -Environment $requiredApiEnvironment + $priorTunnelIdentity = Get-ExactTunnelIdentity ` + -ConfigPath $resolvedCloudflaredConfigPath + $whisperPid = Get-ExactLoopbackListenerPid -Port $WhisperPort + $whisperIdentity = Wait-ProcessIdentity ` + -ProcessId $whisperPid ` + -Role "whisper sidecar" ` + -TimeoutSec $ProcessStopTimeoutSeconds + $meloTtsPid = Get-ExactLoopbackListenerPid -Port $MeloTtsPort + $meloTtsIdentity = Wait-ProcessIdentity ` + -ProcessId $meloTtsPid ` + -Role "MeloTTS sidecar" ` + -TimeoutSec $ProcessStopTimeoutSeconds + $legacySourceRoots = @( + Get-ValidatedLegacySourceRoots ` + -ApiIdentity $priorApiIdentity ` + -AllowedRoots $preflightLegacySourceRoots + ) + $null = Wait-ApiHealth ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec $HealthTimeoutSeconds + $null = Wait-ApiHealth -Uri $PublicHealthUrl -TimeoutSec $HealthTimeoutSeconds + + # LEGACY_BOOTSTRAP_STAGE:tunnel_quiescence + $failureStage = "tunnel_quiescence" + $runtimeMutationStarted = $true + Stop-VerifiedIdentity -Identity $priorTunnelIdentity -Role "legacy tunnel" + Assert-TunnelAbsent ` + -ConfigPath $resolvedCloudflaredConfigPath ` + -TimeoutSec $ProcessStopTimeoutSeconds + + # LEGACY_BOOTSTRAP_STAGE:listener_quiescence + $failureStage = "listener_quiescence" + Stop-VerifiedIdentity -Identity $priorApiIdentity -Role "legacy API" + Assert-LoopbackListenerAbsent ` + -Port $ApiPort ` + -TimeoutSec $ProcessStopTimeoutSeconds + + $sourceRootSha256s = @() + foreach ($legacySourceRoot in $legacySourceRoots) { + $sourceRootSha256s += Get-CanonicalPathSha256 -Path $legacySourceRoot + } + $sortedSourceRootSha256s = @($sourceRootSha256s | Sort-Object) + $sourceRootSetJson = ConvertTo-Json ` + -InputObject @($sortedSourceRootSha256s) ` + -Compress + $capture = [ordered]@{ + schema_version = "vignette.public-upload-offline-quiescence-capture.v2" + status = "quiesced" + captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + source_commit = [string]$legacySourceProof.commit + source_tree = [string]$legacySourceProof.tree + source_root_sha256s = @($sourceRootSha256s) + source_root_set_sha256 = Get-Utf8Sha256 -Value $sourceRootSetJson + api_identity = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity + tunnel_identity = ConvertTo-SafeProcessIdentity -Identity $priorTunnelIdentity + listener_absent = $true + tunnel_absent = $true + listener_endpoint_sha256 = Get-Utf8Sha256 -Value "tcp://127.0.0.1:$ApiPort" + tunnel_config_sha256 = $ExpectedCloudflaredConfigSha256 + } + $captureJson = ConvertTo-Json -InputObject $capture -Depth 8 -Compress + $captureBase64 = [Convert]::ToBase64String( + [System.Text.UTF8Encoding]::new($false).GetBytes($captureJson) + ) + + # LEGACY_BOOTSTRAP_STAGE:offline_initializer + $failureStage = "offline_initializer" + Assert-OfflinedRuntimeRaceGate ` + -TaskSnapshot $taskSnapshot ` + -ConfigPath $resolvedCloudflaredConfigPath + Assert-EnvironmentFilePinned ` + -Path $priorEnvironmentFilePath ` + -ExpectedSha256 $priorEnvironmentFileSha256 ` + -Role "prior API" + Assert-EnvironmentFilePinned ` + -Path $stableEnvironmentFilePath ` + -ExpectedSha256 $stableEnvironmentFileSha256 ` + -Role "new release" + if ([System.IO.File]::Exists($resolvedFreezePath)) { + throw "Offline initializer requires an absent write-freeze target" + } + $freezeAbsentBeforeInitializer = $true + $initializerOutput = @( + & $initializer ` + -StableSourceRoot $resolvedStableSourceRoot ` + -UserUploadDir $UserUploadDir ` + -ManifestStateDir $resolvedPrivateStateDir ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -ExpectedReferenceCount $ExpectedReferenceCount ` + -ExpectedPreservedObjectCount $ExpectedPreservedObjectCount ` + -ExpectedPreservedTotalSizeBytes $ExpectedPreservedTotalSizeBytes ` + -ExpectedPreservedInventorySha256 $ExpectedPreservedInventorySha256 ` + -SourceUploadDir $legacySourceRoots ` + -OfflineQuiescenceCaptureBase64 $captureBase64 ` + -ExpectedOfflineSourceCommit $ExpectedLegacySourceCommit ` + -ExpectedOfflineSourceTree $ExpectedLegacySourceTree ` + -PythonPath $resolvedPythonPath + ) + $freezeTokenSha256 = Get-OwnedFreezeTokenSha256 -Path $resolvedFreezePath + Assert-EnvironmentFilePinned ` + -Path $priorEnvironmentFilePath ` + -ExpectedSha256 $priorEnvironmentFileSha256 ` + -Role "prior API" + Assert-EnvironmentFilePinned ` + -Path $stableEnvironmentFilePath ` + -ExpectedSha256 $stableEnvironmentFileSha256 ` + -Role "new release" + Assert-OfflinedRuntimeRaceGate ` + -TaskSnapshot $taskSnapshot ` + -ConfigPath $resolvedCloudflaredConfigPath + $initializerPayload = ((@($initializerOutput) -join "").Trim()) | ConvertFrom-Json + if ( + $null -eq $initializerPayload -or + $initializerPayload.status -ne "initialized" -or + [string]$initializerPayload.manifest_sha256 -notmatch "^[0-9a-f]{64}$" -or + [string]$initializerPayload.offline_quiescence_receipt_sha256 -notmatch "^[0-9a-f]{64}$" -or + [string]$initializerPayload.write_freeze_token_sha256 -notmatch "^[0-9a-f]{64}$" -or + [int]$initializerPayload.preserved_object_count -ne $ExpectedPreservedObjectCount -or + [long]$initializerPayload.preserved_total_size_bytes -ne + $ExpectedPreservedTotalSizeBytes -or + [string]$initializerPayload.preserved_inventory_sha256 -cne $ExpectedPreservedInventorySha256 + ) { + throw "Offline initializer did not return its privacy-safe proof" + } + $initializerRequiredObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $initializerPayload ` + -Name "required_object_count" ` + -Role "Offline initializer" + $initializerReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $initializerPayload ` + -Name "database_reference_count" ` + -Role "Offline initializer" + if ( + $initializerRequiredObjectCount -gt $initializerReferenceCount -or + $initializerReferenceCount -ne $ExpectedReferenceCount + ) { + throw "Offline initializer DB inventory counts drifted" + } + $initializerPreservedDecodeCounts = Get-PreservedDecodeCountProof ` + -Payload $initializerPayload ` + -PreservedObjectCount $ExpectedPreservedObjectCount ` + -Role "Offline initializer" + $initializerRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof ` + -Payload $initializerPayload ` + -RequiredObjectCount $initializerRequiredObjectCount ` + -RequiredReferenceCount $initializerReferenceCount ` + -Role "Offline initializer" + if ( + $initializerPreservedDecodeCounts.ValidCount -ne + $preservedProbeDecodeCounts.ValidCount -or + $initializerPreservedDecodeCounts.InvalidCount -ne + $preservedProbeDecodeCounts.InvalidCount + ) { + throw "Offline initializer decode inventory drifted from preflight" + } + $manifestSha256 = [string]$initializerPayload.manifest_sha256 + $quiescenceReceiptSha256 = [string]$initializerPayload.offline_quiescence_receipt_sha256 + if ([string]$initializerPayload.write_freeze_token_sha256 -cne $freezeTokenSha256) { + throw "Offline initializer write-freeze proof drift" + } + $manifestPath = Join-Path $resolvedPrivateStateDir "public-avatar-upload-$manifestSha256.json" + $quiescenceReceiptPath = Join-Path $resolvedPrivateStateDir "public-upload-quiescence-$quiescenceReceiptSha256.json" + $resolvedUploadRoot = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedStableSourceRoot ` + -UploadRoot $UserUploadDir ` + -ProbeWritable + $manifestProof = Test-PublicRuntimeUploadManifest ` + -PythonPath $resolvedPythonPath ` + -ProbePath $manifestProbe ` + -UploadRoot $resolvedUploadRoot ` + -ManifestPath $manifestPath ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedWriteFreezePath $resolvedFreezePath + if (-not $manifestProof.Ok) { + throw "Offline initializer manifest validation failed" + } + if ( + [string]$manifestProof.Payload.database_target_sha256 -cne + $priorDatabaseTargetSha256 + ) { + throw "Offline initializer database target drifted from the prior API" + } + if ( + [int]$manifestProof.Payload.preserved_object_count -ne + $ExpectedPreservedObjectCount -or + [long]$manifestProof.Payload.preserved_total_size_bytes -ne + $ExpectedPreservedTotalSizeBytes -or + [string]$manifestProof.Payload.preserved_object_set_sha256 -cne + $ExpectedPreservedInventorySha256 + ) { + throw "Offline initializer preserved inventory proof drifted" + } + $manifestRequiredObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $manifestProof.Payload ` + -Name "required_object_count" ` + -Role "Offline initializer manifest validator" + $manifestRequiredReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $manifestProof.Payload ` + -Name "required_reference_count" ` + -Role "Offline initializer manifest validator" + $manifestCurrentObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $manifestProof.Payload ` + -Name "current_object_count" ` + -Role "Offline initializer manifest validator" + $manifestCurrentReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $manifestProof.Payload ` + -Name "current_reference_count" ` + -Role "Offline initializer manifest validator" + $manifestPreservedDecodeCounts = Get-PreservedDecodeCountProof ` + -Payload $manifestProof.Payload ` + -PreservedObjectCount $ExpectedPreservedObjectCount ` + -Role "Offline initializer manifest validator" + $manifestRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof ` + -Payload $manifestProof.Payload ` + -RequiredObjectCount $manifestRequiredObjectCount ` + -RequiredReferenceCount $manifestRequiredReferenceCount ` + -Role "Offline initializer manifest validator" + $manifestCurrentDecodeCounts = Get-CurrentDecodeInvalidCountProof ` + -Payload $manifestProof.Payload ` + -CurrentObjectCount $manifestCurrentObjectCount ` + -CurrentReferenceCount $manifestCurrentReferenceCount ` + -Role "Offline initializer manifest validator" + if ( + $manifestRequiredObjectCount -ne $initializerRequiredObjectCount -or + $manifestRequiredReferenceCount -ne $initializerReferenceCount -or + $manifestCurrentObjectCount -ne $initializerRequiredObjectCount -or + $manifestCurrentReferenceCount -ne $initializerReferenceCount -or + $manifestPreservedDecodeCounts.ValidCount -ne + $initializerPreservedDecodeCounts.ValidCount -or + $manifestPreservedDecodeCounts.InvalidCount -ne + $initializerPreservedDecodeCounts.InvalidCount -or + $manifestRequiredDecodeCounts.ObjectCount -ne + $initializerRequiredDecodeCounts.ObjectCount -or + $manifestRequiredDecodeCounts.ReferenceCount -ne + $initializerRequiredDecodeCounts.ReferenceCount -or + $manifestCurrentDecodeCounts.ObjectCount -ne + $initializerRequiredDecodeCounts.ObjectCount -or + $manifestCurrentDecodeCounts.ReferenceCount -ne + $initializerRequiredDecodeCounts.ReferenceCount + ) { + throw "Offline initializer decode proof drifted across preflight, manifest, or current DB" + } + + # LEGACY_BOOTSTRAP_STAGE:new_api_frozen + $failureStage = "new_api_frozen" + Assert-ListenerIdentityUnchanged ` + -Port $WhisperPort ` + -Expected $whisperIdentity ` + -Role "whisper sidecar" + Assert-ListenerIdentityUnchanged ` + -Port $MeloTtsPort ` + -Expected $meloTtsIdentity ` + -Role "MeloTTS sidecar" + Assert-EnvironmentFilePinned ` + -Path $priorEnvironmentFilePath ` + -ExpectedSha256 $priorEnvironmentFileSha256 ` + -Role "prior API" + Assert-EnvironmentFilePinned ` + -Path $stableEnvironmentFilePath ` + -ExpectedSha256 $stableEnvironmentFileSha256 ` + -Role "new release" + Assert-OfflinedRuntimeRaceGate ` + -TaskSnapshot $taskSnapshot ` + -ConfigPath $resolvedCloudflaredConfigPath + $newApiLaunchAttempted = $true + $startOutput = @( + & $startScript ` + -Workspace $resolvedStableSourceRoot ` + -ApiPort $ApiPort ` + -Python $resolvedPythonPath ` + -UserUploadDir $resolvedUploadRoot ` + -UserUploadManifestPath $manifestPath ` + -ExpectedUserUploadManifestSha256 $manifestSha256 ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -ForceApiRestart ` + -SkipEngineRestart ` + -SkipWebRestart ` + -SkipCloudflaredRestart ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -ExpectedPythonSha256 $ExpectedPythonSha256 ` + -OfflineBootstrapQuiescenceReceiptPath $quiescenceReceiptPath ` + -ExpectedOfflineBootstrapQuiescenceReceiptSha256 $quiescenceReceiptSha256 ` + -ExpectedOfflineBootstrapLegacySourceCommit $ExpectedLegacySourceCommit ` + -ExpectedOfflineBootstrapLegacySourceTree $ExpectedLegacySourceTree ` + -RecoveryLockPath $bootstrapLock.Path ` + -InheritedRecoveryLockReceiptPath $bootstrapLock.Path ` + -ExpectedInheritedRecoveryLockReceiptSha256 $bootstrapLock.ReceiptSha256 ` + -CoordinatedTaskNames $CoordinatedTaskNames + ) + $null = $startOutput + $newApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort + $newApiIdentity = Wait-ProcessIdentity ` + -ProcessId $newApiPid ` + -Role "new API" ` + -ExpectedCwd (Join-Path $resolvedStableSourceRoot "apps\api") ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-ApiIdentityContract -Identity $newApiIdentity + if ($newApiIdentity.executable_sha256 -cne $ExpectedPythonSha256) { + throw "New API Python identity drift" + } + Assert-NewApiEnvironmentMatches ` + -Expected $requiredApiEnvironment ` + -ActualIdentity $newApiIdentity + $localUploadProof = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $resolvedPythonPath ` + -ProbePath $uploadProbe ` + -ExpectedUploadRoot $resolvedUploadRoot ` + -ExpectedApiCwd (Join-Path $resolvedStableSourceRoot "apps\api") ` + -ExpectedManifestPath $manifestPath ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedWriteFreezePath $resolvedFreezePath ` + -ExpectedDatabaseTargetSha256 ([string]$manifestProof.Payload.database_target_sha256) ` + -ApiPort $ApiPort + if (-not $localUploadProof.Ok) { + throw "New API upload-root identity validation failed" + } + $null = Wait-ApiHealth ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec $HealthTimeoutSeconds ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedFreezeActive $true ` + -ExpectedFreezeTokenSha256 $freezeTokenSha256 + Wait-GoogleAuthContract ` + -Uri "http://127.0.0.1:$ApiPort/auth/config" ` + -TimeoutSec $HealthTimeoutSeconds + + # LEGACY_BOOTSTRAP_STAGE:new_tunnel_public_frozen + $failureStage = "new_tunnel_public_frozen" + Assert-TunnelAbsent ` + -ConfigPath $resolvedCloudflaredConfigPath ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-FileSha256 ` + -Path $resolvedCloudflaredConfigPath ` + -ExpectedSha256 $ExpectedCloudflaredConfigSha256 ` + -Role "cloudflared config" + $tunnelLogSuffix = [Guid]::NewGuid().ToString("N") + $newTunnelProcess = Start-Process ` + -WindowStyle Hidden ` + -FilePath $resolvedCloudflaredPath ` + -ArgumentList @("tunnel", "--config", $resolvedCloudflaredConfigPath, "run") ` + -WorkingDirectory $resolvedStableSourceRoot ` + -RedirectStandardOutput (Join-Path $resolvedPrivateStateDir "new-tunnel-$tunnelLogSuffix.out.log") ` + -RedirectStandardError (Join-Path $resolvedPrivateStateDir "new-tunnel-$tunnelLogSuffix.err.log") ` + -PassThru + $newTunnelIdentity = Wait-ProcessIdentity ` + -ProcessId $newTunnelProcess.Id ` + -Role "new tunnel" ` + -ExpectedCwd $resolvedStableSourceRoot ` + -TimeoutSec $ProcessStopTimeoutSeconds + if ($newTunnelIdentity.executable_sha256 -cne $ExpectedCloudflaredSha256) { + throw "New tunnel executable identity drift" + } + if (-not (Test-ArgumentPair ` + -Arguments @($newTunnelIdentity.argument_list) ` + -Name "--config" ` + -Value $resolvedCloudflaredConfigPath + )) { + throw "New tunnel config identity drift" + } + $null = Wait-ApiHealth ` + -Uri $PublicHealthUrl ` + -TimeoutSec $HealthTimeoutSeconds ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedFreezeActive $true ` + -ExpectedFreezeTokenSha256 $freezeTokenSha256 + Wait-GoogleAuthContract ` + -Uri $publicAuthConfigUrl ` + -TimeoutSec $HealthTimeoutSeconds + Assert-CurrentTunnelIdentity ` + -ConfigPath $resolvedCloudflaredConfigPath ` + -Expected $newTunnelIdentity + Assert-ListenerIdentityUnchanged ` + -Port $ApiPort ` + -Expected $newApiIdentity ` + -Role "new API" + + # LEGACY_BOOTSTRAP_STAGE:no_rollback_boundary + $failureStage = "no_rollback_boundary" + $noRollback = $true + + # LEGACY_BOOTSTRAP_STAGE:write_release + $failureStage = "write_release" + Remove-OwnedFreezeByHash ` + -Path $resolvedFreezePath ` + -ExpectedTokenSha256 $freezeTokenSha256 + $null = Wait-ApiHealth ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec $HealthTimeoutSeconds ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedFreezeActive $false + $null = Wait-ApiHealth ` + -Uri $PublicHealthUrl ` + -TimeoutSec $HealthTimeoutSeconds ` + -ExpectedManifestSha256 $manifestSha256 ` + -ExpectedFreezeActive $false + Wait-GoogleAuthContract ` + -Uri "http://127.0.0.1:$ApiPort/auth/config" ` + -TimeoutSec $HealthTimeoutSeconds + Wait-GoogleAuthContract ` + -Uri $publicAuthConfigUrl ` + -TimeoutSec $HealthTimeoutSeconds + Assert-CurrentTunnelIdentity ` + -ConfigPath $resolvedCloudflaredConfigPath ` + -Expected $newTunnelIdentity + Assert-ListenerIdentityUnchanged ` + -Port $ApiPort ` + -Expected $newApiIdentity ` + -Role "new API" + Assert-ListenerIdentityUnchanged ` + -Port $WhisperPort ` + -Expected $whisperIdentity ` + -Role "whisper sidecar" + Assert-ListenerIdentityUnchanged ` + -Port $MeloTtsPort ` + -Expected $meloTtsIdentity ` + -Role "MeloTTS sidecar" + + # LEGACY_BOOTSTRAP_STAGE:cutover_receipt_publish + $failureStage = "cutover_receipt_publish" + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $taskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $disabledOriginalTaskDefinitionSnapshot + $cutoverReceipt = [ordered]@{ + schema_version = "vignette.legacy-public-upload-bootstrap.v1" + status = "passed" + operational_success = $false + captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + source = [ordered]@{ + git_commit = $ExpectedSourceCommit + git_tree = $ExpectedSourceTree + } + storage = [ordered]@{ + migration_manifest_sha256 = $manifestSha256 + quiescence_receipt_sha256 = $quiescenceReceiptSha256 + database_target_sha256 = [string]$manifestProof.Payload.database_target_sha256 + preserved_object_count = [int]$manifestProof.Payload.preserved_object_count + preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes + preserved_object_set_sha256 = [string]$manifestProof.Payload.preserved_object_set_sha256 + preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount + preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount + required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount + required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount + reference_count = [int]$manifestProof.Payload.current_reference_count + unique_object_count = [int]$manifestProof.Payload.current_object_count + reference_set_sha256 = [string]$manifestProof.Payload.current_reference_set_sha256 + current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount + current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount + write_freeze_released = $true + required_environment_sha256 = $requiredEnvironmentDigest + } + processes = [ordered]@{ + api = ConvertTo-SafeProcessIdentity -Identity $newApiIdentity + tunnel = ConvertTo-SafeProcessIdentity -Identity $newTunnelIdentity + } + validation = [ordered]@{ + local_frozen = $true + public_frozen = $true + local_unfrozen = $true + public_unfrozen = $true + no_rollback_boundary_crossed = $true + } + task_maintenance = [ordered]@{ + disabled = $true + idle = $true + restored = $false + task_count = @($taskSnapshot).Count + original_definition_set_sha256 = [string]$originalTaskDefinitionSnapshot.set_sha256 + disabled_pre_cutover_definition_set_sha256 = [string]$disabledOriginalTaskDefinitionSnapshot.set_sha256 + } + privacy = [ordered]@{ + raw_paths_recorded = $false + raw_urls_recorded = $false + raw_user_ids_recorded = $false + raw_filenames_recorded = $false + raw_secrets_recorded = $false + } + } + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $disabledOriginalTaskDefinitionSnapshot + $cutoverReceiptSha256 = Write-PrivacySafeReceiptCreateOnly ` + -Path $resolvedCutoverReceiptPath ` + -Payload $cutoverReceipt + + # LEGACY_BOOTSTRAP_STAGE:task_maintenance_exit + $failureStage = "task_definition_cutover" + Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot + Assert-FileSha256 ` + -Path $resolvedPythonPath ` + -ExpectedSha256 $ExpectedPythonSha256 ` + -Role "Python before task definition cutover" + Assert-FileSha256 ` + -Path $resolvedCloudflaredPath ` + -ExpectedSha256 $ExpectedCloudflaredSha256 ` + -Role "cloudflared before task definition cutover" + Assert-FileSha256 ` + -Path $resolvedCloudflaredConfigPath ` + -ExpectedSha256 $ExpectedCloudflaredConfigSha256 ` + -Role "cloudflared config before task definition cutover" + $bootInstallAction = { + & $bootTaskInstaller ` + -StableSourceRoot $resolvedStableSourceRoot ` + -Python $resolvedPythonPath ` + -Cloudflared $resolvedCloudflaredPath ` + -CloudflaredConfig $resolvedCloudflaredConfigPath ` + -UserUploadDir $resolvedUploadRoot ` + -UserUploadManifestPath $manifestPath ` + -ExpectedUserUploadManifestSha256 $manifestSha256 ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -TaskName "VignettePublicRuntime" ` + -InitiallyDisabled + }.GetNewClosure() + $watchdogInstallAction = { + & $watchdogTaskInstaller ` + -StableSourceRoot $resolvedStableSourceRoot ` + -TaskName "VignettePublicRuntimeWatchdog" ` + -Python $resolvedPythonPath ` + -UserUploadDir $resolvedUploadRoot ` + -UserUploadManifestPath $manifestPath ` + -ExpectedUserUploadManifestSha256 $manifestSha256 ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -Cloudflared $resolvedCloudflaredPath ` + -CloudflaredConfig $resolvedCloudflaredConfigPath ` + -PublicHealthUrl $PublicHealthUrl ` + -InitiallyDisabled + }.GetNewClosure() + Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled ` + -BootInstaller $bootInstallAction ` + -WatchdogInstaller $watchdogInstallAction ` + -MaintenanceSnapshot $taskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $taskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + $newDisabledTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot + $null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $newDisabledTaskDefinitionSnapshot ` + -StableSourceRoot $resolvedStableSourceRoot ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -PythonPath $resolvedPythonPath ` + -UserUploadDir $resolvedUploadRoot ` + -UserUploadManifestPath $manifestPath ` + -ExpectedUserUploadManifestSha256 $manifestSha256 ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -CloudflaredPath $resolvedCloudflaredPath ` + -CloudflaredConfigPath $resolvedCloudflaredConfigPath ` + -PublicHealthUrl $PublicHealthUrl + Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot + Assert-FileSha256 ` + -Path $resolvedPythonPath ` + -ExpectedSha256 $ExpectedPythonSha256 ` + -Role "Python after task definition cutover" + Assert-FileSha256 ` + -Path $resolvedCloudflaredPath ` + -ExpectedSha256 $ExpectedCloudflaredSha256 ` + -Role "cloudflared after task definition cutover" + Assert-FileSha256 ` + -Path $resolvedCloudflaredConfigPath ` + -ExpectedSha256 $ExpectedCloudflaredConfigSha256 ` + -Role "cloudflared config after task definition cutover" + + $failureStage = "task_maintenance_exit" + $operationalTaskDefinitionSnapshot = Enable-NewPublicRuntimeTaskDefinitions ` + -DisabledSnapshot $newDisabledTaskDefinitionSnapshot + $null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $operationalTaskDefinitionSnapshot ` + -StableSourceRoot $resolvedStableSourceRoot ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -PythonPath $resolvedPythonPath ` + -UserUploadDir $resolvedUploadRoot ` + -UserUploadManifestPath $manifestPath ` + -ExpectedUserUploadManifestSha256 $manifestSha256 ` + -UserUploadWriteFreezePath $resolvedFreezePath ` + -CloudflaredPath $resolvedCloudflaredPath ` + -CloudflaredConfigPath $resolvedCloudflaredConfigPath ` + -PublicHealthUrl $PublicHealthUrl ` + -AllowEnabled + $taskRestoreTruth = Get-PublicRuntimeTaskTruth ` + -Snapshot $taskSnapshot ` + -ExpectedRestored + if (-not [bool]$taskRestoreTruth.expected_state_restored) { + throw "Task restoration truth proof failed" + } + + # LEGACY_BOOTSTRAP_STAGE:task_recovery_receipt_publish + $failureStage = "task_recovery_receipt_publish" + $taskRecoveryReceipt = [ordered]@{ + schema_version = "vignette.legacy-public-upload-task-recovery.v1" + status = "passed" + operational_success = $true + captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + cutover_receipt_sha256 = $cutoverReceiptSha256 + preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes + preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount + preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount + required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount + required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount + current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount + current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount + tasks_restored = $true + task_count = @($taskSnapshot).Count + task_enabled_count = [int]$taskRestoreTruth.enabled_count + task_running_count = [int]$taskRestoreTruth.running_count + task_definitions = [ordered]@{ + original_set_sha256 = [string]$originalTaskDefinitionSnapshot.set_sha256 + installed_disabled_set_sha256 = [string]$newDisabledTaskDefinitionSnapshot.set_sha256 + operational_set_sha256 = [string]$operationalTaskDefinitionSnapshot.set_sha256 + } + privacy = [ordered]@{ + raw_paths_recorded = $false + raw_urls_recorded = $false + raw_user_ids_recorded = $false + raw_filenames_recorded = $false + raw_secrets_recorded = $false + } + } + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $operationalTaskDefinitionSnapshot + $taskRecoveryReceiptSha256 = Write-PrivacySafeReceiptCreateOnly ` + -Path $resolvedTaskRecoveryReceiptPath ` + -Payload $taskRecoveryReceipt + $taskMaintenanceEntered = $false +} catch { + $rollbackSucceeded = $false + $tasksRestored = $false + if ( + $freezeTokenSha256 -notmatch "^[0-9a-f]{64}$" -and + $freezeAbsentBeforeInitializer -and + [System.IO.File]::Exists($resolvedFreezePath) + ) { + try { + $freezeTokenSha256 = Get-OwnedFreezeTokenSha256 -Path $resolvedFreezePath + } catch { + $freezeTokenSha256 = "" + } + } + if ($newApiLaunchAttempted -and $null -eq $newApiIdentity) { + try { + $candidateApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort + $candidateApi = Wait-ProcessIdentity ` + -ProcessId $candidateApiPid ` + -Role "owned new API recovery" ` + -ExpectedCwd (Join-Path $resolvedStableSourceRoot "apps\api") ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-ApiIdentityContract -Identity $candidateApi + if ($candidateApi.executable_sha256 -cne $ExpectedPythonSha256) { + throw "Owned new API recovery identity drift" + } + $newApiIdentity = $candidateApi + } catch { + $newApiIdentity = $null + } + } + if ($null -eq $newTunnelIdentity -and $null -ne $newTunnelProcess) { + try { + $candidateTunnel = Wait-ProcessIdentity ` + -ProcessId $newTunnelProcess.Id ` + -Role "owned new tunnel recovery" ` + -ExpectedCwd $resolvedStableSourceRoot ` + -TimeoutSec $ProcessStopTimeoutSeconds + if ( + $candidateTunnel.executable_sha256 -cne $ExpectedCloudflaredSha256 -or + -not (Test-ArgumentPair ` + -Arguments @($candidateTunnel.argument_list) ` + -Name "--config" ` + -Value $resolvedCloudflaredConfigPath) + ) { + throw "Owned new tunnel recovery identity drift" + } + $newTunnelIdentity = $candidateTunnel + } catch { + $newTunnelIdentity = $null + } + } + if (-not $noRollback) { + try { + if ($runtimeMutationStarted) { + Restore-LegacyRuntime ` + -PriorApi $priorApiIdentity ` + -PriorTunnel $priorTunnelIdentity ` + -ConfigPath $resolvedCloudflaredConfigPath ` + -FreezePath $resolvedFreezePath ` + -FreezeTokenSha256 $freezeTokenSha256 ` + -PrivateStateRoot $resolvedPrivateStateDir ` + -PriorEnvironmentPath $priorEnvironmentFilePath ` + -PriorEnvironmentSha256 $priorEnvironmentFileSha256 ` + -OwnedNewApi $newApiIdentity ` + -OwnedNewTunnel $newTunnelIdentity + } elseif ([System.IO.File]::Exists($resolvedFreezePath)) { + if ($freezeTokenSha256 -notmatch "^[0-9a-f]{64}$") { + throw "Pre-mutation freeze ownership is unknown" + } + Remove-OwnedFreezeByHash ` + -Path $resolvedFreezePath ` + -ExpectedTokenSha256 $freezeTokenSha256 + } + if ($taskMaintenanceEntered) { + Exit-PublicRuntimeTaskMaintenance -Snapshot $taskSnapshot + $restoredTruth = Get-PublicRuntimeTaskTruth ` + -Snapshot $taskSnapshot ` + -ExpectedRestored + $tasksRestored = [bool]$restoredTruth.expected_state_restored + if (-not $tasksRestored) { + throw "Task restoration truth proof failed" + } + $taskMaintenanceEntered = $false + } + $rollbackSucceeded = $true + } catch { + $rollbackSucceeded = $false + } + } + if ($taskMaintenanceEntered) { + try { + Suspend-PublicRuntimeTasks ` + -Snapshot $taskSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + } catch { + } + } + $taskTruth = [ordered]@{ + enabled_count = 0 + running_count = 0 + all_disabled_and_idle = $false + expected_state_restored = $false + } + if (@($taskSnapshot).Count -gt 0) { + try { + $taskTruth = Get-PublicRuntimeTaskTruth -Snapshot $taskSnapshot + } catch { + } + } + try { + $failureReceipt = [ordered]@{ + schema_version = "vignette.legacy-public-upload-bootstrap-failure.v1" + status = "failed" + captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + failure_stage = $failureStage + no_rollback_boundary_crossed = $noRollback + rollback_attempted = $runtimeMutationStarted -and -not $noRollback + rollback_succeeded = $rollbackSucceeded + tasks_restored = $tasksRestored + tasks_remain_disabled = [bool]$taskTruth.all_disabled_and_idle + task_enabled_count = [int]$taskTruth.enabled_count + task_running_count = [int]$taskTruth.running_count + cutover_receipt_published = ($cutoverReceiptSha256 -match "^[0-9a-f]{64}$") + cutover_receipt_sha256 = $cutoverReceiptSha256 + privacy = [ordered]@{ + raw_paths_recorded = $false + raw_urls_recorded = $false + raw_user_ids_recorded = $false + raw_filenames_recorded = $false + raw_secrets_recorded = $false + } + } + $failureJson = ConvertTo-Json -InputObject $failureReceipt -Depth 6 -Compress + $failureDigest = Get-Utf8Sha256 -Value ($failureJson + [Environment]::NewLine) + $failureReceiptPath = Join-Path $resolvedPrivateStateDir "legacy-bootstrap-failure-$failureDigest.json" + $null = Write-PrivacySafeReceiptCreateOnly ` + -Path $failureReceiptPath ` + -Payload $failureReceipt + } catch { + } + if ($noRollback) { + throw "Legacy public upload bootstrap failed after the no-rollback boundary; tasks remain disabled" + } + if ($rollbackSucceeded) { + throw "Legacy public upload bootstrap failed before the boundary; prior runtime and tasks were restored" + } + throw "Legacy public upload bootstrap failed closed before the boundary; tasks remain disabled" +} finally { + if ($null -ne $callerRequiredEnvironmentSnapshot) { + Restore-SelectedProcessEnvironment ` + -Snapshot $callerRequiredEnvironmentSnapshot + } + if ($null -ne $bootstrapLock -and $null -ne $bootstrapLock.Stream) { + $bootstrapLock.Stream.Dispose() + } +} + +$result = [ordered]@{ + status = "passed" + cutover_receipt_sha256 = $cutoverReceiptSha256 + task_recovery_receipt_sha256 = $taskRecoveryReceiptSha256 + preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes + preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount + preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount + required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount + required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount + current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount + current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount +} +Write-Output (ConvertTo-Json -InputObject $result -Compress) diff --git a/scripts/initialize-public-runtime-upload-root.ps1 b/scripts/initialize-public-runtime-upload-root.ps1 new file mode 100644 index 0000000..b807dab --- /dev/null +++ b/scripts/initialize-public-runtime-upload-root.ps1 @@ -0,0 +1,395 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$StableSourceRoot, + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$ManifestStateDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, + [Parameter(Mandatory = $true)] + [ValidateRange(0, 2147483647)] + [int]$ExpectedReferenceCount, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedPreservedObjectCount, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 9223372036854775807)] + [long]$ExpectedPreservedTotalSizeBytes, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedPreservedInventorySha256, + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string[]]$SourceUploadDir, + [string]$OfflineQuiescenceCaptureBase64 = "", + [ValidatePattern("^$|^[0-9a-f]{40}$")] + [string]$ExpectedOfflineSourceCommit = "", + [ValidatePattern("^$|^[0-9a-f]{40}$")] + [string]$ExpectedOfflineSourceTree = "", + [string]$PythonPath = "", + [string]$HealthUrl = "http://127.0.0.1:8001/health", + [ValidateRange(1, 300)] + [int]$FreezeTimeoutSeconds = 30 +) + +$ErrorActionPreference = "Stop" +[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false) +$OutputEncoding = [System.Text.UTF8Encoding]::new($false) + +function Assert-StableInitializerSourceProvenance { + param( + [Parameter(Mandatory = $true)] + [string]$Root + ) + + $gitTopLevelOutput = @(& git.exe -C $Root rev-parse --show-toplevel) + if ($LASTEXITCODE -ne 0 -or $gitTopLevelOutput.Count -eq 0) { + throw "Upload initializer source is not a Git worktree" + } + $gitTopLevel = (Resolve-Path -LiteralPath ((@($gitTopLevelOutput) -join "").Trim())).Path + if (-not [string]::Equals( + $gitTopLevel, + $Root, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Upload initializer source must be its Git toplevel" + } + + $symbolicRefOutput = @(& git.exe -C $Root symbolic-ref -q HEAD) + $symbolicRefExit = $LASTEXITCODE + if ($symbolicRefExit -eq 0 -or $symbolicRefOutput.Count -gt 0) { + throw "Upload initializer source must use a detached HEAD" + } + if ($symbolicRefExit -ne 1) { + throw "Upload initializer could not prove detached HEAD" + } + + $statusOutput = @(& git.exe -C $Root status --porcelain=v1 --untracked-files=all) + if ($LASTEXITCODE -ne 0) { + throw "Upload initializer could not prove source cleanliness" + } + if ($statusOutput.Count -gt 0) { + throw "Upload initializer source must be completely clean" + } + + foreach ($relativePath in @( + "scripts/initialize-public-runtime-upload-root.ps1", + "scripts/initialize-public-runtime-upload-root.py", + "scripts/public_runtime_database_identity.py", + "scripts/public-runtime-upload-root.ps1", + "apps/api/app/config.py", + "apps/api/app/upload_storage.py" + )) { + $trackedOutput = @(& git.exe -C $Root ls-files --error-unmatch -- $relativePath) + if ($LASTEXITCODE -ne 0 -or $trackedOutput.Count -ne 1) { + throw "Upload initializer source chain must be tracked" + } + } +} + +function Resolve-PrivateStateDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [Parameter(Mandatory = $true)] + [string]$PublicRoot, + [Parameter(Mandatory = $true)] + [string]$GitRoot + ) + + $driveAbsolute = $Path -match '^[A-Za-z]:[\\/]' + $uncAbsolute = $Path -match '^\\\\[^\\/]+[\\/][^\\/]+(?:[\\/]|$)' + if (-not $driveAbsolute -and -not $uncAbsolute) { + throw "Private upload state directory must be absolute" + } + $fullPath = Get-PublicRuntimeCanonicalPath -Path $Path + $filesystemRoot = [System.IO.Path]::GetPathRoot($fullPath) + if ([string]::Equals( + $fullPath.TrimEnd('\', '/'), + $filesystemRoot.TrimEnd('\', '/'), + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Private upload state directory cannot be a filesystem root" + } + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $fullPath -Parent $PublicRoot) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $PublicRoot -Parent $fullPath) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $fullPath -Parent $GitRoot) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $GitRoot -Parent $fullPath) + ) { + throw "Private upload state must be disjoint from public and source roots" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $fullPath + if (-not (Test-Path -LiteralPath $fullPath)) { + [System.IO.Directory]::CreateDirectory($fullPath) | Out-Null + } + if (-not (Test-Path -LiteralPath $fullPath -PathType Container)) { + throw "Private upload state directory is unavailable" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $fullPath + return (Resolve-Path -LiteralPath $fullPath).Path +} + +function Remove-OwnedWriteFreeze { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [Parameter(Mandatory = $true)] + [string]$OwnedToken + ) + + if (-not [System.IO.File]::Exists($Path)) { + return $true + } + try { + $payload = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json + if ( + $null -eq $payload -or + $payload.schema_version -ne "vignette.public-upload-write-freeze.v1" -or + $payload.token -ne $OwnedToken + ) { + return $false + } + [System.IO.File]::Delete($Path) + return -not [System.IO.File]::Exists($Path) + } catch { + return $false + } +} + +function Assert-OnlineUploadWritesRecovered { + param( + [Parameter(Mandatory = $true)] + [string]$Uri, + [ValidateRange(1, 300)] + [int]$TimeoutSeconds = 30 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + do { + $health = $null + try { + $health = Invoke-RestMethod ` + -Uri $Uri ` + -Method Get ` + -TimeoutSec 5 ` + -UseBasicParsing + } catch { + $health = $null + } + $freeze = $null + if ($null -ne $health) { + $freeze = $health.upload_write_freeze + } + if ( + $null -ne $freeze -and + $freeze.capable -eq $true -and + $freeze.active -eq $false -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 + ) { + return + } + Start-Sleep -Milliseconds 250 + } while ((Get-Date) -lt $deadline) + + throw "Upload initialization cleanup could not prove write availability" +} + +$resolvedStableSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path +$uploadRootContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-upload-root.ps1" +$initializerWorker = Join-Path $resolvedStableSourceRoot "scripts\initialize-public-runtime-upload-root.py" +$databaseIdentityHelper = Join-Path $resolvedStableSourceRoot "scripts\public_runtime_database_identity.py" +foreach ($requiredFile in @($uploadRootContract, $initializerWorker, $databaseIdentityHelper)) { + if (-not (Test-Path -LiteralPath $requiredFile -PathType Leaf)) { + throw "Upload initializer prerequisite is unavailable" + } +} +Assert-StableInitializerSourceProvenance -Root $resolvedStableSourceRoot +. $uploadRootContract + +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedStableSourceRoot ` + -UploadRoot $UserUploadDir ` + -CreateIfMissing ` + -ProbeWritable +$resolvedManifestStateDir = Resolve-PrivateStateDirectory ` + -Path $ManifestStateDir ` + -PublicRoot $resolvedUserUploadDir ` + -GitRoot $resolvedStableSourceRoot + +$freezeDriveAbsolute = $UserUploadWriteFreezePath -match '^[A-Za-z]:[\\/]' +$freezeUncAbsolute = $UserUploadWriteFreezePath -match '^\\\\[^\\/]+[\\/][^\\/]+(?:[\\/]|$)' +if (-not $freezeDriveAbsolute -and -not $freezeUncAbsolute) { + throw "Upload write-freeze path must be absolute" +} +$fullFreezePath = Get-PublicRuntimeCanonicalPath -Path $UserUploadWriteFreezePath +$freezeParent = [System.IO.Path]::GetDirectoryName($fullFreezePath) +if (-not [string]::Equals( + $freezeParent.TrimEnd('\', '/'), + $resolvedManifestStateDir.TrimEnd('\', '/'), + [System.StringComparison]::OrdinalIgnoreCase +)) { + throw "Upload write-freeze must be a direct child of private state" +} +Assert-PublicRuntimePathHasNoReparsePoint -Path $fullFreezePath + +if ([string]::IsNullOrWhiteSpace($PythonPath)) { + $PythonPath = Join-Path $resolvedStableSourceRoot "apps\api\.venv\Scripts\python.exe" +} +if (-not (Test-Path -LiteralPath $PythonPath -PathType Leaf)) { + throw "Upload initializer Python runtime is unavailable" +} +$resolvedPythonPath = (Resolve-Path -LiteralPath $PythonPath).Path + +$resolvedSources = @() +foreach ($sourceRoot in $SourceUploadDir) { + $sourceDriveAbsolute = $sourceRoot -match '^[A-Za-z]:[\\/]' + $sourceUncAbsolute = $sourceRoot -match '^\\\\[^\\/]+[\\/][^\\/]+(?:[\\/]|$)' + if (-not $sourceDriveAbsolute -and -not $sourceUncAbsolute) { + throw "Source upload directory must be absolute" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $sourceRoot + if (-not (Test-Path -LiteralPath $sourceRoot -PathType Container)) { + throw "Source upload directory is unavailable" + } + $resolvedSource = (Resolve-Path -LiteralPath $sourceRoot).Path + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedSource -Parent $resolvedUserUploadDir) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedUserUploadDir -Parent $resolvedSource) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedSource -Parent $resolvedManifestStateDir) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedManifestStateDir -Parent $resolvedSource) + ) { + throw "Source upload directories must be disjoint from target and private state" + } + $resolvedSources += $resolvedSource +} + +$freezeToken = ( + [Guid]::NewGuid().ToString("N") + + [Guid]::NewGuid().ToString("N") +) +$freezePayload = @{ + schema_version = "vignette.public-upload-write-freeze.v1" + token = $freezeToken +} | ConvertTo-Json -Compress +$freezeBytes = [System.Text.UTF8Encoding]::new($false).GetBytes($freezePayload) +$freezeStream = $null +$freezeOwned = $false +$freezePublished = $false +$offlineQuiescenceMode = -not [string]::IsNullOrWhiteSpace( + $OfflineQuiescenceCaptureBase64 +) +$offlineSourcePinsPresent = ( + -not [string]::IsNullOrWhiteSpace($ExpectedOfflineSourceCommit) -and + -not [string]::IsNullOrWhiteSpace($ExpectedOfflineSourceTree) +) +if ($offlineQuiescenceMode -ne $offlineSourcePinsPresent) { + throw "Offline quiescence capture and lowercase source commit/tree pins are required together" +} +try { + try { + $freezeStream = [System.IO.File]::Open( + $fullFreezePath, + [System.IO.FileMode]::CreateNew, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None + ) + $freezeOwned = $true + $freezeStream.Write($freezeBytes, 0, $freezeBytes.Length) + $freezeStream.Flush($true) + $freezePublished = $true + } finally { + if ($null -ne $freezeStream) { + $freezeStream.Dispose() + } + } + $workerArgs = @( + "-X", "utf8", "-B", $initializerWorker, + "--upload-root", $resolvedUserUploadDir, + "--manifest-state-dir", $resolvedManifestStateDir, + "--write-freeze-path", $fullFreezePath, + "--expected-reference-count", $ExpectedReferenceCount.ToString(), + "--expected-preserved-object-count", $ExpectedPreservedObjectCount.ToString(), + "--expected-preserved-total-size-bytes", $ExpectedPreservedTotalSizeBytes.ToString(), + "--expected-preserved-inventory-sha256", $ExpectedPreservedInventorySha256, + "--health-url", $HealthUrl, + "--freeze-timeout-seconds", $FreezeTimeoutSeconds.ToString() + ) + foreach ($resolvedSource in $resolvedSources) { + $workerArgs += @("--source-root", $resolvedSource) + } + if ($offlineQuiescenceMode) { + if ($OfflineQuiescenceCaptureBase64 -notmatch '^[A-Za-z0-9+/]+={0,2}$') { + throw "Offline quiescence capture is not canonical base64" + } + $workerArgs += @( + "--offline-quiescence-capture-base64", + $OfflineQuiescenceCaptureBase64, + "--expected-offline-source-commit", + $ExpectedOfflineSourceCommit, + "--expected-offline-source-tree", + $ExpectedOfflineSourceTree + ) + } + + Push-Location (Join-Path $resolvedStableSourceRoot "apps\api") + try { + $workerOutput = @(& $resolvedPythonPath @workerArgs) + $workerExitCode = $LASTEXITCODE + } finally { + Pop-Location + } + $serializedOutput = (@($workerOutput) -join "").Trim() + $parsedOutput = $null + if (-not [string]::IsNullOrWhiteSpace($serializedOutput)) { + try { + $parsedOutput = $serializedOutput | ConvertFrom-Json + } catch { + $parsedOutput = $null + } + } + if ( + $workerExitCode -ne 0 -or + $null -eq $parsedOutput -or + $parsedOutput.status -ne "initialized" -or + [long]$parsedOutput.preserved_total_size_bytes -ne + $ExpectedPreservedTotalSizeBytes + ) { + throw "Public avatar upload initialization failed" + } + Write-Output $serializedOutput +} catch { + if ($freezeOwned) { + if ($freezePublished) { + $freezeRemoved = Remove-OwnedWriteFreeze ` + -Path $fullFreezePath ` + -OwnedToken $freezeToken + } else { + $freezeRemoved = $false + } + if (-not $freezePublished -and [System.IO.File]::Exists($fullFreezePath)) { + # CreateNew 뒤 sentinel payload를 완성하기 전에 실패했다면 이 경로는 아직 + # 다른 프로세스에 공개되지 않은 이 호출 소유 파일이다. + try { + [System.IO.File]::Delete($fullFreezePath) + $freezeRemoved = -not [System.IO.File]::Exists($fullFreezePath) + } catch { + $freezeRemoved = $false + } + } elseif (-not $freezePublished) { + $freezeRemoved = $true + } + if (-not $freezeRemoved) { + throw "Upload initialization failed and owned write freeze could not be removed" + } + if (-not $offlineQuiescenceMode) { + Assert-OnlineUploadWritesRecovered ` + -Uri $HealthUrl ` + -TimeoutSeconds $FreezeTimeoutSeconds + } + } + throw +} diff --git a/scripts/initialize-public-runtime-upload-root.py b/scripts/initialize-public-runtime-upload-root.py new file mode 100644 index 0000000..1af3a86 --- /dev/null +++ b/scripts/initialize-public-runtime-upload-root.py @@ -0,0 +1,1664 @@ +#!/usr/bin/env python3 +"""공개 프로필 아바타 저장소를 명시적으로 초기화한다. + +이 작업자만 공개 업로드 루트에 디렉터리와 파일을 만들 수 있다. 원본은 읽기만 +하며 대상은 ``xb`` 모드로만 생성한다. 출력과 매니페스트에는 사용자 식별자, +원본 URL, 파일명 또는 원시 경로를 기록하지 않는다. +""" + +import argparse +import asyncio +import base64 +import hashlib +import json +import os +import re +import stat +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Mapping, Sequence + + +REPO_ROOT = Path(__file__).resolve().parents[1] +API_ROOT = REPO_ROOT / "apps" / "api" +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from app.upload_storage import ( # noqa: E402 + WRITE_FREEZE_SCHEMA_VERSION, + build_privacy_safe_manifest, + canonical_path_sha256, + inspect_public_avatar_image, + public_avatar_relative_path_sha256, + public_avatar_url_to_relative_path, + sha256_file, +) +from public_runtime_database_identity import ( # noqa: E402 + connected_database_target_sha256, +) + + +_ALLOWED_AVATAR_NAME = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.(?:png|jpg|jpeg|webp)$" +) +_SHA256_RE = re.compile(r"^[a-f0-9]{64}$") +_GIT_OBJECT_RE = re.compile(r"^[a-f0-9]{40}$") +_UTC_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$") +_COPY_CHUNK_SIZE = 1024 * 1024 +OFFLINE_CAPTURE_SCHEMA_VERSION = ( + "vignette.public-upload-offline-quiescence-capture.v2" +) +OFFLINE_RECEIPT_SCHEMA_VERSION = ( + "vignette.public-upload-offline-quiescence.v3" +) + + +class InitializationError(RuntimeError): + """개인정보를 포함하지 않는 안정적인 초기화 실패 코드.""" + + +class _RejectRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + raise InitializationError("runtime_health_redirect_rejected") + + +@dataclass(frozen=True, slots=True) +class ReferenceInventory: + objects: tuple[tuple[str, int], ...] + reference_count: int + reference_set_sha256: str + + @property + def unique_object_count(self) -> int: + return len(self.objects) + + +@dataclass(frozen=True, slots=True) +class PreservedObject: + relative_path: str + source_path: Path + size_bytes: int + content_sha256: str + decode_valid: bool + + +@dataclass(frozen=True, slots=True) +class PreservedInventory: + objects: tuple[PreservedObject, ...] + inventory_sha256: str + + @property + def object_count(self) -> int: + return len(self.objects) + + @property + def total_size_bytes(self) -> int: + return sum(item.size_bytes for item in self.objects) + + @property + def decode_valid_count(self) -> int: + return sum(1 for item in self.objects if item.decode_valid) + + @property + def decode_invalid_count(self) -> int: + return self.object_count - self.decode_valid_count + + +@dataclass(frozen=True, slots=True) +class CopyProof: + object_count: int + copied_count: int + reused_exact_count: int + + +@dataclass(frozen=True, slots=True) +class OfflineQuiescenceProof: + receipt_path: Path + receipt_sha256: str + database_target_sha256: str + source_root_sha256s: tuple[str, ...] + source_root_set_sha256: str + preserved_object_count: int + preserved_total_size_bytes: int + preserved_inventory_sha256: str + preserved_decode_valid_count: int + preserved_decode_invalid_count: int + required_decode_invalid_object_count: int + required_decode_invalid_reference_count: int + reference_count: int + unique_object_count: int + reference_set_sha256: str + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json_bytes(value: object) -> bytes: + return ( + json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + "\n" + ).encode("utf-8") + + +def _has_reparse_attribute(path: Path) -> bool: + metadata = path.lstat() + attributes = getattr(metadata, "st_file_attributes", 0) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return path.is_symlink() or bool(attributes & reparse_flag) + + +def _assert_no_reparse_ancestors(path: Path) -> None: + cursor = path + while True: + if cursor.exists() or cursor.is_symlink(): + if _has_reparse_attribute(cursor): + raise InitializationError("path_reparse_point_rejected") + parent = cursor.parent + if parent == cursor: + return + cursor = parent + + +def _validated_absolute_directory(path: Path, *, must_exist: bool = True) -> Path: + if not path.is_absolute(): + raise InitializationError("directory_must_be_absolute") + lexical = Path(os.path.abspath(path)) + _assert_no_reparse_ancestors(lexical) + if must_exist and (not lexical.exists() or not lexical.is_dir()): + raise InitializationError("required_directory_unavailable") + resolved = lexical.resolve(strict=must_exist) + if resolved.parent == resolved: + raise InitializationError("filesystem_root_rejected") + _assert_no_reparse_ancestors(resolved) + return resolved + + +def _validated_regular_file(path: Path) -> Path: + if not path.is_absolute(): + raise InitializationError("file_must_be_absolute") + lexical = Path(os.path.abspath(path)) + _assert_no_reparse_ancestors(lexical) + try: + metadata = lexical.lstat() + except OSError as exc: + raise InitializationError("required_regular_file_unavailable") from exc + if _has_reparse_attribute(lexical) or not stat.S_ISREG(metadata.st_mode): + raise InitializationError("required_regular_file_unavailable") + return lexical.resolve(strict=True) + + +def _same_or_within(candidate: Path, parent: Path) -> bool: + candidate_key = os.path.normcase(str(candidate)) + parent_key = os.path.normcase(str(parent)) + try: + return os.path.commonpath((candidate_key, parent_key)) == parent_key + except ValueError: + return False + + +def _assert_disjoint(left: Path, right: Path) -> None: + if _same_or_within(left, right) or _same_or_within(right, left): + raise InitializationError("private_and_public_roots_must_be_disjoint") + + +def normalize_avatar_reference(value: object) -> str: + if not isinstance(value, str): + raise InitializationError("database_avatar_reference_invalid") + exact_prefix = "/uploads/profile-avatars/" + if not value.startswith(exact_prefix): + raise InitializationError("database_avatar_reference_invalid") + basename = value[len(exact_prefix) :] + if not basename or "/" in basename or "\\" in basename: + raise InitializationError("database_avatar_reference_invalid") + try: + relative = public_avatar_url_to_relative_path(value) + except ValueError as exc: + raise InitializationError("database_avatar_reference_invalid") from exc + if relative != f"profile-avatars/{basename}": + raise InitializationError("database_avatar_reference_invalid") + if ( + "%" in basename + or any(ord(character) < 32 or ord(character) == 127 for character in basename) + or _ALLOWED_AVATAR_NAME.fullmatch(basename) is None + ): + raise InitializationError("database_avatar_reference_invalid") + return relative + + +def build_reference_inventory( + avatar_urls: Iterable[object], *, expected_reference_count: int +) -> ReferenceInventory: + if expected_reference_count < 0: + raise InitializationError("expected_reference_count_invalid") + counts: Counter[str] = Counter() + canonical_names: dict[str, str] = {} + for raw_value in avatar_urls: + normalized = normalize_avatar_reference(raw_value) + identity = normalized.casefold() + canonical_names.setdefault(identity, normalized) + counts[identity] += 1 + reference_count = sum(counts.values()) + if reference_count != expected_reference_count: + raise InitializationError("database_reference_count_mismatch") + objects = tuple( + sorted( + ((canonical_names[key], count) for key, count in counts.items()), + key=lambda item: public_avatar_relative_path_sha256(item[0]), + ) + ) + hashed_counts = [ + { + "path_sha256": public_avatar_relative_path_sha256(name), + "reference_count": count, + } + for name, count in objects + ] + digest = _sha256_bytes(_json_bytes(hashed_counts).rstrip(b"\n")) + return ReferenceInventory(objects, reference_count, digest) + + +def _strict_lower_sha256(value: object, error_code: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise InitializationError(error_code) + return value + + +def _strict_process_identity(value: object) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != { + "pid", + "started_at_utc", + "executable_sha256", + "command_line_sha256", + "cwd_sha256", + }: + raise InitializationError("offline_process_identity_invalid") + pid = value.get("pid") + started_at = value.get("started_at_utc") + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + raise InitializationError("offline_process_identity_invalid") + if not isinstance(started_at, str) or _UTC_TIMESTAMP_RE.fullmatch(started_at) is None: + raise InitializationError("offline_process_identity_invalid") + return { + "pid": pid, + "started_at_utc": started_at, + "executable_sha256": _strict_lower_sha256( + value.get("executable_sha256"), "offline_process_identity_invalid" + ), + "command_line_sha256": _strict_lower_sha256( + value.get("command_line_sha256"), "offline_process_identity_invalid" + ), + "cwd_sha256": _strict_lower_sha256( + value.get("cwd_sha256"), "offline_process_identity_invalid" + ), + } + + +def _source_root_bindings( + source_roots: Sequence[Path], +) -> tuple[tuple[str, ...], str]: + hashes: list[str] = [] + seen_paths: set[str] = set() + for source_root in source_roots: + validated = _validated_absolute_directory(source_root) + key = os.path.normcase(str(validated)) + if key in seen_paths: + raise InitializationError("source_root_duplicate") + seen_paths.add(key) + hashes.append(canonical_path_sha256(validated)) + if not hashes: + raise InitializationError("source_root_required") + ordered_hashes = tuple(hashes) + set_sha256 = _sha256_bytes( + _json_bytes(sorted(ordered_hashes)).rstrip(b"\n") + ) + return ordered_hashes, set_sha256 + + +def decode_offline_quiescence_capture( + encoded_capture: str, + *, + legacy_source_roots: Sequence[Path], + expected_source_commit: str, + expected_source_tree: str, +) -> dict[str, object]: + if not isinstance(encoded_capture, str) or not encoded_capture: + raise InitializationError("offline_quiescence_capture_missing") + try: + raw = base64.b64decode(encoded_capture, validate=True) + if len(raw) > 16 * 1024: + raise InitializationError("offline_quiescence_capture_invalid") + payload = json.loads(raw.decode("utf-8")) + except InitializationError: + raise + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + raise InitializationError("offline_quiescence_capture_invalid") from exc + if not isinstance(payload, dict) or set(payload) != { + "schema_version", + "status", + "captured_at_utc", + "source_commit", + "source_tree", + "source_root_sha256s", + "source_root_set_sha256", + "api_identity", + "tunnel_identity", + "listener_absent", + "tunnel_absent", + "listener_endpoint_sha256", + "tunnel_config_sha256", + }: + raise InitializationError("offline_quiescence_capture_invalid") + if ( + payload.get("schema_version") != OFFLINE_CAPTURE_SCHEMA_VERSION + or payload.get("status") != "quiesced" + or payload.get("listener_absent") is not True + or payload.get("tunnel_absent") is not True + ): + raise InitializationError("offline_quiescence_capture_invalid") + captured_at = payload.get("captured_at_utc") + if not isinstance(captured_at, str) or _UTC_TIMESTAMP_RE.fullmatch(captured_at) is None: + raise InitializationError("offline_quiescence_capture_invalid") + source_commit = payload.get("source_commit") + source_tree = payload.get("source_tree") + if ( + _GIT_OBJECT_RE.fullmatch(expected_source_commit or "") is None + or _GIT_OBJECT_RE.fullmatch(expected_source_tree or "") is None + or source_commit != expected_source_commit + or source_tree != expected_source_tree + or not isinstance(source_commit, str) + or _GIT_OBJECT_RE.fullmatch(source_commit) is None + or not isinstance(source_tree, str) + or _GIT_OBJECT_RE.fullmatch(source_tree) is None + ): + raise InitializationError("offline_quiescence_capture_invalid") + source_root_sha256s, source_root_set_sha256 = _source_root_bindings( + legacy_source_roots + ) + payload_source_hashes = payload.get("source_root_sha256s") + if ( + not isinstance(payload_source_hashes, list) + or tuple(payload_source_hashes) != source_root_sha256s + or payload.get("source_root_set_sha256") != source_root_set_sha256 + ): + raise InitializationError("offline_legacy_source_root_drift") + return { + "schema_version": OFFLINE_CAPTURE_SCHEMA_VERSION, + "status": "quiesced", + "captured_at_utc": captured_at, + "source_commit": source_commit, + "source_tree": source_tree, + "source_root_sha256s": list(source_root_sha256s), + "source_root_set_sha256": source_root_set_sha256, + "api_identity": _strict_process_identity(payload.get("api_identity")), + "tunnel_identity": _strict_process_identity( + payload.get("tunnel_identity") + ), + "listener_absent": True, + "tunnel_absent": True, + "listener_endpoint_sha256": _strict_lower_sha256( + payload.get("listener_endpoint_sha256"), + "offline_quiescence_capture_invalid", + ), + "tunnel_config_sha256": _strict_lower_sha256( + payload.get("tunnel_config_sha256"), + "offline_quiescence_capture_invalid", + ), + } + + +def build_offline_quiescence_receipt( + *, + capture: Mapping[str, object], + inventory: ReferenceInventory, + preserved: PreservedInventory, + database_target_digest: str, +) -> dict[str, object]: + database_digest = _strict_lower_sha256( + database_target_digest, "database_target_invalid" + ) + invalid_object_count, invalid_reference_count = required_decode_invalid_counts( + references=inventory, + preserved=preserved, + ) + return { + "schema_version": OFFLINE_RECEIPT_SCHEMA_VERSION, + "status": "quiesced", + "captured_at_utc": capture["captured_at_utc"], + "source_commit": capture["source_commit"], + "source_tree": capture["source_tree"], + "source_root_sha256s": capture["source_root_sha256s"], + "source_root_set_sha256": capture["source_root_set_sha256"], + "api_identity": capture["api_identity"], + "tunnel_identity": capture["tunnel_identity"], + "listener_absent": True, + "tunnel_absent": True, + "listener_endpoint_sha256": capture["listener_endpoint_sha256"], + "tunnel_config_sha256": capture["tunnel_config_sha256"], + "database_target_sha256": database_digest, + "inventory": { + "preserved_object_count": preserved.object_count, + "preserved_total_size_bytes": preserved.total_size_bytes, + "preserved_inventory_sha256": preserved.inventory_sha256, + "preserved_decode_valid_count": preserved.decode_valid_count, + "preserved_decode_invalid_count": preserved.decode_invalid_count, + "required_decode_invalid_object_count": invalid_object_count, + "required_decode_invalid_reference_count": invalid_reference_count, + "reference_count": inventory.reference_count, + "unique_reference_object_count": inventory.unique_object_count, + "reference_set_sha256": inventory.reference_set_sha256, + }, + "privacy": { + "raw_paths_recorded": False, + "raw_urls_recorded": False, + "raw_user_ids_recorded": False, + "raw_filenames_recorded": False, + "raw_secrets_recorded": False, + }, + } + + +def write_offline_quiescence_receipt_create_only( + *, manifest_state_dir: Path, payload: Mapping[str, object] +) -> tuple[Path, str]: + state_dir = _validated_absolute_directory(manifest_state_dir) + encoded = _json_bytes(dict(payload)) + receipt_sha256 = _sha256_bytes(encoded) + destination = state_dir / f"public-upload-quiescence-{receipt_sha256}.json" + if destination.exists() or destination.is_symlink(): + regular = _validated_regular_file(destination) + if sha256_file(regular) != receipt_sha256 or regular.read_bytes() != encoded: + raise InitializationError("offline_quiescence_receipt_conflict") + return regular, receipt_sha256 + try: + with destination.open("xb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError: + regular = _validated_regular_file(destination) + if sha256_file(regular) != receipt_sha256 or regular.read_bytes() != encoded: + raise InitializationError("offline_quiescence_receipt_conflict") + return regular, receipt_sha256 + regular = _validated_regular_file(destination) + if sha256_file(regular) != receipt_sha256: + raise InitializationError("offline_quiescence_receipt_verification_failed") + return regular, receipt_sha256 + + +def validate_offline_quiescence_receipt( + *, + receipt_path: Path, + expected_receipt_sha256: str, + stable_source_root: Path, + upload_root: Path, + expected_source_commit: str, + expected_source_tree: str, + expected_source_roots: Sequence[Path] | None, + expected_database_target_sha256: str, + expected_preserved_object_count: int, + expected_preserved_total_size_bytes: int, + expected_preserved_inventory_sha256: str, + expected_preserved_decode_valid_count: int, + expected_preserved_decode_invalid_count: int, + expected_required_decode_invalid_object_count: int, + expected_required_decode_invalid_reference_count: int, + expected_reference_count: int, + expected_unique_object_count: int, + expected_reference_set_sha256: str, + expected_source_root_sha256s: Sequence[str] | None = None, + expected_source_root_set_sha256: str = "", +) -> OfflineQuiescenceProof: + if _SHA256_RE.fullmatch(expected_receipt_sha256 or "") is None: + raise InitializationError("offline_quiescence_receipt_hash_invalid") + if _GIT_OBJECT_RE.fullmatch(expected_source_commit or "") is None or _GIT_OBJECT_RE.fullmatch( + expected_source_tree or "" + ) is None: + raise InitializationError("offline_quiescence_source_pin_invalid") + stable_root = _validated_absolute_directory(stable_source_root) + public_root = _validated_absolute_directory(upload_root) + if expected_source_roots: + source_root_sha256s, source_root_set_sha256 = _source_root_bindings( + expected_source_roots + ) + else: + if not expected_source_root_sha256s: + raise InitializationError("offline_quiescence_source_roots_missing") + source_root_sha256s = tuple( + _strict_lower_sha256(value, "offline_quiescence_source_roots_invalid") + for value in expected_source_root_sha256s + ) + if len(set(source_root_sha256s)) != len(source_root_sha256s): + raise InitializationError("offline_quiescence_source_roots_invalid") + source_root_set_sha256 = _sha256_bytes( + _json_bytes(sorted(source_root_sha256s)).rstrip(b"\n") + ) + if source_root_set_sha256 != expected_source_root_set_sha256: + raise InitializationError("offline_quiescence_source_roots_invalid") + regular = _validated_regular_file(receipt_path) + _assert_disjoint(regular, stable_root) + _assert_disjoint(regular, public_root) + for source_root in expected_source_roots or (): + _assert_disjoint(regular, _validated_absolute_directory(source_root)) + if sha256_file(regular) != expected_receipt_sha256: + raise InitializationError("offline_quiescence_receipt_hash_drift") + try: + payload = json.loads(regular.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise InitializationError("offline_quiescence_receipt_invalid") from exc + if not isinstance(payload, dict) or set(payload) != { + "schema_version", + "status", + "captured_at_utc", + "source_commit", + "source_tree", + "source_root_sha256s", + "source_root_set_sha256", + "api_identity", + "tunnel_identity", + "listener_absent", + "tunnel_absent", + "listener_endpoint_sha256", + "tunnel_config_sha256", + "database_target_sha256", + "inventory", + "privacy", + }: + raise InitializationError("offline_quiescence_receipt_invalid") + if ( + payload.get("schema_version") != OFFLINE_RECEIPT_SCHEMA_VERSION + or payload.get("status") != "quiesced" + or payload.get("source_commit") != expected_source_commit + or payload.get("source_tree") != expected_source_tree + or payload.get("listener_absent") is not True + or payload.get("tunnel_absent") is not True + or payload.get("database_target_sha256") + != expected_database_target_sha256 + or payload.get("source_root_sha256s") != list(source_root_sha256s) + or payload.get("source_root_set_sha256") != source_root_set_sha256 + ): + raise InitializationError("offline_quiescence_receipt_contract_drift") + _strict_lower_sha256( + payload.get("source_root_set_sha256"), + "offline_quiescence_receipt_invalid", + ) + payload_source_hashes = payload.get("source_root_sha256s") + if not isinstance(payload_source_hashes, list) or any( + _SHA256_RE.fullmatch(value or "") is None + for value in payload_source_hashes + if isinstance(value, str) + ) or any(not isinstance(value, str) for value in payload_source_hashes): + raise InitializationError("offline_quiescence_receipt_invalid") + _strict_lower_sha256( + payload.get("listener_endpoint_sha256"), + "offline_quiescence_receipt_invalid", + ) + _strict_lower_sha256( + payload.get("tunnel_config_sha256"), + "offline_quiescence_receipt_invalid", + ) + _strict_process_identity(payload.get("api_identity")) + _strict_process_identity(payload.get("tunnel_identity")) + inventory = payload.get("inventory") + if not isinstance(inventory, dict) or set(inventory) != { + "preserved_object_count", + "preserved_total_size_bytes", + "preserved_inventory_sha256", + "preserved_decode_valid_count", + "preserved_decode_invalid_count", + "required_decode_invalid_object_count", + "required_decode_invalid_reference_count", + "reference_count", + "unique_reference_object_count", + "reference_set_sha256", + }: + raise InitializationError("offline_quiescence_receipt_inventory_invalid") + if ( + inventory.get("preserved_object_count") + != expected_preserved_object_count + or inventory.get("preserved_total_size_bytes") + != expected_preserved_total_size_bytes + or inventory.get("preserved_inventory_sha256") + != expected_preserved_inventory_sha256 + or inventory.get("preserved_decode_valid_count") + != expected_preserved_decode_valid_count + or inventory.get("preserved_decode_invalid_count") + != expected_preserved_decode_invalid_count + or inventory.get("required_decode_invalid_object_count") + != expected_required_decode_invalid_object_count + or inventory.get("required_decode_invalid_reference_count") + != expected_required_decode_invalid_reference_count + or inventory.get("reference_count") != expected_reference_count + or inventory.get("unique_reference_object_count") + != expected_unique_object_count + or inventory.get("reference_set_sha256") + != expected_reference_set_sha256 + ): + raise InitializationError("offline_quiescence_receipt_inventory_drift") + privacy = payload.get("privacy") + if not isinstance(privacy, dict) or set(privacy) != { + "raw_paths_recorded", + "raw_urls_recorded", + "raw_user_ids_recorded", + "raw_filenames_recorded", + "raw_secrets_recorded", + } or any(value is not False for value in privacy.values()): + raise InitializationError("offline_quiescence_receipt_privacy_invalid") + return OfflineQuiescenceProof( + receipt_path=regular, + receipt_sha256=expected_receipt_sha256, + database_target_sha256=expected_database_target_sha256, + source_root_sha256s=source_root_sha256s, + source_root_set_sha256=source_root_set_sha256, + preserved_object_count=expected_preserved_object_count, + preserved_total_size_bytes=expected_preserved_total_size_bytes, + preserved_inventory_sha256=expected_preserved_inventory_sha256, + preserved_decode_valid_count=expected_preserved_decode_valid_count, + preserved_decode_invalid_count=expected_preserved_decode_invalid_count, + required_decode_invalid_object_count=( + expected_required_decode_invalid_object_count + ), + required_decode_invalid_reference_count=( + expected_required_decode_invalid_reference_count + ), + reference_count=expected_reference_count, + unique_object_count=expected_unique_object_count, + reference_set_sha256=expected_reference_set_sha256, + ) + + +def _hash_regular_file(path: Path) -> tuple[int, str]: + regular = _validated_regular_file(path) + digest = hashlib.sha256() + size_bytes = 0 + try: + with regular.open("rb") as stream: + before = os.fstat(stream.fileno()) + before_identity = _regular_file_identity(before) + if before_identity is None: + raise InitializationError("source_object_changed_during_hash") + while True: + chunk = stream.read(_COPY_CHUNK_SIZE) + if not chunk: + break + digest.update(chunk) + size_bytes += len(chunk) + after = os.fstat(stream.fileno()) + path_after = regular.lstat() + except InitializationError: + raise + except OSError as exc: + raise InitializationError("source_object_changed_during_hash") from exc + if ( + _has_reparse_attribute(regular) + or before_identity != _regular_file_identity(after) + or before_identity != _regular_file_identity(path_after) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or size_bytes != after.st_size + ): + raise InitializationError("source_object_changed_during_hash") + return size_bytes, digest.hexdigest() + + +def _decode_regular_avatar( + path: Path, + *, + relative_path: str, + expected_size: int, + expected_sha256: str, +) -> bool: + regular = _validated_regular_file(path) + try: + content = regular.read_bytes() + except OSError as exc: + raise InitializationError("source_object_changed_during_decode") from exc + if ( + len(content) != expected_size + or hashlib.sha256(content).hexdigest() != expected_sha256 + ): + raise InitializationError("source_object_changed_during_decode") + return inspect_public_avatar_image(relative_path, content).valid + + +def _preserved_inventory_records( + objects: Iterable[PreservedObject], +) -> list[dict[str, object]]: + return sorted( + ( + { + "path_sha256": public_avatar_relative_path_sha256( + item.relative_path + ), + "content_sha256": item.content_sha256, + "size_bytes": item.size_bytes, + "kind": "profile_avatar", + } + for item in objects + ), + key=lambda item: str(item["path_sha256"]), + ) + + +def scan_preserved_inventory(source_roots: Sequence[Path]) -> PreservedInventory: + validated_sources: list[Path] = [] + source_keys: set[str] = set() + for source_root in source_roots: + validated = _validated_absolute_directory(source_root) + source_key = os.path.normcase(str(validated)) + if source_key in source_keys: + raise InitializationError("source_root_duplicate") + source_keys.add(source_key) + validated_sources.append(validated) + if not validated_sources: + raise InitializationError("source_root_required") + + candidates: dict[str, PreservedObject] = {} + for source_root in validated_sources: + avatar_root = source_root / "profile-avatars" + if not avatar_root.exists() and not avatar_root.is_symlink(): + continue + avatar_root = _validated_absolute_directory(avatar_root) + try: + entries = tuple(avatar_root.iterdir()) + except OSError as exc: + raise InitializationError("source_inventory_scan_failed") from exc + for candidate in entries: + basename = candidate.name + if ( + "%" in basename + or any( + ord(character) < 32 or ord(character) == 127 + for character in basename + ) + or _ALLOWED_AVATAR_NAME.fullmatch(basename) is None + ): + raise InitializationError("source_inventory_entry_invalid") + try: + metadata = candidate.lstat() + except OSError as exc: + raise InitializationError("source_inventory_changed_during_scan") from exc + if _has_reparse_attribute(candidate) or not stat.S_ISREG(metadata.st_mode): + raise InitializationError("source_inventory_entry_invalid") + relative_path = f"profile-avatars/{basename}" + size_bytes, content_sha256 = _hash_regular_file(candidate) + decode_valid = _decode_regular_avatar( + candidate, + relative_path=relative_path, + expected_size=size_bytes, + expected_sha256=content_sha256, + ) + identity = relative_path.casefold() + existing = candidates.get(identity) + if existing is not None: + if ( + existing.size_bytes != size_bytes + or existing.content_sha256 != content_sha256 + ): + raise InitializationError("source_object_conflict") + continue + candidates[identity] = PreservedObject( + relative_path=relative_path, + source_path=candidate, + size_bytes=size_bytes, + content_sha256=content_sha256, + decode_valid=decode_valid, + ) + + objects = tuple( + sorted( + candidates.values(), + key=lambda item: public_avatar_relative_path_sha256( + item.relative_path + ), + ) + ) + records = _preserved_inventory_records(objects) + inventory_sha256 = _sha256_bytes(_json_bytes(records).rstrip(b"\n")) + return PreservedInventory(objects=objects, inventory_sha256=inventory_sha256) + + +def assert_database_references_preserved( + *, references: ReferenceInventory, preserved: PreservedInventory +) -> None: + available = {item.relative_path.casefold() for item in preserved.objects} + if any(name.casefold() not in available for name, _count in references.objects): + raise InitializationError("source_object_missing") + + +def required_decode_invalid_counts( + *, + references: ReferenceInventory, + preserved: PreservedInventory, +) -> tuple[int, int]: + decode_by_path = { + item.relative_path.casefold(): item.decode_valid + for item in preserved.objects + } + invalid_objects = 0 + invalid_references = 0 + for relative_path, reference_count in references.objects: + if decode_by_path.get(relative_path.casefold()) is False: + invalid_objects += 1 + invalid_references += reference_count + return invalid_objects, invalid_references + + +def assert_expected_preserved_inventory( + *, + inventory: PreservedInventory, + expected_object_count: int, + expected_total_size_bytes: int, + expected_inventory_sha256: str, +) -> None: + if expected_object_count <= 0: + raise InitializationError("expected_preserved_object_count_invalid") + expected_digest = _strict_lower_sha256( + expected_inventory_sha256, + "expected_preserved_inventory_sha256_invalid", + ) + if expected_total_size_bytes <= 0: + raise InitializationError("expected_preserved_total_size_bytes_invalid") + if ( + inventory.object_count != expected_object_count + or inventory.total_size_bytes != expected_total_size_bytes + or inventory.inventory_sha256 != expected_digest + ): + raise InitializationError("preserved_inventory_pin_mismatch") + + +def assert_preserved_inventory_stable( + *, expected: PreservedInventory, actual: PreservedInventory +) -> None: + if ( + expected.object_count != actual.object_count + or expected.total_size_bytes != actual.total_size_bytes + or expected.decode_valid_count != actual.decode_valid_count + or expected.decode_invalid_count != actual.decode_invalid_count + or expected.inventory_sha256 != actual.inventory_sha256 + or _preserved_inventory_records(expected.objects) + != _preserved_inventory_records(actual.objects) + or tuple( + (item.relative_path.casefold(), item.decode_valid) + for item in expected.objects + ) + != tuple( + (item.relative_path.casefold(), item.decode_valid) + for item in actual.objects + ) + ): + raise InitializationError("source_inventory_changed_during_copy") + + +def _source_proof( + relative_name: str, source_roots: Sequence[Path] +) -> tuple[Path, int, str]: + candidates: list[tuple[Path, int, str]] = [] + relative_parts = relative_name.split("/") + for source_root in source_roots: + candidate = source_root.joinpath(*relative_parts) + if candidate.exists() or candidate.is_symlink(): + size_bytes, digest = _hash_regular_file(candidate) + candidates.append((candidate, size_bytes, digest)) + if not candidates: + raise InitializationError("source_object_missing") + identities = {(size_bytes, digest) for _, size_bytes, digest in candidates} + if len(identities) != 1: + raise InitializationError("source_object_conflict") + return candidates[0] + + +def _target_matches(path: Path, *, expected_size: int, expected_sha256: str) -> bool: + if not path.exists() and not path.is_symlink(): + return False + size_bytes, digest = _hash_regular_file(path) + return size_bytes == expected_size and digest == expected_sha256 + + +def _regular_file_identity(metadata: os.stat_result) -> tuple[int, int] | None: + if not stat.S_ISREG(metadata.st_mode): + return None + device = int(getattr(metadata, "st_dev", 0)) + inode = int(getattr(metadata, "st_ino", 0)) + if inode <= 0: + return None + return device, inode + + +def _unlink_created_file_if_identity_matches( + path: Path, expected_identity: tuple[int, int] | None +) -> None: + # Once the create-only handle is closed, Windows offers no atomic + # "unlink-this-exact-file-id" primitive through pathlib. A lstat/identity + # check followed by unlink is still racy and could delete another writer's + # replacement. Leave any failed initializer artifact in the unpublished + # target so the freeze remains fail-closed and an operator can inspect it. + del path, expected_identity + + +def _copy_one_create_only( + *, source: Path, target: Path, expected_size: int, expected_sha256: str +) -> bool: + if target.exists() or target.is_symlink(): + if not _target_matches( + target, + expected_size=expected_size, + expected_sha256=expected_sha256, + ): + raise InitializationError("target_object_conflict") + return False + + created_identity: tuple[int, int] | None = None + try: + source_file = _validated_regular_file(source) + _assert_no_reparse_ancestors(target.parent) + with source_file.open("rb") as input_stream, target.open("xb") as output_stream: + source_before = os.fstat(input_stream.fileno()) + source_identity = _regular_file_identity(source_before) + source_path_before = source_file.lstat() + if ( + source_identity is None + or source_identity != _regular_file_identity(source_path_before) + or _has_reparse_attribute(source_file) + ): + raise InitializationError("source_object_changed_during_copy") + created_identity = _regular_file_identity(os.fstat(output_stream.fileno())) + copied_digest = hashlib.sha256() + copied_size = 0 + while True: + chunk = input_stream.read(_COPY_CHUNK_SIZE) + if not chunk: + break + output_stream.write(chunk) + copied_digest.update(chunk) + copied_size += len(chunk) + output_stream.flush() + os.fsync(output_stream.fileno()) + source_after = os.fstat(input_stream.fileno()) + source_path_after = source_file.lstat() + if ( + source_identity != _regular_file_identity(source_after) + or source_identity != _regular_file_identity(source_path_after) + or source_before.st_size != source_after.st_size + or source_before.st_mtime_ns != source_after.st_mtime_ns + or _has_reparse_attribute(source_file) + ): + raise InitializationError("source_object_changed_during_copy") + if copied_size != expected_size or copied_digest.hexdigest() != expected_sha256: + raise InitializationError("source_object_changed_during_copy") + if not _target_matches( + target, + expected_size=expected_size, + expected_sha256=expected_sha256, + ): + raise InitializationError("target_object_verification_failed") + return True + except FileExistsError: + if _target_matches( + target, + expected_size=expected_size, + expected_sha256=expected_sha256, + ): + return False + raise InitializationError("target_object_conflict") + except BaseException: + _unlink_created_file_if_identity_matches(target, created_identity) + raise + + +def _assert_target_inventory_empty_or_exact( + *, + avatar_root: Path, + inventory: PreservedInventory, + allow_empty: bool, +) -> None: + expected = { + item.relative_path.split("/", 1)[1].casefold(): item + for item in inventory.objects + } + try: + entries = tuple(avatar_root.iterdir()) + except OSError as exc: + raise InitializationError("target_inventory_scan_failed") from exc + if not entries and allow_empty: + return + if len(entries) != len(expected): + raise InitializationError("target_inventory_not_empty_or_exact") + seen: set[str] = set() + for candidate in entries: + key = candidate.name.casefold() + preserved = expected.get(key) + if key in seen or preserved is None: + raise InitializationError("target_inventory_not_empty_or_exact") + seen.add(key) + if not _target_matches( + candidate, + expected_size=preserved.size_bytes, + expected_sha256=preserved.content_sha256, + ): + raise InitializationError("target_inventory_not_empty_or_exact") + if seen != set(expected): + raise InitializationError("target_inventory_not_empty_or_exact") + + +def copy_required_objects( + *, + inventory: PreservedInventory, + source_roots: Sequence[Path], + upload_root: Path, +) -> CopyProof: + validated_upload_root = _validated_absolute_directory(upload_root) + validated_sources = tuple( + _validated_absolute_directory(source_root) for source_root in source_roots + ) + if not validated_sources: + raise InitializationError("source_root_required") + for source_root in validated_sources: + _assert_disjoint(source_root, validated_upload_root) + + avatar_root = validated_upload_root / "profile-avatars" + if avatar_root.exists() or avatar_root.is_symlink(): + avatar_root = _validated_absolute_directory(avatar_root) + else: + _assert_no_reparse_ancestors(avatar_root.parent) + avatar_root.mkdir() + avatar_root = _validated_absolute_directory(avatar_root) + + _assert_target_inventory_empty_or_exact( + avatar_root=avatar_root, + inventory=inventory, + allow_empty=True, + ) + copied = 0 + reused = 0 + for preserved_object in inventory.objects: + source = preserved_object.source_path + size_bytes, digest = _hash_regular_file(source) + if ( + size_bytes != preserved_object.size_bytes + or digest != preserved_object.content_sha256 + ): + raise InitializationError("source_inventory_changed_during_copy") + target = avatar_root / preserved_object.relative_path.split("/", 1)[1] + if _copy_one_create_only( + source=source, + target=target, + expected_size=size_bytes, + expected_sha256=digest, + ): + copied += 1 + else: + reused += 1 + _assert_target_inventory_empty_or_exact( + avatar_root=avatar_root, + inventory=inventory, + allow_empty=False, + ) + return CopyProof(inventory.object_count, copied, reused) + + +def _read_freeze_token(freeze_path: Path) -> tuple[str, str]: + regular = _validated_regular_file(freeze_path) + try: + payload = json.loads(regular.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise InitializationError("write_freeze_sentinel_invalid") from exc + token = payload.get("token") if isinstance(payload, dict) else None + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != WRITE_FREEZE_SCHEMA_VERSION + or not isinstance(token, str) + or len(token) < 32 + ): + raise InitializationError("write_freeze_sentinel_invalid") + return token, _sha256_bytes(token.encode("utf-8")) + + +def wait_for_drained_runtime_freeze( + *, health_url: str, token_sha256: str, timeout_seconds: float +) -> None: + parsed_health_url = urllib.parse.urlsplit(health_url) + try: + health_port = parsed_health_url.port + except ValueError as exc: + raise InitializationError("runtime_health_url_invalid") from exc + if ( + parsed_health_url.scheme != "http" + or parsed_health_url.hostname != "127.0.0.1" + or parsed_health_url.username is not None + or parsed_health_url.password is not None + or health_port is None + or parsed_health_url.path != "/health" + or parsed_health_url.query + or parsed_health_url.fragment + ): + raise InitializationError("runtime_health_url_invalid") + deadline = time.monotonic() + timeout_seconds + opener = urllib.request.build_opener(_RejectRedirects()) + while time.monotonic() < deadline: + try: + request = urllib.request.Request( + health_url, headers={"Accept": "application/json"}, method="GET" + ) + with opener.open(request, timeout=2.0) as response: + payload = json.loads(response.read().decode("utf-8")) + freeze = ( + payload.get("upload_write_freeze") + if isinstance(payload, dict) + else None + ) + if ( + isinstance(freeze, dict) + and freeze.get("capable") is True + and freeze.get("active") is True + and freeze.get("valid") is True + and freeze.get("in_flight") == 0 + and freeze.get("token_sha256") == token_sha256 + ): + return + except ( + OSError, + UnicodeError, + json.JSONDecodeError, + urllib.error.URLError, + ): + pass + time.sleep(0.25) + raise InitializationError("runtime_write_freeze_not_drained") + + +async def fetch_database_inventory() -> tuple[list[object], int, str]: + try: + import asyncpg + from app.config import settings + except ImportError as exc: + raise InitializationError("database_client_unavailable") from exc + + connection = None + try: + connection = await asyncpg.connect( + settings.database_url, command_timeout=settings.db_command_timeout + ) + async with connection.transaction(isolation="repeatable_read", readonly=True): + database_target_digest = await connected_database_target_sha256(connection) + await connection.execute( + "SELECT set_config('app.ai_context', '', true), " + "set_config('app.current_role', 'admin', true)" + ) + rows = await connection.fetch( + """ + SELECT avatar_url + FROM app.app_user + WHERE avatar_url IS NOT NULL + AND left(avatar_url, 9) = '/uploads/' + ORDER BY avatar_url + """ + ) + active_private_audio_count = int( + await connection.fetchval( + """ + SELECT count(*) + FROM app.multimodal_audio_asset AS audio + WHERE audio.retained_until > now() + AND NOT EXISTS ( + SELECT 1 + FROM audit.multimodal_deletion_tombstone AS tombstone + WHERE tombstone.session_id = audio.session_id + AND tombstone.scope = 'audio' + ) + """ + ) + ) + return ( + [row["avatar_url"] for row in rows], + active_private_audio_count, + database_target_digest, + ) + except InitializationError: + raise + except Exception as exc: + raise InitializationError("database_inventory_query_failed") from exc + finally: + if connection is not None: + await connection.close() + + +def assert_no_active_private_audio(active_private_audio_count: int) -> None: + if active_private_audio_count != 0: + raise InitializationError("active_private_audio_requires_separate_migration") + + +def privacy_safe_manifest( + *, + upload_root: Path, + inventory: ReferenceInventory, + preserved: PreservedInventory, + database_target_sha256: str, + freeze_token_sha256: str, + freeze_path: Path, + offline_quiescence: OfflineQuiescenceProof | None = None, +) -> dict[str, object]: + if _SHA256_RE.fullmatch(freeze_token_sha256) is None: + raise InitializationError("write_freeze_token_hash_invalid") + manifest = build_privacy_safe_manifest( + upload_root=upload_root, + references=[ + (name, "profile_avatar") + for name, reference_count in inventory.objects + for _ in range(reference_count) + ], + preserved_paths=[item.relative_path for item in preserved.objects], + database_target_sha256=database_target_sha256, + write_freeze_token_sha256=freeze_token_sha256, + write_freeze_path=freeze_path, + ) + ( + required_decode_invalid_object_count, + required_decode_invalid_reference_count, + ) = required_decode_invalid_counts( + references=inventory, + preserved=preserved, + ) + if ( + manifest.get("database_target_sha256") != database_target_sha256 + or manifest.get("preserved_object_count") != preserved.object_count + or manifest.get("preserved_total_size_bytes") + != preserved.total_size_bytes + or manifest.get("preserved_decode_valid_count") + != preserved.decode_valid_count + or manifest.get("preserved_decode_invalid_count") + != preserved.decode_invalid_count + or manifest.get("required_decode_invalid_object_count") + != required_decode_invalid_object_count + or manifest.get("required_decode_invalid_reference_count") + != required_decode_invalid_reference_count + or manifest.get("preserved_object_set_sha256") + != preserved.inventory_sha256 + or manifest.get("required_object_count") != inventory.unique_object_count + or manifest.get("required_reference_count") != inventory.reference_count + or manifest.get("reference_set_sha256") != inventory.reference_set_sha256 + ): + raise InitializationError("manifest_inventory_contract_drift") + if offline_quiescence is not None: + manifest["offline_quiescence"] = { + "receipt_sha256": offline_quiescence.receipt_sha256, + "database_target_sha256": offline_quiescence.database_target_sha256, + "source_root_sha256s": list( + offline_quiescence.source_root_sha256s + ), + "source_root_set_sha256": offline_quiescence.source_root_set_sha256, + "preserved_object_count": offline_quiescence.preserved_object_count, + "preserved_total_size_bytes": ( + offline_quiescence.preserved_total_size_bytes + ), + "preserved_inventory_sha256": ( + offline_quiescence.preserved_inventory_sha256 + ), + "preserved_decode_valid_count": ( + offline_quiescence.preserved_decode_valid_count + ), + "preserved_decode_invalid_count": ( + offline_quiescence.preserved_decode_invalid_count + ), + "required_decode_invalid_object_count": ( + offline_quiescence.required_decode_invalid_object_count + ), + "required_decode_invalid_reference_count": ( + offline_quiescence.required_decode_invalid_reference_count + ), + "reference_count": offline_quiescence.reference_count, + "unique_object_count": offline_quiescence.unique_object_count, + "reference_set_sha256": offline_quiescence.reference_set_sha256, + } + return manifest + + +def write_manifest_create_only( + *, manifest_state_dir: Path, payload: Mapping[str, object] +) -> tuple[Path, str]: + manifest_state_dir = _validated_absolute_directory(manifest_state_dir) + encoded = _json_bytes(dict(payload)) + manifest_sha256 = _sha256_bytes(encoded) + destination = manifest_state_dir / f"public-avatar-upload-{manifest_sha256}.json" + if destination.exists() or destination.is_symlink(): + regular = _validated_regular_file(destination) + if sha256_file(regular) != manifest_sha256 or regular.read_bytes() != encoded: + raise InitializationError("manifest_create_conflict") + return regular, manifest_sha256 + try: + with destination.open("xb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError: + regular = _validated_regular_file(destination) + if sha256_file(regular) != manifest_sha256 or regular.read_bytes() != encoded: + raise InitializationError("manifest_create_conflict") + return regular, manifest_sha256 + regular = _validated_regular_file(destination) + if sha256_file(regular) != manifest_sha256: + raise InitializationError("manifest_verification_failed") + return regular, manifest_sha256 + + +def privacy_safe_result( + *, + manifest_path: Path, + manifest_sha256: str, + upload_root: Path, + inventory: ReferenceInventory, + preserved: PreservedInventory, + copy_proof: CopyProof, + freeze_token_sha256: str, + offline_quiescence: OfflineQuiescenceProof | None = None, +) -> dict[str, object]: + ( + required_decode_invalid_object_count, + required_decode_invalid_reference_count, + ) = required_decode_invalid_counts( + references=inventory, + preserved=preserved, + ) + result = { + "status": "initialized", + "manifest_sha256": manifest_sha256, + "manifest_path_sha256": canonical_path_sha256(manifest_path), + "root_path_sha256": canonical_path_sha256(upload_root), + "preserved_object_count": preserved.object_count, + "preserved_total_size_bytes": preserved.total_size_bytes, + "preserved_inventory_sha256": preserved.inventory_sha256, + "preserved_decode_valid_count": preserved.decode_valid_count, + "preserved_decode_invalid_count": preserved.decode_invalid_count, + "required_decode_invalid_object_count": ( + required_decode_invalid_object_count + ), + "required_decode_invalid_reference_count": ( + required_decode_invalid_reference_count + ), + "required_object_count": inventory.unique_object_count, + "database_reference_count": inventory.reference_count, + "database_reference_set_sha256": inventory.reference_set_sha256, + "copied_object_count": copy_proof.copied_count, + "reused_exact_object_count": copy_proof.reused_exact_count, + "write_freeze_token_sha256": freeze_token_sha256, + } + if offline_quiescence is not None: + result["offline_quiescence_receipt_sha256"] = ( + offline_quiescence.receipt_sha256 + ) + result["offline_quiescence_receipt_path_sha256"] = canonical_path_sha256( + offline_quiescence.receipt_path + ) + return result + + +async def initialize(args: argparse.Namespace) -> dict[str, object]: + upload_root = _validated_absolute_directory(Path(args.upload_root)) + state_dir = _validated_absolute_directory(Path(args.manifest_state_dir)) + freeze_path = _validated_regular_file(Path(args.write_freeze_path)) + if freeze_path.parent != state_dir: + raise InitializationError("write_freeze_must_be_direct_private_state_child") + _assert_disjoint(upload_root, state_dir) + source_roots = tuple( + _validated_absolute_directory(Path(value)) for value in args.source_root + ) + if not source_roots: + raise InitializationError("source_root_required") + for source_root in source_roots: + _assert_disjoint(source_root, upload_root) + _assert_disjoint(source_root, state_dir) + + _, freeze_token_sha256 = _read_freeze_token(freeze_path) + offline_capture: dict[str, object] | None = None + offline_pin_pair_present = bool( + args.expected_offline_source_commit and args.expected_offline_source_tree + ) + if bool(args.offline_quiescence_capture_base64) != offline_pin_pair_present: + raise InitializationError("offline_quiescence_source_pins_required_together") + if args.offline_quiescence_capture_base64: + offline_capture = decode_offline_quiescence_capture( + args.offline_quiescence_capture_base64, + legacy_source_roots=source_roots, + expected_source_commit=args.expected_offline_source_commit, + expected_source_tree=args.expected_offline_source_tree, + ) + else: + wait_for_drained_runtime_freeze( + health_url=args.health_url, + token_sha256=freeze_token_sha256, + timeout_seconds=args.freeze_timeout_seconds, + ) + ( + avatar_urls, + active_private_audio_count, + database_digest, + ) = await fetch_database_inventory() + assert_no_active_private_audio(active_private_audio_count) + inventory = build_reference_inventory( + avatar_urls, expected_reference_count=args.expected_reference_count + ) + preserved = scan_preserved_inventory(source_roots) + assert_expected_preserved_inventory( + inventory=preserved, + expected_object_count=args.expected_preserved_object_count, + expected_total_size_bytes=args.expected_preserved_total_size_bytes, + expected_inventory_sha256=args.expected_preserved_inventory_sha256, + ) + assert_database_references_preserved( + references=inventory, + preserved=preserved, + ) + ( + required_decode_invalid_object_count, + required_decode_invalid_reference_count, + ) = required_decode_invalid_counts( + references=inventory, + preserved=preserved, + ) + offline_quiescence: OfflineQuiescenceProof | None = None + if offline_capture is not None: + receipt_payload = build_offline_quiescence_receipt( + capture=offline_capture, + inventory=inventory, + preserved=preserved, + database_target_digest=database_digest, + ) + receipt_path, receipt_sha256 = ( + write_offline_quiescence_receipt_create_only( + manifest_state_dir=state_dir, + payload=receipt_payload, + ) + ) + offline_quiescence = validate_offline_quiescence_receipt( + receipt_path=receipt_path, + expected_receipt_sha256=receipt_sha256, + stable_source_root=REPO_ROOT, + upload_root=upload_root, + expected_source_commit=str(offline_capture["source_commit"]), + expected_source_tree=str(offline_capture["source_tree"]), + expected_source_roots=source_roots, + expected_database_target_sha256=database_digest, + expected_preserved_object_count=preserved.object_count, + expected_preserved_total_size_bytes=preserved.total_size_bytes, + expected_preserved_inventory_sha256=preserved.inventory_sha256, + expected_preserved_decode_valid_count=preserved.decode_valid_count, + expected_preserved_decode_invalid_count=preserved.decode_invalid_count, + expected_required_decode_invalid_object_count=( + required_decode_invalid_object_count + ), + expected_required_decode_invalid_reference_count=( + required_decode_invalid_reference_count + ), + expected_reference_count=inventory.reference_count, + expected_unique_object_count=inventory.unique_object_count, + expected_reference_set_sha256=inventory.reference_set_sha256, + ) + copy_proof = copy_required_objects( + inventory=preserved, + source_roots=source_roots, + upload_root=upload_root, + ) + preserved_after_copy = scan_preserved_inventory(source_roots) + assert_expected_preserved_inventory( + inventory=preserved_after_copy, + expected_object_count=args.expected_preserved_object_count, + expected_total_size_bytes=args.expected_preserved_total_size_bytes, + expected_inventory_sha256=args.expected_preserved_inventory_sha256, + ) + assert_preserved_inventory_stable( + expected=preserved, + actual=preserved_after_copy, + ) + payload = privacy_safe_manifest( + upload_root=upload_root, + inventory=inventory, + preserved=preserved, + database_target_sha256=database_digest, + freeze_token_sha256=freeze_token_sha256, + freeze_path=freeze_path, + offline_quiescence=offline_quiescence, + ) + manifest_path, manifest_sha256 = write_manifest_create_only( + manifest_state_dir=state_dir, + payload=payload, + ) + if manifest_path.parent != state_dir: + raise InitializationError("manifest_private_state_boundary_failed") + return privacy_safe_result( + manifest_path=manifest_path, + manifest_sha256=manifest_sha256, + upload_root=upload_root, + inventory=inventory, + preserved=preserved, + copy_proof=copy_proof, + freeze_token_sha256=freeze_token_sha256, + offline_quiescence=offline_quiescence, + ) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Initialize the public avatar upload root without overwrites." + ) + parser.add_argument("--upload-root", required=True) + parser.add_argument("--manifest-state-dir", required=True) + parser.add_argument("--write-freeze-path", required=True) + parser.add_argument("--expected-reference-count", type=int, required=True) + parser.add_argument("--expected-preserved-object-count", type=int, required=True) + parser.add_argument( + "--expected-preserved-total-size-bytes", type=int, required=True + ) + parser.add_argument("--expected-preserved-inventory-sha256", required=True) + parser.add_argument("--source-root", action="append", required=True) + parser.add_argument("--health-url", default="http://127.0.0.1:8001/health") + parser.add_argument("--freeze-timeout-seconds", type=float, default=30.0) + parser.add_argument("--offline-quiescence-capture-base64", default="") + parser.add_argument("--expected-offline-source-commit", default="") + parser.add_argument("--expected-offline-source-tree", default="") + return parser.parse_args(argv) + + +def _verify_preserved_inventory_cli(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + description="Verify a caller-pinned, privacy-safe preserved avatar inventory." + ) + parser.add_argument("--source-root", action="append", required=True) + parser.add_argument("--expected-preserved-object-count", type=int, required=True) + parser.add_argument( + "--expected-preserved-total-size-bytes", type=int, required=True + ) + parser.add_argument("--expected-preserved-inventory-sha256", required=True) + args = parser.parse_args(argv) + try: + inventory = scan_preserved_inventory( + tuple(Path(value) for value in args.source_root) + ) + assert_expected_preserved_inventory( + inventory=inventory, + expected_object_count=args.expected_preserved_object_count, + expected_total_size_bytes=args.expected_preserved_total_size_bytes, + expected_inventory_sha256=args.expected_preserved_inventory_sha256, + ) + except InitializationError as exc: + print( + json.dumps( + {"status": "failed", "reason": str(exc)}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 1 + except BaseException: + print( + json.dumps( + {"status": "failed", "reason": "preserved_inventory_probe_failed"}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 1 + print( + json.dumps( + { + "status": "verified", + "preserved_object_count": inventory.object_count, + "preserved_total_size_bytes": inventory.total_size_bytes, + "preserved_inventory_sha256": inventory.inventory_sha256, + "preserved_decode_valid_count": inventory.decode_valid_count, + "preserved_decode_invalid_count": inventory.decode_invalid_count, + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + effective_argv = list(sys.argv[1:] if argv is None else argv) + if effective_argv and effective_argv[0] == "probe-preserved-inventory": + return _verify_preserved_inventory_cli(effective_argv[1:]) + try: + result = asyncio.run(initialize(parse_args(effective_argv))) + except InitializationError as exc: + print( + json.dumps( + {"status": "failed", "reason": str(exc)}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 1 + except BaseException: + print( + json.dumps( + {"status": "failed", "reason": "unexpected_initializer_failure"}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 1 + print( + json.dumps( + result, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install-public-runtime-task.ps1 b/scripts/install-public-runtime-task.ps1 index 5dea5a6..5945150 100644 --- a/scripts/install-public-runtime-task.ps1 +++ b/scripts/install-public-runtime-task.ps1 @@ -3,9 +3,23 @@ param( [string]$StableSourceRoot, [string]$TaskName = "VignettePublicRuntimeWatchdog", [int]$IntervalMinutes = 5, + [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, [string[]]$AdditionalPublicHealthUrls = @(), [switch]$SkipPublicHealth, [switch]$SkipCloudflaredRestart, + [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", + [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", + [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health", + [switch]$InitiallyDisabled, [switch]$RunNow ) @@ -14,6 +28,9 @@ $ErrorActionPreference = "Stop" if ($IntervalMinutes -lt 1) { throw "IntervalMinutes must be 1 or greater" } +if ($InitiallyDisabled -and $RunNow) { + throw "InitiallyDisabled and RunNow cannot be combined" +} $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime-task.ps1" @@ -21,6 +38,10 @@ $watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" $taskLauncher = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime-task.vbs" +$uploadRootContract = Join-Path $resolvedSourceRoot "scripts\public-runtime-upload-root.ps1" +$uploadRootProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-runtime-upload-root.py" +$uploadManifestProbe = Join-Path $resolvedSourceRoot "scripts\validate-public-runtime-upload-manifest.py" +$databaseIdentityHelper = Join-Path $resolvedSourceRoot "scripts\public_runtime_database_identity.py" function Invoke-GitText { param([string[]]$Arguments) @@ -32,7 +53,17 @@ function Invoke-GitText { return (@($value) -join [Environment]::NewLine).Trim() } -foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe, $taskLauncher)) { +foreach ($requiredScript in @( + $installerScript, + $watchScript, + $startScript, + $voiceSidecarProbe, + $taskLauncher, + $uploadRootContract, + $uploadRootProbe, + $uploadManifestProbe, + $databaseIdentityHelper +)) { if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Public runtime script not found at $requiredScript" } @@ -75,7 +106,13 @@ foreach ($relativePath in @( "scripts/watch-public-runtime.ps1", "scripts/start-public-runtime.ps1", "scripts/probe-public-voice-sidecars.py", - "scripts/watch-public-runtime-task.vbs" + "scripts/watch-public-runtime-task.vbs", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -84,6 +121,36 @@ $sourceCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD") $sourceTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}") $watchdogSha256 = (Get-FileHash -LiteralPath $watchScript -Algorithm SHA256).Hash.ToLowerInvariant() $startScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant() +$resolvedPython = (Resolve-Path -LiteralPath $Python).Path +$resolvedCloudflared = (Resolve-Path -LiteralPath $Cloudflared).Path +$resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfig).Path +$pythonSha256 = (Get-FileHash -LiteralPath $resolvedPython -Algorithm SHA256).Hash.ToLowerInvariant() +$cloudflaredSha256 = (Get-FileHash -LiteralPath $resolvedCloudflared -Algorithm SHA256).Hash.ToLowerInvariant() +$cloudflaredConfigSha256 = (Get-FileHash -LiteralPath $resolvedCloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant() +. $uploadRootContract +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $UserUploadDir ` + -ProbeWritable +$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadManifestPath ` + -RequireFile +$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadWriteFreezePath +$uploadManifestProof = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $uploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +if (-not $uploadManifestProof.Ok) { + throw "Refusing task installation: public upload migration receipt or current DB inventory is invalid" +} $wscript = Join-Path $env:SystemRoot "System32\wscript.exe" if (!(Test-Path -LiteralPath $wscript -PathType Leaf)) { @@ -100,7 +167,18 @@ $actionArguments = @( "-ExpectedSourceCommit $sourceCommit", "-ExpectedSourceTree $sourceTree", "-ExpectedWatchdogSha256 $watchdogSha256", - "-ExpectedStartScriptSha256 $startScriptSha256" + "-ExpectedStartScriptSha256 $startScriptSha256", + "-ExpectedPythonSha256 $pythonSha256", + "-ExpectedCloudflaredSha256 $cloudflaredSha256", + "-ExpectedCloudflaredConfigSha256 $cloudflaredConfigSha256", + "-Python `"$resolvedPython`"", + "-Cloudflared `"$resolvedCloudflared`"", + "-CloudflaredConfig `"$resolvedCloudflaredConfig`"", + "-UserUploadDir `"$resolvedUserUploadDir`"", + "-UserUploadManifestPath `"$resolvedUserUploadManifestPath`"", + "-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256", + "-UserUploadWriteFreezePath `"$resolvedUserUploadWriteFreezePath`"", + "-PublicHealthUrl `"$PublicHealthUrl`"" ) if ($SkipPublicHealth) { $actionArguments += "-SkipPublicHealth" @@ -125,6 +203,7 @@ $repeatTrigger = New-ScheduledTaskTrigger ` -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) $settings = New-ScheduledTaskSettingsSet ` + -Disable:$InitiallyDisabled ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` -ExecutionTimeLimit (New-TimeSpan -Minutes 60) ` @@ -145,18 +224,31 @@ $task = New-ScheduledTask ` -Principal $principal ` -Description $description -Register-ScheduledTask -TaskName $TaskName -InputObject $task -Force | Out-Null +Register-ScheduledTask ` + -TaskName $TaskName ` + -TaskPath "\" ` + -InputObject $task ` + -Force ` + | Out-Null + +if ($InitiallyDisabled) { + $registered = Get-ScheduledTask -TaskName $TaskName -TaskPath "\" -ErrorAction Stop + if ([bool]$registered.Settings.Enabled -or [string]$registered.State -eq "Running") { + throw "New watchdog task did not remain disabled and idle" + } +} Write-Output "Installed scheduled task '$TaskName' for $userId" Write-Output "Action: $wscript `"$taskLauncher`" $($actionArguments -join ' ')" Write-Output "Pinned source: root=$resolvedSourceRoot commit=$sourceCommit tree=$sourceTree" Write-Output "Pinned scripts: watchdog_sha256=$watchdogSha256 start_sha256=$startScriptSha256" +Write-Output "Pinned user upload root: $resolvedUserUploadDir" Write-Output "Interval: every $IntervalMinutes minute(s), plus at user logon" if ($AdditionalPublicHealthUrls.Count -gt 0) { Write-Output "Additional public health URLs: $($AdditionalPublicHealthUrls -join ', ')" } if ($RunNow) { - Start-ScheduledTask -TaskName $TaskName + Start-ScheduledTask -TaskName $TaskName -TaskPath "\" Write-Output "Started scheduled task '$TaskName'" } diff --git a/scripts/probe-public-runtime-database-identity.py b/scripts/probe-public-runtime-database-identity.py new file mode 100644 index 0000000..3eb246e --- /dev/null +++ b/scripts/probe-public-runtime-database-identity.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Emit only the credential-free identity of the connected runtime database.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +API_ROOT = REPO_ROOT / "apps" / "api" +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from app.config import settings # noqa: E402 +from public_runtime_database_identity import ( # noqa: E402 + connected_database_target_sha256, +) + + +async def _probe() -> str: + import asyncpg + + connection = await asyncpg.connect( + settings.database_url, + command_timeout=settings.db_command_timeout, + ) + try: + return await connected_database_target_sha256(connection) + finally: + await connection.close() + + +def main() -> int: + try: + digest = asyncio.run(_probe()) + except BaseException: + print( + json.dumps( + {"status": "failed", "reason": "database_identity_probe_failed"}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 1 + print( + json.dumps( + {"status": "passed", "database_target_sha256": digest}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe-public-runtime-upload-root.py b/scripts/probe-public-runtime-upload-root.py new file mode 100644 index 0000000..2e13114 --- /dev/null +++ b/scripts/probe-public-runtime-upload-root.py @@ -0,0 +1,369 @@ +"""Prove the exact loopback API listener inherited the pinned runtime contract. + +The probe first resolves the unique PID that owns the exact +``127.0.0.1:`` listening socket. It then inspects only that PID's +argv, cwd, upload receipt environment, and a secret-free database target +fingerprint. Process-wide scans and look-alike uvicorn commands are deliberately +not accepted as runtime identity proof. + +Output is privacy-safe: no full environment or arbitrary command line is +emitted. +""" + +from __future__ import annotations + +import argparse +import json +import ntpath +import re +from typing import Any, Iterable, Mapping, Sequence + + +LOOPBACK_ADDRESS = "127.0.0.1" +LOWER_SHA256 = re.compile(r"^[0-9a-f]{64}$") +RUNTIME_ENVIRONMENT_NAMES = ( + "USER_UPLOAD_DIR", + "USER_UPLOAD_MANIFEST_REQUIRED", + "USER_UPLOAD_MANIFEST_PATH", + "USER_UPLOAD_MANIFEST_SHA256", + "USER_UPLOAD_WRITE_FREEZE_PATH", + "PUBLIC_RUNTIME_DB_TARGET_SHA256", +) + + +def _normalized_windows_path(value: str) -> str: + return ntpath.normcase(ntpath.normpath(value)) + + +def _option_value(argv: Sequence[str], option: str) -> str | None: + """Return an option's sole split-form value, or None when ambiguous.""" + + indexes = [index for index, value in enumerate(argv) if value == option] + if len(indexes) != 1: + return None + index = indexes[0] + if index + 1 >= len(argv): + return None + return argv[index + 1] + + +def _matches_public_api(argv: Sequence[str], api_port: int) -> bool: + values = [str(value) for value in argv] + return ( + values.count("uvicorn") == 1 + and values.count("app.main:app") == 1 + and _option_value(values, "--host") == LOOPBACK_ADDRESS + and _option_value(values, "--port") == str(api_port) + and _option_value(values, "--workers") == "1" + ) + + +def _snapshot_environment(snapshot: Mapping[str, Any]) -> Mapping[str, Any]: + environment = snapshot.get("environment") + if isinstance(environment, Mapping): + return environment + # Compatibility with the original pure-function fixtures. The live probe + # never relies on this flattened representation. + return {"USER_UPLOAD_DIR": snapshot.get("user_upload_root")} + + +def evaluate_listener_binding( + listener_pids: Iterable[int], + process_snapshots: Iterable[Mapping[str, Any]], + *, + expected_root: str, + expected_api_cwd: str, + expected_manifest_path: str, + expected_manifest_sha256: str, + expected_write_freeze_path: str, + expected_database_target_sha256: str, + api_port: int, +) -> tuple[int, dict[str, Any]]: + """Evaluate snapshots only for the unique exact loopback listener PID.""" + + unique_listener_pids = sorted({int(pid) for pid in listener_pids}) + if len(unique_listener_pids) != 1: + return 1, { + "status": "failed", + "reason": "api_listener_count_mismatch", + "listening_processes": len(unique_listener_pids), + } + + listener_pid = unique_listener_pids[0] + listener_snapshots = [ + snapshot + for snapshot in process_snapshots + if int(snapshot.get("pid", -1)) == listener_pid + ] + if len(listener_snapshots) != 1: + return 1, { + "status": "failed", + "reason": "listener_process_unavailable", + "pid": listener_pid, + } + + snapshot = listener_snapshots[0] + if not _matches_public_api(snapshot.get("argv") or [], api_port): + return 1, { + "status": "failed", + "reason": "listener_command_mismatch", + "pid": listener_pid, + } + + actual_cwd = snapshot.get("cwd") + if not isinstance(actual_cwd, str) or not actual_cwd.strip(): + return 1, { + "status": "failed", + "reason": "listener_cwd_missing", + "pid": listener_pid, + } + if _normalized_windows_path(actual_cwd) != _normalized_windows_path( + expected_api_cwd + ): + return 1, { + "status": "failed", + "reason": "listener_cwd_drift", + "pid": listener_pid, + "expected_api_cwd": expected_api_cwd, + "actual_api_cwd": actual_cwd, + } + + environment = _snapshot_environment(snapshot) + actual_root = environment.get("USER_UPLOAD_DIR") + if not isinstance(actual_root, str) or not actual_root.strip(): + return 1, { + "status": "failed", + "reason": "user_upload_dir_missing", + "pid": listener_pid, + } + if _normalized_windows_path(actual_root) != _normalized_windows_path(expected_root): + return 1, { + "status": "failed", + "reason": "user_upload_dir_drift", + "pid": listener_pid, + "expected_root": expected_root, + "actual_root": actual_root, + } + + if environment.get("USER_UPLOAD_MANIFEST_REQUIRED") != "true": + return 1, { + "status": "failed", + "reason": "upload_manifest_required_mismatch", + "pid": listener_pid, + } + + actual_manifest_path = environment.get("USER_UPLOAD_MANIFEST_PATH") + if not isinstance(actual_manifest_path, str) or not actual_manifest_path.strip(): + return 1, { + "status": "failed", + "reason": "upload_manifest_path_missing", + "pid": listener_pid, + } + if _normalized_windows_path(actual_manifest_path) != _normalized_windows_path( + expected_manifest_path + ): + return 1, { + "status": "failed", + "reason": "upload_manifest_path_drift", + "pid": listener_pid, + } + + actual_manifest_sha256 = environment.get("USER_UPLOAD_MANIFEST_SHA256") + if ( + not isinstance(actual_manifest_sha256, str) + or LOWER_SHA256.fullmatch(actual_manifest_sha256) is None + or LOWER_SHA256.fullmatch(expected_manifest_sha256) is None + or actual_manifest_sha256 != expected_manifest_sha256 + ): + return 1, { + "status": "failed", + "reason": "upload_manifest_sha256_drift", + "pid": listener_pid, + } + + actual_freeze_path = environment.get("USER_UPLOAD_WRITE_FREEZE_PATH") + if not isinstance(actual_freeze_path, str) or not actual_freeze_path.strip(): + return 1, { + "status": "failed", + "reason": "upload_write_freeze_path_missing", + "pid": listener_pid, + } + if _normalized_windows_path(actual_freeze_path) != _normalized_windows_path( + expected_write_freeze_path + ): + return 1, { + "status": "failed", + "reason": "upload_write_freeze_path_drift", + "pid": listener_pid, + } + + actual_database_target_sha256 = environment.get("PUBLIC_RUNTIME_DB_TARGET_SHA256") + if ( + not isinstance(actual_database_target_sha256, str) + or LOWER_SHA256.fullmatch(actual_database_target_sha256) is None + or LOWER_SHA256.fullmatch(expected_database_target_sha256) is None + or actual_database_target_sha256 != expected_database_target_sha256 + ): + return 1, { + "status": "failed", + "reason": "database_target_sha256_drift", + "pid": listener_pid, + } + + return 0, { + "status": "passed", + "pid": listener_pid, + "user_upload_root": actual_root, + "api_cwd": actual_cwd, + "listener": f"{LOOPBACK_ADDRESS}:{api_port}", + "manifest_sha256": actual_manifest_sha256, + "database_target_sha256": actual_database_target_sha256, + } + + +def evaluate_snapshots( + snapshots: Iterable[Mapping[str, Any]], + *, + expected_root: str, + api_port: int, +) -> tuple[int, dict[str, Any]]: + """Compatibility wrapper for legacy unit fixtures. + + Live execution does not call this function. It preserves the historical + argv/root-only contract so older focused tests remain useful without + weakening listener-bound production proof. + """ + + matches = [ + snapshot + for snapshot in snapshots + if _matches_public_api(snapshot.get("argv") or [], api_port) + ] + if len(matches) != 1: + return 1, { + "status": "failed", + "reason": "api_process_count_mismatch", + "matching_processes": len(matches), + } + + match = matches[0] + actual_root = _snapshot_environment(match).get("USER_UPLOAD_DIR") + if not isinstance(actual_root, str) or not actual_root.strip(): + return 1, { + "status": "failed", + "reason": "user_upload_dir_missing", + "pid": int(match["pid"]), + } + if _normalized_windows_path(actual_root) != _normalized_windows_path(expected_root): + return 1, { + "status": "failed", + "reason": "user_upload_dir_drift", + "pid": int(match["pid"]), + "expected_root": expected_root, + "actual_root": actual_root, + } + return 0, { + "status": "passed", + "pid": int(match["pid"]), + "user_upload_root": actual_root, + } + + +def _connection_address(connection: Any) -> tuple[str, int] | None: + address = connection.laddr + if not address: + return None + try: + return str(address.ip), int(address.port) + except AttributeError: + if len(address) < 2: + return None + return str(address[0]), int(address[1]) + + +def _collect_listener_pids(api_port: int) -> list[int]: + import psutil + + listener_pids: set[int] = set() + for connection in psutil.net_connections(kind="tcp"): + if connection.status != psutil.CONN_LISTEN or connection.pid is None: + continue + address = _connection_address(connection) + if address == (LOOPBACK_ADDRESS, api_port): + listener_pids.add(int(connection.pid)) + return sorted(listener_pids) + + +def _collect_process_snapshots(listener_pids: Iterable[int]) -> list[dict[str, Any]]: + import psutil + + snapshots: list[dict[str, Any]] = [] + for pid in sorted({int(value) for value in listener_pids}): + try: + process = psutil.Process(pid) + argv = list(process.cmdline()) + cwd = process.cwd() + environment = process.environ() + except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess): + continue + snapshots.append( + { + "pid": pid, + "argv": argv, + "cwd": cwd, + # Never retain unrelated process secrets (including DATABASE_URL) + # in the snapshot or result payload. + "environment": { + name: environment.get(name) for name in RUNTIME_ENVIRONMENT_NAMES + }, + } + ) + return snapshots + + +def _failure(reason: str) -> tuple[int, dict[str, Any]]: + return 1, {"status": "failed", "reason": reason} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--expected-root", required=True) + parser.add_argument("--expected-api-cwd", required=True) + parser.add_argument("--expected-manifest-path", required=True) + parser.add_argument("--expected-manifest-sha256", required=True) + parser.add_argument("--expected-write-freeze-path", required=True) + parser.add_argument("--expected-database-target-sha256", required=True) + parser.add_argument("--api-port", type=int, required=True) + args = parser.parse_args() + + try: + listener_pids = _collect_listener_pids(args.api_port) + snapshots = _collect_process_snapshots(listener_pids) + # Close the process-exit/PID-reuse race: the same sole PID must still + # own the socket after its identity was inspected. + listener_pids_after = _collect_listener_pids(args.api_port) + if listener_pids_after != listener_pids: + exit_code, payload = _failure("api_listener_changed_during_probe") + else: + exit_code, payload = evaluate_listener_binding( + listener_pids, + snapshots, + expected_root=args.expected_root, + expected_api_cwd=args.expected_api_cwd, + expected_manifest_path=args.expected_manifest_path, + expected_manifest_sha256=args.expected_manifest_sha256, + expected_write_freeze_path=args.expected_write_freeze_path, + expected_database_target_sha256=(args.expected_database_target_sha256), + api_port=args.api_port, + ) + except Exception: + # psutil permission/platform failures are proof failures. Avoid + # printing exception strings because they can contain process details. + exit_code, payload = _failure("listener_inspection_failed") + + print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/public-runtime-task-definition-cutover.ps1 b/scripts/public-runtime-task-definition-cutover.ps1 new file mode 100644 index 0000000..c6e6d9a --- /dev/null +++ b/scripts/public-runtime-task-definition-cutover.ps1 @@ -0,0 +1,392 @@ +function Get-PublicRuntimeTaskXmlSha256 { + param([Parameter(Mandatory = $true)][string]$Xml) + + $bytes = [System.Text.UTF8Encoding]::new($false).GetBytes($Xml) + $hasher = [System.Security.Cryptography.SHA256]::Create() + try { + return ([BitConverter]::ToString($hasher.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $hasher.Dispose() + } +} + +function Get-PublicRuntimeTaskXmlContractSha256 { + param([Parameter(Mandatory = $true)][string]$Xml) + + try { + [xml]$document = $Xml + $enabledNodes = @( + $document.SelectNodes("//*[local-name()='Settings']/*[local-name()='Enabled']") + ) + if ($enabledNodes.Count -ne 1) { + throw "Task XML must contain exactly one settings Enabled node" + } + $enabledNodes[0].InnerText = "TASK_OPERATIONAL_STATE" + return Get-PublicRuntimeTaskXmlSha256 -Xml $document.OuterXml + } catch { + throw "Public runtime task XML contract normalization failed" + } +} + +function Get-PublicRuntimeTaskDefinitionSetSha256 { + param([Parameter(Mandatory = $true)][object[]]$Entries) + + $lines = @() + foreach ($entry in @($Entries | Sort-Object role)) { + if ( + [string]$entry.role -notmatch "^(boot|watchdog)$" -or + [string]$entry.xml_sha256 -notmatch "^[0-9a-f]{64}$" + ) { + throw "Public runtime task definition digest input is invalid" + } + $enabled = "false" + if ([bool]$entry.enabled) { + $enabled = "true" + } + $lines += ([string]$entry.role + ":" + [string]$entry.xml_sha256 + ":" + $enabled) + } + if ($lines.Count -ne 2) { + throw "Public runtime task definition set must contain exactly two roles" + } + return Get-PublicRuntimeTaskXmlSha256 -Xml (@($lines) -join "`n") +} + +function Get-ExactRootScheduledTaskForDefinition { + param([Parameter(Mandatory = $true)][string]$TaskName) + + if ( + [string]::IsNullOrWhiteSpace($TaskName) -or + $TaskName -match "[\\/\x00-\x1f\x7f]" + ) { + throw "Public runtime scheduled task name is invalid" + } + $matches = @( + Get-ScheduledTask ` + -TaskName $TaskName ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue + ) + if ($matches.Count -ne 1) { + throw "Public runtime root scheduled task is missing or ambiguous" + } + return $matches[0] +} + +function Assert-PublicRuntimeCoordinatedTaskNamesExact { + param([Parameter(Mandatory = $true)][string[]]$TaskNames) + + $actual = @($TaskNames | Sort-Object -Unique) + $expected = @("VignettePublicRuntime", "VignettePublicRuntimeWatchdog") + if ( + $actual.Count -ne 2 -or + -not ($actual -ccontains $expected[0]) -or + -not ($actual -ccontains $expected[1]) + ) { + throw "Public runtime task-definition cutover requires the exact two root tasks" + } +} + +function Get-PublicRuntimeTaskDefinitionSnapshot { + param( + [string]$BootTaskName = "VignettePublicRuntime", + [string]$WatchdogTaskName = "VignettePublicRuntimeWatchdog", + [switch]$RequireEnabled + ) + + if ([string]::Equals( + $BootTaskName, + $WatchdogTaskName, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Public runtime boot and watchdog task names must differ" + } + $entries = @() + foreach ($spec in @( + [pscustomobject][ordered]@{ role = "boot"; task_name = $BootTaskName }, + [pscustomobject][ordered]@{ role = "watchdog"; task_name = $WatchdogTaskName } + )) { + $task = Get-ExactRootScheduledTaskForDefinition -TaskName $spec.task_name + if ($RequireEnabled -and -not [bool]$task.Settings.Enabled) { + throw "Expected public runtime task is not enabled before bootstrap" + } + $actions = @($task.Actions) + if ($actions.Count -ne 1) { + throw "Public runtime task must have exactly one action" + } + $xml = [string](Export-ScheduledTask -InputObject $task -ErrorAction Stop) + if ([string]::IsNullOrWhiteSpace($xml)) { + throw "Public runtime scheduled task XML snapshot is empty" + } + $entries += [pscustomobject][ordered]@{ + role = [string]$spec.role + task_name = [string]$spec.task_name + xml_sha256 = Get-PublicRuntimeTaskXmlSha256 -Xml $xml + contract_sha256 = Get-PublicRuntimeTaskXmlContractSha256 -Xml $xml + enabled = [bool]$task.Settings.Enabled + action_execute = [string]$actions[0].Execute + action_arguments = [string]$actions[0].Arguments + action_working_directory = [string]$actions[0].WorkingDirectory + } + } + return [pscustomobject][ordered]@{ + entries = @($entries) + set_sha256 = Get-PublicRuntimeTaskDefinitionSetSha256 -Entries $entries + } +} + +function Assert-PublicRuntimeTaskDefinitionSnapshotCurrent { + param([Parameter(Mandatory = $true)][object]$ExpectedSnapshot) + + $boot = @($ExpectedSnapshot.entries | Where-Object role -eq "boot") + $watchdog = @($ExpectedSnapshot.entries | Where-Object role -eq "watchdog") + if ($boot.Count -ne 1 -or $watchdog.Count -ne 1) { + throw "Expected public runtime task definition snapshot is invalid" + } + $actual = Get-PublicRuntimeTaskDefinitionSnapshot ` + -BootTaskName ([string]$boot[0].task_name) ` + -WatchdogTaskName ([string]$watchdog[0].task_name) + if ([string]$actual.set_sha256 -cne [string]$ExpectedSnapshot.set_sha256) { + throw "Public runtime task definition set drifted" + } + return $actual +} + +function Get-PublicRuntimeExpectedTaskActionContracts { + param( + [Parameter(Mandatory = $true)][string]$StableSourceRoot, + [Parameter(Mandatory = $true)][string]$ExpectedSourceCommit, + [Parameter(Mandatory = $true)][string]$ExpectedSourceTree, + [Parameter(Mandatory = $true)][string]$PythonPath, + [Parameter(Mandatory = $true)][string]$UserUploadDir, + [Parameter(Mandatory = $true)][string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)][string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)][string]$UserUploadWriteFreezePath, + [Parameter(Mandatory = $true)][string]$CloudflaredPath, + [Parameter(Mandatory = $true)][string]$CloudflaredConfigPath, + [Parameter(Mandatory = $true)][string]$PublicHealthUrl + ) + + $root = (Resolve-Path -LiteralPath $StableSourceRoot).Path + $bootScript = (Resolve-Path -LiteralPath (Join-Path $root "scripts\boot-public-runtime.ps1")).Path + $watchScript = (Resolve-Path -LiteralPath (Join-Path $root "scripts\watch-public-runtime.ps1")).Path + $startScript = (Resolve-Path -LiteralPath (Join-Path $root "scripts\start-public-runtime.ps1")).Path + $watchLauncher = (Resolve-Path -LiteralPath (Join-Path $root "scripts\watch-public-runtime-task.vbs")).Path + $bootSha256 = (Get-FileHash -LiteralPath $bootScript -Algorithm SHA256).Hash.ToLowerInvariant() + $watchSha256 = (Get-FileHash -LiteralPath $watchScript -Algorithm SHA256).Hash.ToLowerInvariant() + $startSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant() + $resolvedPython = (Resolve-Path -LiteralPath $PythonPath).Path + $resolvedCloudflared = (Resolve-Path -LiteralPath $CloudflaredPath).Path + $resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfigPath).Path + $pythonSha256 = (Get-FileHash -LiteralPath $resolvedPython -Algorithm SHA256).Hash.ToLowerInvariant() + $cloudflaredSha256 = (Get-FileHash -LiteralPath $resolvedCloudflared -Algorithm SHA256).Hash.ToLowerInvariant() + $cloudflaredConfigSha256 = (Get-FileHash -LiteralPath $resolvedCloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant() + $windowsPowerShell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $wscript = Join-Path $env:SystemRoot "System32\wscript.exe" + + $bootArguments = @( + "-NoProfile", + "-ExecutionPolicy Bypass", + "-WindowStyle Hidden", + "-File `"$bootScript`"", + "-StableSourceRoot `"$root`"", + "-ExpectedSourceCommit $ExpectedSourceCommit", + "-ExpectedSourceTree $ExpectedSourceTree", + "-ExpectedBootScriptSha256 $bootSha256", + "-ExpectedStartScriptSha256 $startSha256", + "-ExpectedPythonSha256 $pythonSha256", + "-ExpectedCloudflaredSha256 $cloudflaredSha256", + "-ExpectedCloudflaredConfigSha256 $cloudflaredConfigSha256", + "-Python `"$resolvedPython`"", + "-Cloudflared `"$resolvedCloudflared`"", + "-CloudflaredConfig `"$resolvedCloudflaredConfig`"", + "-UserUploadDir `"$UserUploadDir`"", + "-UserUploadManifestPath `"$UserUploadManifestPath`"", + "-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256", + "-UserUploadWriteFreezePath `"$UserUploadWriteFreezePath`"" + ) -join " " + $watchArguments = @( + "-File `"$watchScript`"", + "-StableSourceRoot `"$root`"", + "-ExpectedSourceCommit $ExpectedSourceCommit", + "-ExpectedSourceTree $ExpectedSourceTree", + "-ExpectedWatchdogSha256 $watchSha256", + "-ExpectedStartScriptSha256 $startSha256", + "-ExpectedPythonSha256 $pythonSha256", + "-ExpectedCloudflaredSha256 $cloudflaredSha256", + "-ExpectedCloudflaredConfigSha256 $cloudflaredConfigSha256", + "-Python `"$resolvedPython`"", + "-Cloudflared `"$resolvedCloudflared`"", + "-CloudflaredConfig `"$resolvedCloudflaredConfig`"", + "-UserUploadDir `"$UserUploadDir`"", + "-UserUploadManifestPath `"$UserUploadManifestPath`"", + "-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256", + "-UserUploadWriteFreezePath `"$UserUploadWriteFreezePath`"", + "-PublicHealthUrl `"$PublicHealthUrl`"" + ) -join " " + return @( + [pscustomobject][ordered]@{ + role = "boot" + execute = $windowsPowerShell + arguments = $bootArguments + working_directory = $root + }, + [pscustomobject][ordered]@{ + role = "watchdog" + execute = $wscript + arguments = "`"$watchLauncher`" $watchArguments" + working_directory = $root + } + ) +} + +function Assert-NewPublicRuntimeTaskDefinitionsPinned { + param( + [Parameter(Mandatory = $true)][object]$Snapshot, + [Parameter(Mandatory = $true)][string]$StableSourceRoot, + [Parameter(Mandatory = $true)][string]$ExpectedSourceCommit, + [Parameter(Mandatory = $true)][string]$ExpectedSourceTree, + [Parameter(Mandatory = $true)][string]$PythonPath, + [Parameter(Mandatory = $true)][string]$UserUploadDir, + [Parameter(Mandatory = $true)][string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)][string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)][string]$UserUploadWriteFreezePath, + [Parameter(Mandatory = $true)][string]$CloudflaredPath, + [Parameter(Mandatory = $true)][string]$CloudflaredConfigPath, + [Parameter(Mandatory = $true)][string]$PublicHealthUrl, + [switch]$AllowEnabled + ) + + if (@($Snapshot.entries).Count -ne 2) { + throw "New public runtime task definition set is incomplete" + } + $contracts = @( + Get-PublicRuntimeExpectedTaskActionContracts ` + -StableSourceRoot $StableSourceRoot ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -PythonPath $PythonPath ` + -UserUploadDir $UserUploadDir ` + -UserUploadManifestPath $UserUploadManifestPath ` + -ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 ` + -UserUploadWriteFreezePath $UserUploadWriteFreezePath ` + -CloudflaredPath $CloudflaredPath ` + -CloudflaredConfigPath $CloudflaredConfigPath ` + -PublicHealthUrl $PublicHealthUrl + ) + foreach ($entry in @($Snapshot.entries)) { + if (-not $AllowEnabled -and [bool]$entry.enabled) { + throw "New public runtime task definition must remain disabled until operational receipt" + } + $contract = @($contracts | Where-Object role -eq ([string]$entry.role)) + if ( + $contract.Count -ne 1 -or + -not [string]::Equals( + [string]$entry.action_execute, + [string]$contract[0].execute, + [System.StringComparison]::OrdinalIgnoreCase + ) -or + [string]$entry.action_arguments -cne [string]$contract[0].arguments -or + -not [string]::Equals( + [string]$entry.action_working_directory, + [string]$contract[0].working_directory, + [System.StringComparison]::OrdinalIgnoreCase + ) + ) { + throw "New public runtime task action contract drift" + } + } + return $Snapshot +} + +function Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled { + param( + [Parameter(Mandatory = $true)][scriptblock]$BootInstaller, + [Parameter(Mandatory = $true)][scriptblock]$WatchdogInstaller, + [Parameter(Mandatory = $true)][object[]]$MaintenanceSnapshot, + [int]$TimeoutSec = 30 + ) + + try { + $null = & $BootInstaller + $null = & $WatchdogInstaller + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $MaintenanceSnapshot ` + -TimeoutSec $TimeoutSec + } catch { + Suspend-PublicRuntimeTasks ` + -Snapshot $MaintenanceSnapshot ` + -TimeoutSec $TimeoutSec + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $MaintenanceSnapshot ` + -TimeoutSec $TimeoutSec + throw + } +} + +function Enable-NewPublicRuntimeTaskDefinitions { + param([Parameter(Mandatory = $true)][object]$DisabledSnapshot) + + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $DisabledSnapshot + $boot = @($DisabledSnapshot.entries | Where-Object role -eq "boot") + $watchdog = @($DisabledSnapshot.entries | Where-Object role -eq "watchdog") + if ($boot.Count -ne 1 -or $watchdog.Count -ne 1) { + throw "Disabled public runtime task definition snapshot is invalid" + } + try { + foreach ($entry in @($DisabledSnapshot.entries)) { + Enable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop ` + | Out-Null + } + } catch { + foreach ($entry in @($DisabledSnapshot.entries)) { + Disable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue ` + | Out-Null + } + $failedTruth = Get-PublicRuntimeTaskDefinitionSnapshot ` + -BootTaskName ([string]$boot[0].task_name) ` + -WatchdogTaskName ([string]$watchdog[0].task_name) + if (@($failedTruth.entries | Where-Object enabled).Count -ne 0) { + throw "Public runtime task enable failed and compensation could not disable both tasks" + } + throw + } + $operational = Get-PublicRuntimeTaskDefinitionSnapshot ` + -BootTaskName ([string]$boot[0].task_name) ` + -WatchdogTaskName ([string]$watchdog[0].task_name) ` + -RequireEnabled + foreach ($prior in @($DisabledSnapshot.entries)) { + $current = @($operational.entries | Where-Object role -eq ([string]$prior.role)) + if ( + $current.Count -ne 1 -or + [string]$current[0].contract_sha256 -cne [string]$prior.contract_sha256 -or + [string]$current[0].action_arguments -cne [string]$prior.action_arguments -or + -not [string]::Equals( + [string]$current[0].action_execute, + [string]$prior.action_execute, + [System.StringComparison]::OrdinalIgnoreCase + ) -or + -not [string]::Equals( + [string]$current[0].action_working_directory, + [string]$prior.action_working_directory, + [System.StringComparison]::OrdinalIgnoreCase + ) + ) { + foreach ($entry in @($DisabledSnapshot.entries)) { + Disable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue ` + | Out-Null + } + throw "Public runtime task definition drifted while becoming operational" + } + } + return $operational +} diff --git a/scripts/public-runtime-task-maintenance.ps1 b/scripts/public-runtime-task-maintenance.ps1 new file mode 100644 index 0000000..b964c42 --- /dev/null +++ b/scripts/public-runtime-task-maintenance.ps1 @@ -0,0 +1,220 @@ +$taskDefinitionCutoverContract = Join-Path $PSScriptRoot "public-runtime-task-definition-cutover.ps1" +if (-not (Test-Path -LiteralPath $taskDefinitionCutoverContract -PathType Leaf)) { + throw "Public runtime task definition cutover contract is unavailable" +} +. $taskDefinitionCutoverContract + +function Assert-PublicRuntimeTasksDisabledAndIdle { + param( + [Parameter(Mandatory = $true)][object[]]$Snapshot, + [int]$TimeoutSec = 30 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $allDisabledAndIdle = $true + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists) { + $unexpectedTask = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue + if ($null -ne $unexpectedTask) { + throw "A coordinated public runtime task appeared during maintenance: $($entry.task_name)" + } + continue + } + $task = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop + if ([bool]$task.Settings.Enabled -or [string]$task.State -eq "Running") { + $allDisabledAndIdle = $false + break + } + } + if ($allDisabledAndIdle) { + return + } + Start-Sleep -Milliseconds 250 + } while ((Get-Date) -lt $deadline) + + throw "Public runtime scheduled tasks did not become disabled and idle" +} + +function Get-PublicRuntimeTaskMaintenanceState { + param([Parameter(Mandatory = $true)][object[]]$Snapshot) + + $allDisabledAndIdle = $true + $restoredToSnapshot = $true + $verified = $true + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists) { + try { + $unexpectedTask = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue + if ($null -ne $unexpectedTask) { + $allDisabledAndIdle = $false + $restoredToSnapshot = $false + } + } catch { + $verified = $false + $allDisabledAndIdle = $false + $restoredToSnapshot = $false + } + continue + } + try { + $task = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop + $isEnabled = [bool]$task.Settings.Enabled + $isRunning = [string]$task.State -eq "Running" + if ($isEnabled -or $isRunning) { + $allDisabledAndIdle = $false + } + if ($isEnabled -ne [bool]$entry.was_enabled) { + $restoredToSnapshot = $false + } + } catch { + $verified = $false + $allDisabledAndIdle = $false + $restoredToSnapshot = $false + } + } + + return [pscustomobject][ordered]@{ + verified = $verified + all_disabled_and_idle = $allDisabledAndIdle + restored_to_snapshot = $restoredToSnapshot + } +} + +function Suspend-PublicRuntimeTasks { + param( + [Parameter(Mandatory = $true)][object[]]$Snapshot, + [int]$TimeoutSec = 30 + ) + + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists) { + continue + } + Disable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop ` + | Out-Null + } + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $Snapshot ` + -TimeoutSec $TimeoutSec +} + +function Enter-PublicRuntimeTaskMaintenance { + param( + [Parameter(Mandatory = $true)][string[]]$TaskNames, + [int]$TimeoutSec = 30 + ) + + $snapshot = @() + try { + # 모든 원상복구 정보를 먼저 캡처한다. disable 도중 두 번째 task에서 오류가 나도 + # 첫 번째 task를 포함한 완전한 snapshot으로 되돌릴 수 있어야 한다. + foreach ($taskName in @($TaskNames | Sort-Object -Unique)) { + if ([string]::IsNullOrWhiteSpace($taskName)) { + throw "Public runtime scheduled task name is empty" + } + $task = Get-ScheduledTask ` + -TaskName $taskName ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue + if ($null -eq $task) { + $snapshot += [pscustomobject][ordered]@{ + task_name = $taskName + exists = $false + was_enabled = $false + } + continue + } + $wasEnabled = [bool]$task.Settings.Enabled + $snapshot += [pscustomobject][ordered]@{ + task_name = $taskName + exists = $true + was_enabled = $wasEnabled + } + } + + $result = @($snapshot) + Suspend-PublicRuntimeTasks ` + -Snapshot $result ` + -TimeoutSec $TimeoutSec + return $result + } catch { + foreach ($entry in @($snapshot)) { + if ([bool]$entry.exists -and [bool]$entry.was_enabled) { + Enable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction SilentlyContinue ` + | Out-Null + } + } + foreach ($entry in @($snapshot)) { + if (-not [bool]$entry.exists) { + continue + } + $task = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop + if ([bool]$task.Settings.Enabled -ne [bool]$entry.was_enabled) { + throw "Public runtime task maintenance entry rollback failed: $($entry.task_name)" + } + } + throw + } +} + +function Exit-PublicRuntimeTaskMaintenance { + param([Parameter(Mandatory = $true)][object[]]$Snapshot) + + try { + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists -or -not [bool]$entry.was_enabled) { + continue + } + Enable-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop ` + | Out-Null + } + + foreach ($entry in @($Snapshot)) { + if (-not [bool]$entry.exists -or -not [bool]$entry.was_enabled) { + continue + } + $task = Get-ScheduledTask ` + -TaskName ([string]$entry.task_name) ` + -TaskPath "\" ` + -ErrorAction Stop + if (-not [bool]$task.Settings.Enabled) { + throw "Public runtime scheduled task was not re-enabled: $($entry.task_name)" + } + } + $restoredTruth = Get-PublicRuntimeTaskMaintenanceState -Snapshot $Snapshot + if ( + -not [bool]$restoredTruth.verified -or + -not [bool]$restoredTruth.restored_to_snapshot + ) { + throw "Public runtime scheduled task set drifted during maintenance" + } + } catch { + Suspend-PublicRuntimeTasks -Snapshot $Snapshot -TimeoutSec 30 + throw + } +} diff --git a/scripts/public-runtime-upload-root.ps1 b/scripts/public-runtime-upload-root.ps1 new file mode 100644 index 0000000..1dfe5c8 --- /dev/null +++ b/scripts/public-runtime-upload-root.ps1 @@ -0,0 +1,382 @@ +# Public runtime의 사용자 업로드 저장소 계약 SSOT. +# 이 파일은 함수만 선언하며 dot-source 시 외부 상태를 변경하지 않는다. + +function Get-PublicRuntimeCanonicalPath { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + $fullPath = [System.IO.Path]::GetFullPath($Path) + $pathRoot = [System.IO.Path]::GetPathRoot($fullPath) + if ([string]::IsNullOrWhiteSpace($pathRoot)) { + throw "Path does not have a filesystem root" + } + $comparisonPath = $fullPath.TrimEnd('\', '/') + $comparisonRoot = $pathRoot.TrimEnd('\', '/') + if ([string]::Equals( + $comparisonPath, + $comparisonRoot, + [System.StringComparison]::OrdinalIgnoreCase + )) { + return $pathRoot + } + return $comparisonPath +} + +function Test-PublicRuntimePathIsSameOrChild { + param( + [Parameter(Mandatory = $true)] + [string]$Candidate, + [Parameter(Mandatory = $true)] + [string]$Parent + ) + + $candidateFull = Get-PublicRuntimeCanonicalPath -Path $Candidate + $parentFull = Get-PublicRuntimeCanonicalPath -Path $Parent + if ([string]::Equals( + $candidateFull, + $parentFull, + [System.StringComparison]::OrdinalIgnoreCase + )) { + return $true + } + $prefix = $parentFull + if (-not $prefix.EndsWith([System.IO.Path]::DirectorySeparatorChar)) { + $prefix += [System.IO.Path]::DirectorySeparatorChar + } + return $candidateFull.StartsWith( + $prefix, + [System.StringComparison]::OrdinalIgnoreCase + ) +} + +function Assert-PublicRuntimePathHasNoReparsePoint { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + $cursor = Get-PublicRuntimeCanonicalPath -Path $Path + while (-not [string]::IsNullOrWhiteSpace($cursor)) { + if (Test-Path -LiteralPath $cursor) { + $item = Get-Item -LiteralPath $cursor -Force + if (($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "USER_UPLOAD_DIR cannot traverse a symlink or reparse point: $cursor" + } + } + $parent = [System.IO.Directory]::GetParent($cursor) + if ($null -eq $parent) { + break + } + $cursor = $parent.FullName + } +} + +function Resolve-PublicRuntimeUploadRoot { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SourceRoot, + [Parameter(Mandatory = $true)] + [string]$UploadRoot, + [switch]$CreateIfMissing, + [switch]$ProbeWritable + ) + + if ([string]::IsNullOrWhiteSpace($UploadRoot)) { + throw "USER_UPLOAD_DIR is empty" + } + $driveAbsolute = $UploadRoot -match '^[A-Za-z]:[\\/]' + $uncAbsolute = $UploadRoot -match '^\\\\[^\\/]+[\\/][^\\/]+(?:[\\/]|$)' + if (-not $driveAbsolute -and -not $uncAbsolute) { + throw "USER_UPLOAD_DIR must be a fully qualified absolute Windows path" + } + if (-not (Test-Path -LiteralPath $SourceRoot -PathType Container)) { + throw "Public runtime source root is unavailable: $SourceRoot" + } + + $resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path + $gitRootOutput = @(& git.exe -C $resolvedSourceRoot rev-parse --show-toplevel) + if ($LASTEXITCODE -ne 0 -or $gitRootOutput.Count -eq 0) { + throw "Could not resolve the public runtime Git root" + } + $gitRoot = (Resolve-Path -LiteralPath ((@($gitRootOutput) -join [Environment]::NewLine).Trim())).Path + if (-not [string]::Equals( + $resolvedSourceRoot, + $gitRoot, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Public runtime source root must match its Git toplevel" + } + + $fullUploadPath = [System.IO.Path]::GetFullPath($UploadRoot) + $filesystemRoot = [System.IO.Path]::GetPathRoot($fullUploadPath) + if ([string]::IsNullOrWhiteSpace($filesystemRoot)) { + throw "USER_UPLOAD_DIR does not have a filesystem root" + } + if ([string]::Equals( + $fullUploadPath.TrimEnd('\', '/'), + $filesystemRoot.TrimEnd('\', '/'), + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "USER_UPLOAD_DIR cannot be a filesystem or UNC share root" + } + $fullUploadRoot = Get-PublicRuntimeCanonicalPath -Path $fullUploadPath + $uploadParent = [System.IO.Directory]::GetParent($fullUploadRoot) + if ($null -eq $uploadParent) { + throw "USER_UPLOAD_DIR cannot be a filesystem root" + } + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $fullUploadRoot -Parent $gitRoot) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $gitRoot -Parent $fullUploadRoot) + ) { + throw "USER_UPLOAD_DIR must be disjoint from the public runtime Git root" + } + + # 기존 ancestor와 생성 후 최종 경로를 모두 검사해 junction/symlink를 통한 + # source-tree 또는 다른 저장소로의 우회를 차단한다. + Assert-PublicRuntimePathHasNoReparsePoint -Path $fullUploadRoot + if (Test-Path -LiteralPath $fullUploadRoot) { + if (-not (Test-Path -LiteralPath $fullUploadRoot -PathType Container)) { + throw "USER_UPLOAD_DIR is not a directory: $fullUploadRoot" + } + } elseif ($CreateIfMissing) { + [System.IO.Directory]::CreateDirectory($fullUploadRoot) | Out-Null + } else { + throw "USER_UPLOAD_DIR does not exist: $fullUploadRoot" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $fullUploadRoot + $resolvedUploadRoot = (Resolve-Path -LiteralPath $fullUploadRoot).Path + + if ($ProbeWritable) { + $probePath = Join-Path $resolvedUploadRoot ( + ".vignette-write-probe-{0}.tmp" -f [Guid]::NewGuid().ToString("N") + ) + $stream = $null + try { + $stream = [System.IO.File]::Open( + $probePath, + [System.IO.FileMode]::CreateNew, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None + ) + $stream.WriteByte(0) + $stream.Flush($true) + } catch { + throw "USER_UPLOAD_DIR is not writable: $resolvedUploadRoot ($($_.Exception.Message))" + } finally { + if ($null -ne $stream) { + $stream.Dispose() + } + if ([System.IO.File]::Exists($probePath)) { + [System.IO.File]::Delete($probePath) + } + } + } + + return $resolvedUploadRoot +} + +function Resolve-PublicRuntimePrivateStatePath { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SourceRoot, + [Parameter(Mandatory = $true)] + [string]$UploadRoot, + [Parameter(Mandatory = $true)] + [string]$StatePath, + [switch]$RequireFile + ) + + if ([string]::IsNullOrWhiteSpace($StatePath) -or -not [System.IO.Path]::IsPathRooted($StatePath)) { + throw "Public upload private state path must be absolute" + } + $fullStatePath = Get-PublicRuntimeCanonicalPath -Path $StatePath + $stateRoot = [System.IO.Path]::GetPathRoot($fullStatePath) + if ([string]::Equals( + $fullStatePath.TrimEnd('\', '/'), + $stateRoot.TrimEnd('\', '/'), + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Public upload private state path cannot be a filesystem root" + } + foreach ($publicBoundary in @($SourceRoot, $UploadRoot)) { + if ( + (Test-PublicRuntimePathIsSameOrChild -Candidate $fullStatePath -Parent $publicBoundary) -or + (Test-PublicRuntimePathIsSameOrChild -Candidate $publicBoundary -Parent $fullStatePath) + ) { + throw "Public upload manifest/freeze state must be disjoint from source and public upload roots" + } + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $fullStatePath + if ($RequireFile) { + if (-not (Test-Path -LiteralPath $fullStatePath -PathType Leaf)) { + throw "Public upload private state file is missing: $fullStatePath" + } + return (Resolve-Path -LiteralPath $fullStatePath).Path + } + $parent = [System.IO.Path]::GetDirectoryName($fullStatePath) + if ([string]::IsNullOrWhiteSpace($parent) -or -not (Test-Path -LiteralPath $parent -PathType Container)) { + throw "Public upload private state parent directory is missing: $parent" + } + Assert-PublicRuntimePathHasNoReparsePoint -Path $parent + return $fullStatePath +} + +function Test-PublicRuntimeUploadManifest { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$PythonPath, + [Parameter(Mandatory = $true)] + [string]$ProbePath, + [Parameter(Mandatory = $true)] + [string]$UploadRoot, + [Parameter(Mandatory = $true)] + [string]$ManifestPath, + [Parameter(Mandatory = $true)] + [string]$ExpectedManifestSha256, + [Parameter(Mandatory = $true)] + [string]$ExpectedWriteFreezePath + ) + + foreach ($requiredFile in @($PythonPath, $ProbePath, $ManifestPath)) { + if (-not (Test-Path -LiteralPath $requiredFile -PathType Leaf)) { + return [pscustomobject]@{ + Name = "api-upload-manifest" + Ok = $false + Detail = "upload manifest prerequisite missing" + Payload = $null + } + } + } + $probeArgs = @( + "-X", "utf8", "-B", $ProbePath, + "--upload-root", $UploadRoot, + "--manifest-path", $ManifestPath, + "--expected-manifest-sha256", $ExpectedManifestSha256, + "--expected-write-freeze-path", $ExpectedWriteFreezePath + ) + $previousErrorActionPreference = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + $output = @(& $PythonPath @probeArgs 2>$null) + $probeExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + $detail = (@($output) -join "").Trim() + $payload = $null + if (-not [string]::IsNullOrWhiteSpace($detail)) { + try { + $payload = $detail | ConvertFrom-Json + } catch { + $payload = $null + } + } + if ($null -eq $payload) { + $detail = "upload manifest probe failed without valid JSON" + } + return [pscustomobject]@{ + Name = "api-upload-manifest" + Ok = $probeExit -eq 0 -and $null -ne $payload -and $payload.status -eq "passed" + Detail = $detail + Payload = $payload + } +} + +function Test-PublicRuntimeApiUploadRoot { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$PythonPath, + [Parameter(Mandatory = $true)] + [string]$ProbePath, + [Parameter(Mandatory = $true)] + [string]$ExpectedUploadRoot, + [Parameter(Mandatory = $true)] + [string]$ExpectedApiCwd, + [Parameter(Mandatory = $true)] + [string]$ExpectedManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedManifestSha256, + [Parameter(Mandatory = $true)] + [string]$ExpectedWriteFreezePath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedDatabaseTargetSha256, + [Parameter(Mandatory = $true)] + [int]$ApiPort + ) + + foreach ($requiredFile in @($PythonPath, $ProbePath)) { + if (-not (Test-Path -LiteralPath $requiredFile -PathType Leaf)) { + return [pscustomobject]@{ + Name = "api-upload-root" + Ok = $false + Detail = "upload root process probe prerequisite missing: $requiredFile" + Payload = $null + ListenerPid = $null + } + } + } + + $probeArgs = @( + "-X", "utf8", "-B", $ProbePath, + "--expected-root", $ExpectedUploadRoot, + "--expected-api-cwd", $ExpectedApiCwd, + "--expected-manifest-path", $ExpectedManifestPath, + "--expected-manifest-sha256", $ExpectedManifestSha256, + "--expected-write-freeze-path", $ExpectedWriteFreezePath, + "--expected-database-target-sha256", $ExpectedDatabaseTargetSha256, + "--api-port", $ApiPort.ToString() + ) + $previousErrorActionPreference = $ErrorActionPreference + try { + # Windows PowerShell 5.1이 native stderr를 ErrorRecord로 승격하지 않게 이 + # secret-free probe 경계에서만 Continue로 낮춘다. + $ErrorActionPreference = "Continue" + $output = @(& $PythonPath @probeArgs 2>$null) + $probeExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + $detail = (@($output) -join "").Trim() + if ([string]::IsNullOrWhiteSpace($detail)) { + $detail = "upload root process probe failed without output" + } + $payload = $null + if (-not [string]::IsNullOrWhiteSpace($detail)) { + try { + $payload = $detail | ConvertFrom-Json + } catch { + $payload = $null + } + } + $listenerPid = $null + if ($null -ne $payload -and $payload.status -eq "passed") { + try { + $candidatePid = [int]$payload.pid + if ($candidatePid -gt 0) { + $listenerPid = $candidatePid + } + } catch { + $listenerPid = $null + } + } + return [pscustomobject]@{ + Name = "api-upload-root" + Ok = ( + $probeExit -eq 0 -and + $null -ne $payload -and + $payload.status -eq "passed" -and + $null -ne $listenerPid + ) + Detail = $detail + Payload = $payload + ListenerPid = $listenerPid + } +} diff --git a/scripts/public_runtime_database_identity.py b/scripts/public_runtime_database_identity.py new file mode 100644 index 0000000..f026199 --- /dev/null +++ b/scripts/public_runtime_database_identity.py @@ -0,0 +1,74 @@ +"""Credential-free identity for the *connected* public-runtime PostgreSQL target. + +The connection itself is the authority. A configured DSN is deliberately not +hashed because proxies, aliases, defaults, and credential changes can make a +URL identity disagree with the server that actually owns the inventory. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + + +DATABASE_IDENTITY_QUERY = """ +SELECT + current_database()::text AS database_name, + current_user::text AS database_role, + COALESCE(inet_server_addr()::text, 'local') AS server_address, + inet_server_port() AS server_port +""" + + +def database_target_sha256( + *, + database_name: str, + database_role: str, + server_address: str, + server_port: int, +) -> str: + """Return a lowercase SHA256 over a secret-free connected DB identity.""" + + if ( + not isinstance(database_name, str) + or not database_name.strip() + or not isinstance(database_role, str) + or not database_role.strip() + or not isinstance(server_address, str) + or not server_address.strip() + ): + raise ValueError("database target identity is incomplete") + if ( + not isinstance(server_port, int) + or isinstance(server_port, bool) + or server_port < 1 + or server_port > 65535 + ): + raise ValueError("database target port is invalid") + canonical = json.dumps( + { + "database": database_name, + "database_role": database_role, + "server_address": server_address, + "server_port": server_port, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8", errors="strict") + return hashlib.sha256(canonical).hexdigest() + + +async def connected_database_target_sha256(connection: Any) -> str: + """Read and hash the target identity through an already-open connection.""" + + target = await connection.fetchrow(DATABASE_IDENTITY_QUERY) + if target is None: + raise ValueError("database target identity is unavailable") + return database_target_sha256( + database_name=str(target["database_name"]), + database_role=str(target["database_role"]), + server_address=str(target["server_address"]), + server_port=int(target["server_port"]), + ) diff --git a/scripts/register-boot-task.ps1 b/scripts/register-boot-task.ps1 index ed9168f..2e96b07 100644 --- a/scripts/register-boot-task.ps1 +++ b/scripts/register-boot-task.ps1 @@ -10,7 +10,20 @@ param( [Parameter(Mandatory = $true)] [string]$StableSourceRoot, - [string]$TaskName = "VignettePublicRuntime" + [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", + [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", + [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, + [string]$TaskName = "VignettePublicRuntime", + [switch]$InitiallyDisabled ) $ErrorActionPreference = "Stop" @@ -20,6 +33,10 @@ $RegisterScript = Join-Path $resolvedSourceRoot "scripts\register-boot-task.ps1" $BootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $StartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $VoiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" +$UploadRootContract = Join-Path $resolvedSourceRoot "scripts\public-runtime-upload-root.ps1" +$UploadRootProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-runtime-upload-root.py" +$UploadManifestProbe = Join-Path $resolvedSourceRoot "scripts\validate-public-runtime-upload-manifest.py" +$DatabaseIdentityHelper = Join-Path $resolvedSourceRoot "scripts\public_runtime_database_identity.py" $UserName = "$env:USERDOMAIN\$env:USERNAME" function Invoke-GitText { @@ -32,7 +49,16 @@ function Invoke-GitText { return (@($value) -join [Environment]::NewLine).Trim() } -foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript, $VoiceSidecarProbe)) { +foreach ($requiredScript in @( + $RegisterScript, + $BootScript, + $StartScript, + $VoiceSidecarProbe, + $UploadRootContract, + $UploadRootProbe, + $UploadManifestProbe, + $DatabaseIdentityHelper +)) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Public runtime script not found: $requiredScript" } @@ -74,7 +100,13 @@ foreach ($relativePath in @( "scripts/register-boot-task.ps1", "scripts/boot-public-runtime.ps1", "scripts/start-public-runtime.ps1", - "scripts/probe-public-voice-sidecars.py" + "scripts/probe-public-voice-sidecars.py", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -83,6 +115,36 @@ $sourceCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD") $sourceTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}") $bootScriptSha256 = (Get-FileHash -LiteralPath $BootScript -Algorithm SHA256).Hash.ToLowerInvariant() $startScriptSha256 = (Get-FileHash -LiteralPath $StartScript -Algorithm SHA256).Hash.ToLowerInvariant() +$resolvedPython = (Resolve-Path -LiteralPath $Python).Path +$resolvedCloudflared = (Resolve-Path -LiteralPath $Cloudflared).Path +$resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfig).Path +$pythonSha256 = (Get-FileHash -LiteralPath $resolvedPython -Algorithm SHA256).Hash.ToLowerInvariant() +$cloudflaredSha256 = (Get-FileHash -LiteralPath $resolvedCloudflared -Algorithm SHA256).Hash.ToLowerInvariant() +$cloudflaredConfigSha256 = (Get-FileHash -LiteralPath $resolvedCloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant() +. $UploadRootContract +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $UserUploadDir ` + -ProbeWritable +$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadManifestPath ` + -RequireFile +$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadWriteFreezePath +$uploadManifestProof = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $UploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +if (-not $uploadManifestProof.Ok) { + throw "Refusing task registration: public upload migration receipt or current DB inventory is invalid" +} $bootArguments = @( "-NoProfile", @@ -93,11 +155,25 @@ $bootArguments = @( "-ExpectedSourceCommit $sourceCommit", "-ExpectedSourceTree $sourceTree", "-ExpectedBootScriptSha256 $bootScriptSha256", - "-ExpectedStartScriptSha256 $startScriptSha256" + "-ExpectedStartScriptSha256 $startScriptSha256", + "-ExpectedPythonSha256 $pythonSha256", + "-ExpectedCloudflaredSha256 $cloudflaredSha256", + "-ExpectedCloudflaredConfigSha256 $cloudflaredConfigSha256", + "-Python `"$resolvedPython`"", + "-Cloudflared `"$resolvedCloudflared`"", + "-CloudflaredConfig `"$resolvedCloudflaredConfig`"", + "-UserUploadDir `"$resolvedUserUploadDir`"", + "-UserUploadManifestPath `"$resolvedUserUploadManifestPath`"", + "-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256", + "-UserUploadWriteFreezePath `"$resolvedUserUploadWriteFreezePath`"" ) +$windowsPowerShell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" +if (-not (Test-Path -LiteralPath $windowsPowerShell -PathType Leaf)) { + throw "Windows PowerShell 5.1 executable is unavailable" +} $action = New-ScheduledTaskAction ` - -Execute "powershell.exe" ` + -Execute $windowsPowerShell ` -Argument ($bootArguments -join " ") ` -WorkingDirectory $resolvedSourceRoot @@ -111,6 +187,7 @@ $principal = New-ScheduledTaskPrincipal ` -RunLevel Limited $settings = New-ScheduledTaskSettingsSet ` + -Disable:$InitiallyDisabled ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` -MultipleInstances IgnoreNew ` @@ -119,6 +196,7 @@ $settings = New-ScheduledTaskSettingsSet ` Register-ScheduledTask ` -TaskName $TaskName ` + -TaskPath "\" ` -Action $action ` -Trigger $trigger ` -Principal $principal ` @@ -126,7 +204,14 @@ Register-ScheduledTask ` -Description "Vignette public runtime auto-recovery from detached clean commit $sourceCommit at logon" ` -Force | Out-Null -$task = Get-ScheduledTask -TaskName $TaskName +if ($InitiallyDisabled) { + $registered = Get-ScheduledTask -TaskName $TaskName -TaskPath "\" -ErrorAction Stop + if ([bool]$registered.Settings.Enabled -or [string]$registered.State -eq "Running") { + throw "New boot task did not remain disabled and idle" + } +} + +$task = Get-ScheduledTask -TaskName $TaskName -TaskPath "\" Write-Output ("Registered : " + $TaskName) Write-Output ("State : " + $task.State) Write-Output ("User : " + $principal.UserId) @@ -134,6 +219,7 @@ Write-Output ("Trigger : AtLogOn (" + $UserName + ")") Write-Output ("Command : " + $action.Execute + " " + $action.Argument) Write-Output ("Source : root=" + $resolvedSourceRoot + " commit=" + $sourceCommit + " tree=" + $sourceTree) Write-Output ("Hashes : boot=" + $bootScriptSha256 + " start=" + $startScriptSha256) +Write-Output ("Uploads : " + $resolvedUserUploadDir) Write-Output "" Write-Output ("Run now : " + $action.Execute + " " + $action.Argument) Write-Output ("Unregister : Unregister-ScheduledTask -TaskName " + $TaskName + " -Confirm:`$false") diff --git a/scripts/start-public-runtime.ps1 b/scripts/start-public-runtime.ps1 index 1c31566..db46c30 100644 --- a/scripts/start-public-runtime.ps1 +++ b/scripts/start-public-runtime.ps1 @@ -18,6 +18,15 @@ [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, [switch]$SkipEngineRestart, [switch]$ForceApiRestart, [switch]$SkipWebRestart, @@ -33,11 +42,360 @@ [string]$ExpectedCloudflaredConfigSha256 = "", [string]$RuntimeProvenancePath = "", [ValidateRange(1, 60)] - [int]$ProcessStopTimeoutSeconds = 15 + [int]$ProcessStopTimeoutSeconds = 15, + [string[]]$CoordinatedTaskNames = @( + "VignettePublicRuntimeWatchdog", + "VignettePublicRuntime" + ), + [string]$OfflineBootstrapQuiescenceReceiptPath = "", + [ValidatePattern("^$|^[0-9a-f]{64}$")] + [string]$ExpectedOfflineBootstrapQuiescenceReceiptSha256 = "", + [ValidatePattern("^$|^[0-9a-f]{40}$")] + [string]$ExpectedOfflineBootstrapLegacySourceCommit = "", + [ValidatePattern("^$|^[0-9a-f]{40}$")] + [string]$ExpectedOfflineBootstrapLegacySourceTree = "", + [string]$InheritedRecoveryLockReceiptPath = "", + [ValidatePattern("^$|^[0-9a-f]{64}$")] + [string]$ExpectedInheritedRecoveryLockReceiptSha256 = "" ) $ErrorActionPreference = "Stop" +function Repair-CaseInsensitiveProcessEnvironment { + # Windows 환경 변수 이름은 대소문자를 구분하지 않지만, 비-Windows 부모가 만든 + # 프로세스 블록에는 Path/PATH 같은 충돌 키가 함께 들어올 수 있다. Windows + # PowerShell 5.1의 Start-Process는 이 블록을 case-insensitive 사전으로 옮기다가 + # 충돌하므로, 실행 전 .NET의 Windows 환경 갱신 경로를 한 번 거쳐 정규화한다. + # .NET Framework의 GetEnvironmentVariables()는 충돌 키를 이미 하나로 접어서 + # 보여주므로 개수를 세어서는 원본 환경 블록의 중복을 발견할 수 없다. 현재 + # 프로세스가 실제로 해석하는 값을 먼저 보존하고 두 대표 casing을 각각 지운 뒤 + # canonical `Path` 하나만 다시 만든다. + $effectivePath = [System.Environment]::GetEnvironmentVariable( + "Path", + [System.EnvironmentVariableTarget]::Process + ) + if ([string]::IsNullOrWhiteSpace($effectivePath)) { + throw "Process Path environment is empty" + } + [System.Environment]::SetEnvironmentVariable( + "Path", + $null, + [System.EnvironmentVariableTarget]::Process + ) + [System.Environment]::SetEnvironmentVariable( + "PATH", + $null, + [System.EnvironmentVariableTarget]::Process + ) + [System.Environment]::SetEnvironmentVariable( + "Path", + $effectivePath, + [System.EnvironmentVariableTarget]::Process + ) +} + +Repair-CaseInsensitiveProcessEnvironment + +$resolvedWorkspace = (Resolve-Path -LiteralPath $Workspace).Path +$uploadRootContract = Join-Path $resolvedWorkspace "scripts\public-runtime-upload-root.ps1" +$uploadRootProbe = Join-Path $resolvedWorkspace "scripts\probe-public-runtime-upload-root.py" +$uploadManifestProbe = Join-Path $resolvedWorkspace "scripts\validate-public-runtime-upload-manifest.py" +$databaseIdentityHelper = Join-Path $resolvedWorkspace "scripts\public_runtime_database_identity.py" +$offlineQuiescenceProbe = Join-Path $resolvedWorkspace "scripts\validate-public-runtime-offline-quiescence.py" +$taskMaintenanceContract = Join-Path $resolvedWorkspace "scripts\public-runtime-task-maintenance.ps1" +$taskDefinitionCutoverContract = Join-Path $resolvedWorkspace "scripts\public-runtime-task-definition-cutover.ps1" +$bootTaskInstaller = Join-Path $resolvedWorkspace "scripts\register-boot-task.ps1" +$watchdogTaskInstaller = Join-Path $resolvedWorkspace "scripts\install-public-runtime-task.ps1" +foreach ($uploadContractFile in @( + $uploadRootContract, + $uploadRootProbe, + $uploadManifestProbe, + $databaseIdentityHelper, + $offlineQuiescenceProbe, + $taskMaintenanceContract, + $taskDefinitionCutoverContract, + $bootTaskInstaller, + $watchdogTaskInstaller +)) { + if (-not (Test-Path -LiteralPath $uploadContractFile -PathType Leaf)) { + throw "Public runtime upload-root contract file not found: $uploadContractFile" + } +} +$Workspace = $resolvedWorkspace +. $taskMaintenanceContract +. $taskDefinitionCutoverContract +$offlineReceiptPathPresent = -not [string]::IsNullOrWhiteSpace( + $OfflineBootstrapQuiescenceReceiptPath +) +$offlineReceiptHashPresent = -not [string]::IsNullOrWhiteSpace( + $ExpectedOfflineBootstrapQuiescenceReceiptSha256 +) +if ($offlineReceiptPathPresent -ne $offlineReceiptHashPresent) { + throw "Offline bootstrap quiescence receipt path and lowercase SHA256 are required together" +} +$offlineBootstrapMode = $offlineReceiptPathPresent -and $offlineReceiptHashPresent +$offlineLegacyPinsPresent = ( + -not [string]::IsNullOrWhiteSpace($ExpectedOfflineBootstrapLegacySourceCommit) -and + -not [string]::IsNullOrWhiteSpace($ExpectedOfflineBootstrapLegacySourceTree) +) +$inheritedLockPathPresent = -not [string]::IsNullOrWhiteSpace( + $InheritedRecoveryLockReceiptPath +) +$inheritedLockHashPresent = -not [string]::IsNullOrWhiteSpace( + $ExpectedInheritedRecoveryLockReceiptSha256 +) +if ($inheritedLockPathPresent -ne $inheritedLockHashPresent) { + throw "Inherited recovery-lock receipt path and lowercase SHA256 are required together" +} +$inheritedRecoveryLockMode = $inheritedLockPathPresent -and $inheritedLockHashPresent +if ($offlineBootstrapMode) { + if (-not $offlineLegacyPinsPresent -or -not $inheritedRecoveryLockMode) { + throw "Offline bootstrap requires legacy source pins and an inherited recovery-lock receipt" + } +} elseif ($offlineLegacyPinsPresent -or $inheritedRecoveryLockMode) { + throw "Legacy source pins and inherited recovery lock are valid only in offline bootstrap mode" +} + +function Get-RequiredPrivacySafeCount { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][string]$Role + ) + + $property = $Payload.PSObject.Properties[$Name] + if ($null -eq $property -or $null -eq $property.Value) { + throw "$Role did not return privacy-safe count $Name" + } + $typeCode = [System.Type]::GetTypeCode($property.Value.GetType()) + if (@( + [System.TypeCode]::Byte, + [System.TypeCode]::SByte, + [System.TypeCode]::Int16, + [System.TypeCode]::UInt16, + [System.TypeCode]::Int32, + [System.TypeCode]::UInt32, + [System.TypeCode]::Int64, + [System.TypeCode]::UInt64 + ) -notcontains $typeCode) { + throw "$Role returned non-integer privacy-safe count $Name" + } + $value = [decimal]$property.Value + if ($value -lt 0 -or $value -gt [int]::MaxValue) { + throw "$Role returned out-of-range privacy-safe count $Name" + } + return [int]$value +} + +function Get-PreservedDecodeCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$PreservedObjectCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $validCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "preserved_decode_valid_count" ` + -Role $Role + $invalidCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "preserved_decode_invalid_count" ` + -Role $Role + if (($validCount + $invalidCount) -ne $PreservedObjectCount) { + throw "$Role preserved decode counts do not cover the exact object inventory" + } + return [pscustomobject]@{ + ValidCount = $validCount + InvalidCount = $invalidCount + } +} + +function Get-RequiredDecodeInvalidCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$RequiredObjectCount, + [Parameter(Mandatory = $true)][int]$RequiredReferenceCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $objectCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "required_decode_invalid_object_count" ` + -Role $Role + $referenceCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "required_decode_invalid_reference_count" ` + -Role $Role + if ( + $objectCount -gt $RequiredObjectCount -or + $referenceCount -gt $RequiredReferenceCount + ) { + throw "$Role required decode-invalid counts exceed the bound DB inventory" + } + return [pscustomobject]@{ + ObjectCount = $objectCount + ReferenceCount = $referenceCount + } +} + +function Get-CurrentDecodeInvalidCountProof { + param( + [Parameter(Mandatory = $true)][object]$Payload, + [Parameter(Mandatory = $true)][int]$CurrentObjectCount, + [Parameter(Mandatory = $true)][int]$CurrentReferenceCount, + [Parameter(Mandatory = $true)][string]$Role + ) + + $objectCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "current_decode_invalid_object_count" ` + -Role $Role + $referenceCount = Get-RequiredPrivacySafeCount ` + -Payload $Payload ` + -Name "current_decode_invalid_reference_count" ` + -Role $Role + if ( + $objectCount -gt $CurrentObjectCount -or + $referenceCount -gt $CurrentReferenceCount + ) { + throw "$Role current decode-invalid counts exceed the bound DB inventory" + } + return [pscustomobject]@{ + ObjectCount = $objectCount + ReferenceCount = $referenceCount + } +} + +function Test-StartPathIsSameOrChild { + param( + [string]$Candidate, + [string]$Parent + ) + + $candidateFull = [System.IO.Path]::GetFullPath($Candidate).TrimEnd('\', '/') + $parentFull = [System.IO.Path]::GetFullPath($Parent).TrimEnd('\', '/') + if ([string]::Equals( + $candidateFull, + $parentFull, + [System.StringComparison]::OrdinalIgnoreCase + )) { + return $true + } + return $candidateFull.StartsWith( + $parentFull + [System.IO.Path]::DirectorySeparatorChar, + [System.StringComparison]::OrdinalIgnoreCase + ) +} + +function Assert-StartPathHasNoReparsePoint { + param([string]$Path) + + $cursor = [System.IO.Path]::GetFullPath($Path) + while (-not [string]::IsNullOrWhiteSpace($cursor)) { + if (Test-Path -LiteralPath $cursor) { + $item = Get-Item -LiteralPath $cursor -Force + if (($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Inherited recovery-lock receipt cannot traverse a reparse point" + } + } + $parent = [System.IO.Directory]::GetParent($cursor) + if ($null -eq $parent) { + break + } + $cursor = $parent.FullName + } +} + +function Assert-InheritedRecoveryLockReceipt { + param( + [string]$ReceiptPath, + [string]$ExpectedReceiptSha256, + [string]$LockPath, + [string]$SourceRoot, + [string]$UploadRoot, + [string]$SourceCommit, + [string]$SourceTree + ) + + if (-not [System.IO.Path]::IsPathRooted($ReceiptPath)) { + throw "Inherited recovery-lock receipt must be absolute" + } + $resolvedReceipt = (Resolve-Path -LiteralPath $ReceiptPath).Path + $resolvedLock = [System.IO.Path]::GetFullPath($LockPath) + if (-not [string]::Equals( + $resolvedReceipt, + $resolvedLock, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Inherited recovery-lock receipt must be the active recovery lock" + } + foreach ($boundary in @($SourceRoot, $UploadRoot)) { + if ( + (Test-StartPathIsSameOrChild -Candidate $resolvedReceipt -Parent $boundary) -or + (Test-StartPathIsSameOrChild -Candidate $boundary -Parent $resolvedReceipt) + ) { + throw "Inherited recovery-lock receipt must be private and disjoint" + } + } + Assert-StartPathHasNoReparsePoint -Path $resolvedReceipt + $item = Get-Item -LiteralPath $resolvedReceipt -Force + if ($item.PSIsContainer) { + throw "Inherited recovery-lock receipt must be a regular file" + } + $actualSha256 = ( + Get-FileHash -LiteralPath $resolvedReceipt -Algorithm SHA256 + ).Hash.ToLowerInvariant() + if ($actualSha256 -cne $ExpectedReceiptSha256) { + throw "Inherited recovery-lock receipt SHA256 drift" + } + $payload = Get-Content -LiteralPath $resolvedReceipt -Raw -Encoding UTF8 | ConvertFrom-Json + $actualKeys = @($payload.PSObject.Properties.Name | Sort-Object) + $expectedKeys = @( + "nonce_sha256", + "owner_pid", + "owner_started_at_utc", + "schema_version", + "source_commit", + "source_tree", + "status" + ) | Sort-Object + if ((@($actualKeys) -join "`n") -cne (@($expectedKeys) -join "`n")) { + throw "Inherited recovery-lock receipt schema drift" + } + $currentStartedAtUtc = ( + Get-Process -Id $PID -ErrorAction Stop + ).StartTime.ToUniversalTime().ToString("o") + if ( + $payload.schema_version -ne "vignette.public-runtime-inherited-lock.v1" -or + $payload.status -ne "held" -or + [int]$payload.owner_pid -ne $PID -or + [string]$payload.owner_started_at_utc -cne $currentStartedAtUtc -or + [string]$payload.source_commit -cne $SourceCommit -or + [string]$payload.source_tree -cne $SourceTree -or + [string]$payload.nonce_sha256 -notmatch "^[0-9a-f]{64}$" + ) { + throw "Inherited recovery-lock receipt identity drift" + } + + $exclusiveProbe = $null + try { + $exclusiveProbe = [System.IO.File]::Open( + $resolvedReceipt, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) + throw "Inherited recovery lock is not held" + } catch [System.IO.IOException] { + return + } finally { + if ($null -ne $exclusiveProbe) { + $exclusiveProbe.Dispose() + } + } +} + function Enter-RecoveryLock { param( [string]$LockPath, @@ -73,9 +431,21 @@ function Enter-RecoveryLock { # boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다. # lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다. -$recoveryLock = Enter-RecoveryLock ` - -LockPath $RecoveryLockPath ` - -WaitSeconds $RecoveryLockWaitSeconds +$recoveryLock = $null +if ($inheritedRecoveryLockMode) { + Assert-InheritedRecoveryLockReceipt ` + -ReceiptPath $InheritedRecoveryLockReceiptPath ` + -ExpectedReceiptSha256 $ExpectedInheritedRecoveryLockReceiptSha256 ` + -LockPath $RecoveryLockPath ` + -SourceRoot $resolvedWorkspace ` + -UploadRoot $UserUploadDir ` + -SourceCommit $ExpectedSourceCommit ` + -SourceTree $ExpectedSourceTree +} else { + $recoveryLock = Enter-RecoveryLock ` + -LockPath $RecoveryLockPath ` + -WaitSeconds $RecoveryLockWaitSeconds +} try { # 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다 @@ -145,6 +515,135 @@ function Wait-JsonHealth { throw "Timed out waiting for healthy response from $Uri" } +function Get-Utf8Sha256 { + param([Parameter(Mandatory = $true)][string]$Value) + + $algorithm = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = [System.Text.UTF8Encoding]::new($false).GetBytes($Value) + $digest = $algorithm.ComputeHash($bytes) + return ([System.BitConverter]::ToString($digest)).Replace("-", "").ToLowerInvariant() + } finally { + $algorithm.Dispose() + } +} + +function Assert-PublicUploadWriteFreezeReady { + param( + [Parameter(Mandatory = $true)][string]$HealthUri, + [Parameter(Mandatory = $true)][string]$ExpectedTokenSha256, + [int]$TimeoutSec = 30 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5 + $freeze = $null + if ($null -ne $health) { + $freeze = $health.upload_write_freeze + } + if ( + $null -ne $freeze -and + $freeze.capable -eq $true -and + $freeze.active -eq $true -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 -and + [string]$freeze.token_sha256 -ceq $ExpectedTokenSha256 + ) { + return $health + } + Start-Sleep -Seconds 1 + } while ((Get-Date) -lt $deadline) + throw "Fresh public promotion requires a drained upload-write freeze" +} + +function Exit-PublicUploadWriteFreeze { + param( + [Parameter(Mandatory = $true)][string]$FreezePath, + [Parameter(Mandatory = $true)][string]$ExpectedTokenSha256, + [Parameter(Mandatory = $true)][string]$HealthUri, + [int]$TimeoutSec = 30 + ) + + if (-not (Test-Path -LiteralPath $FreezePath -PathType Leaf)) { + throw "Upload-write freeze sentinel is missing before release" + } + $payload = Get-Content -LiteralPath $FreezePath -Raw -Encoding UTF8 | ConvertFrom-Json + if ( + $null -eq $payload -or + [string]$payload.schema_version -cne "vignette.public-upload-write-freeze.v1" -or + [string]::IsNullOrWhiteSpace([string]$payload.token) -or + (Get-Utf8Sha256 -Value ([string]$payload.token)) -cne $ExpectedTokenSha256 + ) { + throw "Upload-write freeze sentinel ownership proof failed" + } + [System.IO.File]::Delete($FreezePath) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5 + $freeze = $null + if ($null -ne $health) { + $freeze = $health.upload_write_freeze + } + if ( + $null -ne $freeze -and + $freeze.capable -eq $true -and + $freeze.active -eq $false -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 + ) { + return + } + Start-Sleep -Seconds 1 + } while ((Get-Date) -lt $deadline) + throw "Fresh public rollback did not restore upload-write availability" +} + +function Assert-PublicUploadWritesAvailable { + param( + [Parameter(Mandatory = $true)][string]$HealthUri, + [int]$TimeoutSec = 30 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + $health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5 + $freeze = $null + if ($null -ne $health) { + $freeze = $health.upload_write_freeze + } + if ( + $null -ne $freeze -and + $freeze.capable -eq $true -and + $freeze.active -eq $false -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 + ) { + return + } + Start-Sleep -Seconds 1 + } while ((Get-Date) -lt $deadline) + throw "Fresh public rollback did not restore upload-write availability" +} + +function Test-PublicUploadManifestHealth { + param( + [object]$Health, + [Parameter(Mandatory = $true)] + [string]$ExpectedManifestSha256 + ) + + if ($null -eq $Health -or $null -eq $Health.upload_manifest) { + return $false + } + return ( + $Health.upload_manifest.required -eq $true -and + $Health.upload_manifest.validated -eq $true -and + [string]$Health.upload_manifest.manifest_sha256 -ceq $ExpectedManifestSha256 + ) +} + function Test-PortListener { param([int]$Port) @@ -164,10 +663,33 @@ function Get-ListenerProcessIds { -State Listen ` -LocalPort $Port ` -ErrorAction SilentlyContinue | + Where-Object { $_.LocalAddress -eq "127.0.0.1" } | Select-Object -ExpandProperty OwningProcess -Unique ) } +function Get-ExactLoopbackListenerProcess { + param( + [int]$Port, + [string]$Role + ) + + $listenerIds = @(Get-ListenerProcessIds -Port $Port) + if ($listenerIds.Count -gt 1) { + throw "$Role has more than one 127.0.0.1:$Port listener PID" + } + if ($listenerIds.Count -eq 0) { + return $null + } + $process = Get-CimInstance Win32_Process ` + -Filter "ProcessId = $([int]$listenerIds[0])" ` + -ErrorAction SilentlyContinue + if ($null -eq $process) { + throw "$Role listener PID disappeared before identity capture" + } + return $process +} + function Test-VoiceSidecarReady { param( [ValidateSet("stt", "tts")] @@ -395,11 +917,36 @@ function Stop-ProcessesBounded { function Stop-ProcessTreeBounded { param( - [int]$RootProcessId, + [System.Diagnostics.Process]$RootProcess, [int]$TimeoutSec, [string]$Role ) + if ($null -eq $RootProcess) { + throw "Cannot stop $Role without its owned process handle" + } + $RootProcessId = $RootProcess.Id + try { + $null = $RootProcess.Handle + $expectedStartTimeUtc = $RootProcess.StartTime.ToUniversalTime() + } catch { + if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) { + return @($RootProcessId) + } + throw "Cannot verify the owned $Role process handle for PID $RootProcessId" + } + + $currentRoot = Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue + if ($null -eq $currentRoot) { + return @($RootProcessId) + } + if ($currentRoot.StartTime.ToUniversalTime() -ne $expectedStartTimeUtc) { + throw "Refusing to stop reused PID $RootProcessId for $Role" + } + if ($currentRoot.SessionId -ne [System.Diagnostics.Process]::GetCurrentProcess().SessionId) { + throw "Refusing to stop $Role outside the current process session: PID $RootProcessId" + } + $taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe" $previousErrorActionPreference = $ErrorActionPreference try { @@ -546,6 +1093,42 @@ function Wait-ProcessIdentity { throw "Timed out reading $Role process identity for PID $ProcessId" } +function Get-VerifiedProcessFromIdentity { + param( + [System.Collections.IDictionary]$Identity, + [string]$Role, + [int]$TimeoutSec = 15 + ) + + if ($null -eq $Identity) { + return $null + } + $processId = [int]$Identity.pid + $process = Get-CimInstance Win32_Process ` + -Filter "ProcessId = $processId" ` + -ErrorAction SilentlyContinue + if ($null -eq $process) { + return $null + } + $current = Wait-ProcessIdentity ` + -ProcessId $processId ` + -Role $Role ` + -ExpectedCwd ([string]$Identity.cwd) ` + -TimeoutSec $TimeoutSec + foreach ($field in @( + "pid", + "started_at_utc", + "executable_sha256", + "command_line_sha256", + "cwd" + )) { + if ($current[$field].ToString() -cne $Identity[$field].ToString()) { + throw "Refusing to stop drifted or reused $Role process: $field" + } + } + return $process +} + function ConvertTo-SafeProcessIdentity { param([System.Collections.IDictionary]$Identity) @@ -737,6 +1320,8 @@ function Restore-PriorPublicRuntime { param( [System.Collections.IDictionary]$PriorApi, [System.Collections.IDictionary]$PriorCloudflared, + [System.Collections.IDictionary]$ReplacementApi, + [System.Collections.IDictionary]$ReplacementCloudflared, [System.Collections.IDictionary]$PriorLocalVoiceContract, [System.Collections.IDictionary]$PriorPublicVoiceContract, [System.Collections.IDictionary]$EnvironmentSnapshot, @@ -747,12 +1332,55 @@ function Restore-PriorPublicRuntime { [int]$TimeoutSec ) - $null = @( - Stop-UvicornByPort ` - -AppImport "app.main:app" ` - -Port $ApiPortValue ` - -TimeoutSec $TimeoutSec + # Rollback도 ingress를 먼저 닫은 뒤 listener를 바꾼다. 공개 ingress가 살아 + # 있는 상태에서 API/root를 되돌리면 rollback 중 새 write가 prior/new root에 + # 갈라질 수 있으므로 replacement tunnel의 exact identity와 absence를 먼저 증명한다. + $resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path + $verifiedReplacementCloudflared = Get-VerifiedProcessFromIdentity ` + -Identity $ReplacementCloudflared ` + -Role "failed fresh cloudflared" ` + -TimeoutSec $TimeoutSec + if ($null -ne $verifiedReplacementCloudflared) { + $null = @( + Stop-ProcessesBounded ` + -Processes @($verifiedReplacementCloudflared) ` + -TimeoutSec $TimeoutSec ` + -Role "failed fresh cloudflared" + ) + } + $unexpectedCloudflared = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath $resolvedConfigPath ` + -ExactPath ) + if ($unexpectedCloudflared.Count -ne 0) { + throw "Refusing rollback because the tunnel config is owned by an unpinned process" + } + + $replacementApiListener = Get-ExactLoopbackListenerProcess ` + -Port $ApiPortValue ` + -Role "failed fresh API" + if ($null -ne $replacementApiListener) { + if ( + $null -eq $ReplacementApi -or + [int]$replacementApiListener.ProcessId -ne [int]$ReplacementApi.pid + ) { + throw "Refusing rollback because 127.0.0.1:$ApiPortValue is owned by an unpinned listener" + } + $verifiedReplacementApi = Get-VerifiedProcessFromIdentity ` + -Identity $ReplacementApi ` + -Role "failed fresh API" ` + -TimeoutSec $TimeoutSec + $null = @( + Stop-ProcessesBounded ` + -Processes @($verifiedReplacementApi) ` + -TimeoutSec $TimeoutSec ` + -Role "failed fresh API listener" + ) + } + if (@(Get-ListenerProcessIds -Port $ApiPortValue).Count -ne 0) { + throw "Failed fresh API listener remains on 127.0.0.1:$ApiPortValue" + } Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot $priorApiProcess = Start-PinnedPriorProcess ` -Identity $PriorApi ` @@ -769,6 +1397,13 @@ function Restore-PriorPublicRuntime { throw "Restored prior API identity drift: $field" } } + $restoredListenerIds = @(Get-ListenerProcessIds -Port $ApiPortValue) + if ( + $restoredListenerIds.Count -ne 1 -or + [int]$restoredListenerIds[0] -ne [int]$priorApiProcess.Id + ) { + throw "Restored prior API does not own the exact 127.0.0.1:$ApiPortValue listener" + } $null = Wait-JsonHealth ` -Uri "http://127.0.0.1:$ApiPortValue/health" ` -IsHealthy { @@ -789,18 +1424,6 @@ function Restore-PriorPublicRuntime { throw "Restored prior runtime does not have the exact local voice sidecars" } - $resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path - $currentCloudflared = @( - Get-CloudflaredProcessesForConfig ` - -ConfigPath $resolvedConfigPath ` - -ExactPath - ) - $null = @( - Stop-ProcessesBounded ` - -Processes $currentCloudflared ` - -TimeoutSec $TimeoutSec ` - -Role "failed fresh cloudflared" - ) $priorCloudflaredProcess = Start-PinnedPriorProcess ` -Identity $PriorCloudflared ` -Role "cloudflared" ` @@ -922,7 +1545,7 @@ function Initialize-RuntimeProvenanceOutput { $encoding = [System.Text.UTF8Encoding]::new($false) [System.IO.File]::WriteAllText($probeSource, "probe-source", $encoding) [System.IO.File]::WriteAllText($probeTarget, "probe-target", $encoding) - [System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup) + [System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup, $true) [System.IO.File]::Delete($probeTarget) [System.IO.File]::Delete($probeBackup) } finally { @@ -966,7 +1589,7 @@ function Write-Utf8TextAtomically { } if ([System.IO.File]::Exists($OutputPath)) { - [System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath) + [System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath, $true) } elseif (Test-Path -LiteralPath $OutputPath) { throw "Fresh public provenance output became a non-file before commit: $OutputPath" } else { @@ -995,10 +1618,16 @@ function Write-FailedFreshPromotionEvidence { param( [string]$OutputPath, [string]$FailureStage, + [bool]$RollbackAttempted, [bool]$RollbackSucceeded, + [bool]$CurrentRuntimeRetained, + [bool]$TasksRemainDisabled, + [bool]$TasksRestored, + [bool]$TaskStateVerified, [object]$RollbackResult, [string]$SourceCommit, - [string]$SourceTree + [string]$SourceTree, + [string]$UserUploadRoot ) # Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도 @@ -1006,24 +1635,185 @@ function Write-FailedFreshPromotionEvidence { $failedPath = "$OutputPath.failed.log" $payload = [ordered]@{ schema_version = "vignette.public-runtime-launch-failure.v1" - status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" } + status = if ($CurrentRuntimeRetained) { + "failed_current_runtime_retained" + } elseif ($RollbackSucceeded) { + "failed_rolled_back" + } else { + "failed_rollback" + } captured_at_utc = (Get-Date).ToUniversalTime().ToString("o") failure_stage = $FailureStage source = [ordered]@{ git_commit = $SourceCommit.ToLowerInvariant() git_tree = $SourceTree.ToLowerInvariant() } + storage = [ordered]@{ + user_upload_root = $UserUploadRoot + } rollback = [ordered]@{ - attempted = $true + attempted = $RollbackAttempted succeeded = $RollbackSucceeded result = $RollbackResult } + current_runtime_retained = $CurrentRuntimeRetained + tasks_remain_disabled = $TasksRemainDisabled + tasks_restored_to_snapshot = $TasksRestored + task_state_verified = $TaskStateVerified } $json = ConvertTo-Json -InputObject $payload -Depth 8 Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine) return $failedPath } +function Assert-OfflineBootstrapContract { + param( + [string]$SourceRoot, + [string]$SourceCommit, + [string]$SourceTree, + [string]$PythonPath, + [string]$PythonSha256 + ) + + if ($RequireFreshPublicProvenance) { + throw "Offline bootstrap quiescence receipt is mutually exclusive with fresh promotion" + } + if (-not $ForceApiRestart) { + throw "Offline bootstrap requires -ForceApiRestart" + } + if (-not $SkipEngineRestart -or -not $SkipWebRestart -or -not $SkipCloudflaredRestart) { + throw "Offline bootstrap is API-only and requires all engine, web, and cloudflared restart skips" + } + if ($RouteCloudflareDns) { + throw "Offline bootstrap forbids DNS route mutation" + } + foreach ($sourcePin in @($SourceCommit, $SourceTree)) { + if ($sourcePin -notmatch "^[0-9a-f]{40}$") { + throw "Offline bootstrap requires lowercase source commit and tree pins" + } + } + if ($PythonSha256 -notmatch "^[0-9a-f]{64}$") { + throw "Offline bootstrap requires a lowercase Python SHA256 pin" + } + + $resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path + $expectedStartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" + if (-not [string]::Equals( + (Resolve-Path -LiteralPath $PSCommandPath).Path, + (Resolve-Path -LiteralPath $expectedStartScript).Path, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Offline bootstrap launcher must execute from the pinned stable source root" + } + $gitRoot = Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("rev-parse", "--show-toplevel") + if (-not [string]::Equals( + (Resolve-Path -LiteralPath $gitRoot).Path, + $resolvedSourceRoot, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Offline bootstrap source root does not match its Git toplevel" + } + $symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD + $symbolicHeadExit = $LASTEXITCODE + if ($symbolicHeadExit -eq 0) { + throw "Offline bootstrap requires detached HEAD, not branch $symbolicHead" + } + if ($symbolicHeadExit -ne 1) { + throw "Offline bootstrap could not prove detached HEAD" + } + $actualCommit = Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("rev-parse", "--verify", "HEAD") + $actualTree = Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("rev-parse", "--verify", "HEAD^{tree}") + if ($actualCommit -cne $SourceCommit -or $actualTree -cne $SourceTree) { + throw "Offline bootstrap source commit or tree drift" + } + $dirty = Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("status", "--porcelain=v1", "--untracked-files=normal") + if ($dirty) { + throw "Offline bootstrap requires a clean stable source" + } + foreach ($relativePath in @( + "scripts/start-public-runtime.ps1", + "scripts/public-runtime-task-maintenance.ps1", + "scripts/public-runtime-task-definition-cutover.ps1", + "scripts/validate-public-runtime-offline-quiescence.py", + "scripts/initialize-public-runtime-upload-root.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" + )) { + Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) ` + | Out-Null + } + $actualPythonSha256 = ( + Get-FileHash -LiteralPath $PythonPath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + if ($actualPythonSha256 -cne $PythonSha256) { + throw "Offline bootstrap Python SHA256 drift" + } +} + +function Test-OfflineBootstrapQuiescenceReceipt { + param( + [string]$PythonPath, + [string]$ProbePath, + [string]$ReceiptPath, + [string]$ExpectedReceiptSha256, + [string]$ManifestPath, + [string]$ExpectedManifestSha256, + [string]$StableSourceRoot, + [string]$UploadRoot, + [string]$ExpectedLegacySourceCommit, + [string]$ExpectedLegacySourceTree + ) + + $probeArgs = @( + "-X", "utf8", "-B", $ProbePath, + "--receipt-path", $ReceiptPath, + "--expected-receipt-sha256", $ExpectedReceiptSha256, + "--manifest-path", $ManifestPath, + "--expected-manifest-sha256", $ExpectedManifestSha256, + "--stable-source-root", $StableSourceRoot, + "--upload-root", $UploadRoot, + "--expected-legacy-source-commit", $ExpectedLegacySourceCommit, + "--expected-legacy-source-tree", $ExpectedLegacySourceTree + ) + $previousErrorActionPreference = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + $output = @(& $PythonPath @probeArgs 2>$null) + $probeExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + $detail = (@($output) -join "").Trim() + $payload = $null + if (-not [string]::IsNullOrWhiteSpace($detail)) { + try { + $payload = $detail | ConvertFrom-Json + } catch { + $payload = $null + } + } + $safeDetail = "invalid_json" + if ($null -ne $payload) { + $safeDetail = [string]$payload.status + } + return [pscustomobject]@{ + Ok = $probeExit -eq 0 -and $null -ne $payload -and $payload.status -eq "passed" + Detail = $safeDetail + Payload = $payload + } +} + function Assert-FreshPublicProvenanceContract { param( [string]$SourceRoot, @@ -1112,6 +1902,23 @@ function Assert-FreshPublicProvenanceContract { if ($dirty) { throw "Fresh public promotion requires a clean stable source" } + foreach ($relativePath in @( + "scripts/start-public-runtime.ps1", + "scripts/public-runtime-upload-root.ps1", + "scripts/public-runtime-task-maintenance.ps1", + "scripts/public-runtime-task-definition-cutover.ps1", + "scripts/register-boot-task.ps1", + "scripts/install-public-runtime-task.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" + )) { + Invoke-StableGitText ` + -SourceRoot $resolvedSourceRoot ` + -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null + } foreach ($pin in @( [pscustomobject]@{ Path = $PythonPath; Sha256 = $PythonSha256; Label = "Python" }, @@ -1184,6 +1991,7 @@ function Set-CloudflaredIngress { $freshMutationStarted = $false $freshPromotionCommitted = $false +$freshNoRollback = $false $freshFailureStage = "preflight" $freshPriorApiIdentity = $null $freshPriorCloudflaredIdentity = $null @@ -1191,6 +1999,16 @@ $freshPriorLocalVoiceContract = $null $freshPriorPublicVoiceContract = $null $freshEnvironmentSnapshot = $null $resolvedRuntimeProvenancePath = $null +$resolvedTaskRecoveryReceiptPath = $null +$freshStoppedTunnelIds = @() +$freshTaskMaintenanceSnapshot = @() +$freshTaskMaintenanceEntered = $false +$freshTaskMaintenanceWasEntered = $false +$freshTasksRestored = $false +$freshOriginalTaskDefinitionSnapshot = $null +$freshDisabledOriginalTaskDefinitionSnapshot = $null +$freshNewDisabledTaskDefinitionSnapshot = $null +$freshOperationalTaskDefinitionSnapshot = $null $freshManagedEnvironmentNames = @( "ENVIRONMENT", "ENGINE_URL", @@ -1208,34 +2026,99 @@ $freshManagedEnvironmentNames = @( "VIGNETTE_MELOTTS_TTS_URL", "FRONTEND_BASE_URL", "CORS_ORIGINS", - "FRONTEND_ORIGIN_MAP" + "FRONTEND_ORIGIN_MAP", + "USER_UPLOAD_DIR", + "USER_UPLOAD_MANIFEST_REQUIRED", + "USER_UPLOAD_MANIFEST_PATH", + "USER_UPLOAD_MANIFEST_SHA256", + "USER_UPLOAD_WRITE_FREEZE_PATH", + "PUBLIC_RUNTIME_DB_TARGET_SHA256" ) trap { $caught = $_ if ( $RequireFreshPublicProvenance -and - $freshMutationStarted -and - -not $freshPromotionCommitted + $freshTaskMaintenanceWasEntered ) { $rollbackResult = $null + $rollbackAttempted = $false $rollbackSucceeded = $false + $tasksRemainDisabled = $false + $tasksRestored = $false + $taskStateVerified = $false $rollbackFailureType = "none" + if (-not $freshNoRollback) { + try { + if ($freshMutationStarted) { + $rollbackAttempted = $true + $rollbackResult = Restore-PriorPublicRuntime ` + -PriorApi $freshPriorApiIdentity ` + -PriorCloudflared $freshPriorCloudflaredIdentity ` + -ReplacementApi $apiLaunchIdentity ` + -ReplacementCloudflared $cloudflaredLaunchIdentity ` + -PriorLocalVoiceContract $freshPriorLocalVoiceContract ` + -PriorPublicVoiceContract $freshPriorPublicVoiceContract ` + -EnvironmentSnapshot $freshEnvironmentSnapshot ` + -ConfigPath $CloudflaredConfig ` + -ApiPortValue $ApiPort ` + -HealthUrl $PublicHealthUrl ` + -VoiceHealthUrl $CanonicalPublicVoiceHealthUrl ` + -TimeoutSec $ProcessStopTimeoutSeconds + } else { + $rollbackResult = [ordered]@{ runtime_mutation = $false } + } + if (Test-Path -LiteralPath $resolvedUserUploadWriteFreezePath -PathType Leaf) { + if ($null -eq $uploadManifestProof) { + throw "Cannot prove upload-write freeze ownership during recovery" + } + Exit-PublicUploadWriteFreeze ` + -FreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec 30 + } else { + Assert-PublicUploadWritesAvailable ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec 30 + } + Exit-PublicRuntimeTaskMaintenance ` + -Snapshot $freshTaskMaintenanceSnapshot + $freshTaskMaintenanceEntered = $false + $freshTasksRestored = $true + $rollbackSucceeded = $true + } catch { + $rollbackFailureType = $_.Exception.GetType().Name + } + } + if (-not $rollbackSucceeded -and $freshTaskMaintenanceEntered) { + try { + Suspend-PublicRuntimeTasks ` + -Snapshot $freshTaskMaintenanceSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + $tasksRemainDisabled = $true + } catch { + $rollbackFailureType = $_.Exception.GetType().Name + } + } + try { - $rollbackResult = Restore-PriorPublicRuntime ` - -PriorApi $freshPriorApiIdentity ` - -PriorCloudflared $freshPriorCloudflaredIdentity ` - -PriorLocalVoiceContract $freshPriorLocalVoiceContract ` - -PriorPublicVoiceContract $freshPriorPublicVoiceContract ` - -EnvironmentSnapshot $freshEnvironmentSnapshot ` - -ConfigPath $CloudflaredConfig ` - -ApiPortValue $ApiPort ` - -HealthUrl $PublicHealthUrl ` - -VoiceHealthUrl $CanonicalPublicVoiceHealthUrl ` - -TimeoutSec $ProcessStopTimeoutSeconds - $rollbackSucceeded = $true + $taskTruth = Get-PublicRuntimeTaskMaintenanceState ` + -Snapshot $freshTaskMaintenanceSnapshot + $taskStateVerified = [bool]$taskTruth.verified + $tasksRemainDisabled = ( + $taskStateVerified -and + [bool]$taskTruth.all_disabled_and_idle + ) + $tasksRestored = ( + $taskStateVerified -and + [bool]$taskTruth.restored_to_snapshot + ) + $freshTasksRestored = $tasksRestored } catch { - $rollbackFailureType = $_.Exception.GetType().Name + $taskStateVerified = $false + $tasksRemainDisabled = $false + $tasksRestored = $false } $failedEvidencePath = "" @@ -1244,17 +2127,26 @@ trap { $failedEvidencePath = Write-FailedFreshPromotionEvidence ` -OutputPath $resolvedRuntimeProvenancePath ` -FailureStage $freshFailureStage ` + -RollbackAttempted $rollbackAttempted ` -RollbackSucceeded $rollbackSucceeded ` + -CurrentRuntimeRetained $freshNoRollback ` + -TasksRemainDisabled $tasksRemainDisabled ` + -TasksRestored $tasksRestored ` + -TaskStateVerified $taskStateVerified ` -RollbackResult $rollbackResult ` -SourceCommit $ExpectedSourceCommit ` - -SourceTree $ExpectedSourceTree + -SourceTree $ExpectedSourceTree ` + -UserUploadRoot $resolvedUserUploadDir } catch { $failedEvidencePath = "unavailable" } } + if ($freshNoRollback) { + throw "Fresh public promotion failed at $freshFailureStage after the no-rollback boundary; the new runtime/root was retained and full operational success was not published. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" + } if ($rollbackSucceeded) { - throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" + throw "Fresh public promotion failed at $freshFailureStage; the pinned prior runtime/write availability was restored and scheduled tasks were re-enabled. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" } throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" } @@ -1296,9 +2188,188 @@ if ($RequireFreshPublicProvenance) { -CloudflaredSha256 $ExpectedCloudflaredSha256 ` -ConfigPath $CloudflaredConfig ` -ConfigSha256 $ExpectedCloudflaredConfigSha256 +} +if ($offlineBootstrapMode) { + Assert-OfflineBootstrapContract ` + -SourceRoot $Workspace ` + -SourceCommit $ExpectedSourceCommit ` + -SourceTree $ExpectedSourceTree ` + -PythonPath $Python ` + -PythonSha256 $ExpectedPythonSha256 +} +. $uploadRootContract +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot $resolvedWorkspace ` + -UploadRoot $UserUploadDir ` + -ProbeWritable +$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedWorkspace ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadManifestPath ` + -RequireFile +$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedWorkspace ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadWriteFreezePath +$resolvedOfflineBootstrapQuiescenceReceiptPath = "" +if ($offlineBootstrapMode) { + $resolvedOfflineBootstrapQuiescenceReceiptPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedWorkspace ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $OfflineBootstrapQuiescenceReceiptPath ` + -RequireFile +} +if ($RequireFreshPublicProvenance) { + $freshFailureStage = "task_maintenance" + Assert-PublicRuntimeCoordinatedTaskNamesExact ` + -TaskNames $CoordinatedTaskNames + $freshOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot ` + -RequireEnabled + $freshTaskMaintenanceSnapshot = @( + Enter-PublicRuntimeTaskMaintenance ` + -TaskNames $CoordinatedTaskNames ` + -TimeoutSec $ProcessStopTimeoutSeconds + ) + $freshTaskMaintenanceEntered = $true + $freshTaskMaintenanceWasEntered = $true + $freshDisabledOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot + foreach ($taskDefinition in @($freshDisabledOriginalTaskDefinitionSnapshot.entries)) { + if ([bool]$taskDefinition.enabled) { + throw "Original public runtime task definition remained enabled during maintenance" + } + } +} +$uploadManifestProof = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $uploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +if (-not $uploadManifestProof.Ok) { + throw "Public upload migration receipt or current DB inventory is invalid: $($uploadManifestProof.Detail)" +} +$manifestPreservedObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $uploadManifestProof.Payload ` + -Name "preserved_object_count" ` + -Role "Public upload manifest validator" +$manifestRequiredObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $uploadManifestProof.Payload ` + -Name "required_object_count" ` + -Role "Public upload manifest validator" +$manifestRequiredReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $uploadManifestProof.Payload ` + -Name "required_reference_count" ` + -Role "Public upload manifest validator" +$manifestCurrentObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $uploadManifestProof.Payload ` + -Name "current_object_count" ` + -Role "Public upload manifest validator" +$manifestCurrentReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $uploadManifestProof.Payload ` + -Name "current_reference_count" ` + -Role "Public upload manifest validator" +if ( + $manifestRequiredObjectCount -gt $manifestRequiredReferenceCount -or + $manifestCurrentObjectCount -gt $manifestCurrentReferenceCount +) { + throw "Public upload manifest validator returned impossible DB inventory counts" +} +$manifestPreservedDecodeCounts = Get-PreservedDecodeCountProof ` + -Payload $uploadManifestProof.Payload ` + -PreservedObjectCount $manifestPreservedObjectCount ` + -Role "Public upload manifest validator" +$manifestRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof ` + -Payload $uploadManifestProof.Payload ` + -RequiredObjectCount $manifestRequiredObjectCount ` + -RequiredReferenceCount $manifestRequiredReferenceCount ` + -Role "Public upload manifest validator" +$manifestCurrentDecodeCounts = Get-CurrentDecodeInvalidCountProof ` + -Payload $uploadManifestProof.Payload ` + -CurrentObjectCount $manifestCurrentObjectCount ` + -CurrentReferenceCount $manifestCurrentReferenceCount ` + -Role "Public upload manifest validator" +$expectedDatabaseTargetSha256 = [string]$uploadManifestProof.Payload.database_target_sha256 +if ($expectedDatabaseTargetSha256 -notmatch "^[0-9a-f]{64}$") { + throw "Public upload inventory proof did not return a valid database target identity" +} +$offlineBootstrapReceiptProof = $null +if ($offlineBootstrapMode) { + $offlineBootstrapReceiptProof = Test-OfflineBootstrapQuiescenceReceipt ` + -PythonPath $Python ` + -ProbePath $offlineQuiescenceProbe ` + -ReceiptPath $resolvedOfflineBootstrapQuiescenceReceiptPath ` + -ExpectedReceiptSha256 $ExpectedOfflineBootstrapQuiescenceReceiptSha256 ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -StableSourceRoot $resolvedWorkspace ` + -UploadRoot $resolvedUserUploadDir ` + -ExpectedLegacySourceCommit $ExpectedOfflineBootstrapLegacySourceCommit ` + -ExpectedLegacySourceTree $ExpectedOfflineBootstrapLegacySourceTree + if (-not $offlineBootstrapReceiptProof.Ok) { + throw "Offline bootstrap quiescence receipt is invalid" + } + $offlinePayload = $offlineBootstrapReceiptProof.Payload + $offlinePreservedObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $offlinePayload ` + -Name "preserved_object_count" ` + -Role "Offline bootstrap quiescence validator" + $offlineRequiredObjectCount = Get-RequiredPrivacySafeCount ` + -Payload $offlinePayload ` + -Name "unique_object_count" ` + -Role "Offline bootstrap quiescence validator" + $offlineRequiredReferenceCount = Get-RequiredPrivacySafeCount ` + -Payload $offlinePayload ` + -Name "reference_count" ` + -Role "Offline bootstrap quiescence validator" + $offlinePreservedDecodeCounts = Get-PreservedDecodeCountProof ` + -Payload $offlinePayload ` + -PreservedObjectCount $offlinePreservedObjectCount ` + -Role "Offline bootstrap quiescence validator" + $offlineRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof ` + -Payload $offlinePayload ` + -RequiredObjectCount $offlineRequiredObjectCount ` + -RequiredReferenceCount $offlineRequiredReferenceCount ` + -Role "Offline bootstrap quiescence validator" + if ( + [string]$offlinePayload.database_target_sha256 -cne $expectedDatabaseTargetSha256 -or + $offlinePreservedObjectCount -ne $manifestPreservedObjectCount -or + [long]$offlinePayload.preserved_total_size_bytes -ne + [long]$uploadManifestProof.Payload.preserved_total_size_bytes -or + [string]$offlinePayload.preserved_inventory_sha256 -cne + [string]$uploadManifestProof.Payload.preserved_object_set_sha256 -or + $offlineRequiredReferenceCount -ne $manifestRequiredReferenceCount -or + $offlineRequiredObjectCount -ne $manifestRequiredObjectCount -or + $offlineRequiredReferenceCount -ne $manifestCurrentReferenceCount -or + $offlineRequiredObjectCount -ne $manifestCurrentObjectCount -or + [string]$offlinePayload.reference_set_sha256 -cne + [string]$uploadManifestProof.Payload.reference_set_sha256 -or + [string]$offlinePayload.reference_set_sha256 -cne [string]$uploadManifestProof.Payload.current_reference_set_sha256 -or + $offlinePreservedDecodeCounts.ValidCount -ne + $manifestPreservedDecodeCounts.ValidCount -or + $offlinePreservedDecodeCounts.InvalidCount -ne + $manifestPreservedDecodeCounts.InvalidCount -or + $offlineRequiredDecodeCounts.ObjectCount -ne + $manifestRequiredDecodeCounts.ObjectCount -or + $offlineRequiredDecodeCounts.ReferenceCount -ne + $manifestRequiredDecodeCounts.ReferenceCount -or + $offlineRequiredDecodeCounts.ObjectCount -ne + $manifestCurrentDecodeCounts.ObjectCount -or + $offlineRequiredDecodeCounts.ReferenceCount -ne + $manifestCurrentDecodeCounts.ReferenceCount -or + $offlinePayload.listener_absent -ne $true -or + $offlinePayload.tunnel_absent -ne $true + ) { + throw "Offline bootstrap quiescence receipt drifted from the current database inventory" + } +} + +if ($RequireFreshPublicProvenance) { $resolvedRuntimeProvenancePath = Initialize-RuntimeProvenanceOutput ` -OutputPath $RuntimeProvenancePath + $resolvedTaskRecoveryReceiptPath = Initialize-RuntimeProvenanceOutput ` + -OutputPath "$resolvedRuntimeProvenancePath.tasks-restored.log" # 승격 모드에서 config를 재작성하면 사전 pin과 실제 tunnel 입력이 달라진다. # exact ingress가 이미 들어 있는 경우에만 이후 프로세스 mutation으로 진행한다. @@ -1310,11 +2381,24 @@ if ($RequireFreshPublicProvenance) { -WebPortValue $WebPort ` -RequireUnchanged - $priorApiProcesses = @( - Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort - ) - if ($priorApiProcesses.Count -ne 1) { - throw "Fresh public promotion requires exactly one prior API process for transactional rollback" + $priorApiListenerProof = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd $ApiDir ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort + if (-not $priorApiListenerProof.Ok) { + throw "Fresh public promotion requires one exact prior API listener identity: $($priorApiListenerProof.Detail)" + } + $priorApiProcess = Get-CimInstance Win32_Process ` + -Filter "ProcessId = $($priorApiListenerProof.ListenerPid)" ` + -ErrorAction SilentlyContinue + if ($null -eq $priorApiProcess) { + throw "Fresh public promotion lost the exact prior API listener before capture" } $priorCloudflaredProcesses = @( Get-CloudflaredProcessesForConfig ` @@ -1325,8 +2409,9 @@ if ($RequireFreshPublicProvenance) { throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback" } $freshPriorApiIdentity = Wait-ProcessIdentity ` - -ProcessId $priorApiProcesses[0].ProcessId ` + -ProcessId $priorApiListenerProof.ListenerPid ` -Role "prior api" ` + -ExpectedCwd $ApiDir ` -TimeoutSec $ProcessStopTimeoutSeconds $freshPriorCloudflaredIdentity = Wait-ProcessIdentity ` -ProcessId $priorCloudflaredProcesses[0].ProcessId ` @@ -1365,6 +2450,10 @@ if ($RequireFreshPublicProvenance) { -TimeoutSec 30 $freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract ` -Health $priorPublicVoiceHealth + $null = Assert-PublicUploadWriteFreezeReady ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 ` + -TimeoutSec 30 $freshEnvironmentSnapshot = Save-ManagedEnvironment ` -Names $freshManagedEnvironmentNames } @@ -1429,6 +2518,12 @@ $env:VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE = $WhisperLanguage $env:VIGNETTE_VOICE_TTS_PROVIDER = "melotts" $env:VIGNETTE_MELOTTS_TTS_URL = "http://127.0.0.1:$MeloTtsPort" $env:FRONTEND_BASE_URL = "https://vignette.chanpaca.net" +$env:USER_UPLOAD_DIR = $resolvedUserUploadDir +$env:USER_UPLOAD_MANIFEST_REQUIRED = "true" +$env:USER_UPLOAD_MANIFEST_PATH = $resolvedUserUploadManifestPath +$env:USER_UPLOAD_MANIFEST_SHA256 = $ExpectedUserUploadManifestSha256 +$env:USER_UPLOAD_WRITE_FREEZE_PATH = $resolvedUserUploadWriteFreezePath +$env:PUBLIC_RUNTIME_DB_TARGET_SHA256 = $expectedDatabaseTargetSha256 $frontendOrigins = @("https://vignette.chanpaca.net", "https://vnet.18ka.net", "https://vignette-b1q.pages.dev") $localViteOrigins = @() foreach ($port in 5170..5180) { @@ -1445,8 +2540,8 @@ $env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{ # 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를 # 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다. if (-not (Test-VoiceSidecarReady -Component "stt")) { - if ($RequireFreshPublicProvenance) { - throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying" + if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) { + throw "Public upload-root promotion will not mutate local_whisper; restore the exact sidecar before retrying" } if (Test-PortListener -Port $WhisperPort) { throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol" @@ -1462,8 +2557,8 @@ if (-not (Test-VoiceSidecarReady -Component "stt")) { } if (-not (Test-VoiceSidecarReady -Component "tts")) { - if ($RequireFreshPublicProvenance) { - throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying" + if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) { + throw "Public upload-root promotion will not mutate MeloTTS; restore the exact sidecar before retrying" } if (Test-PortListener -Port $MeloTtsPort) { throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract" @@ -1485,12 +2580,23 @@ if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarRe $health = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/health" $voiceHealth = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/voice/health" +$apiUploadRootHealth = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd $ApiDir ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort $apiControlPlaneReady = ( $null -ne $health -and $health.environment -eq "prod" -and $health.db -eq $true -and $health.engine -eq $true -and - (Test-VoiceApiReady -Health $voiceHealth) + (Test-VoiceApiReady -Health $voiceHealth) -and + $apiUploadRootHealth.Ok ) $proc = $null $apiStoppedProcessIds = @() @@ -1501,19 +2607,51 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) { if ($RequireFreshPublicProvenance) { $freshFailureStage = "api_cutover" $freshMutationStarted = $true + # 공개 ingress를 먼저 닫은 뒤 API를 교체한다. 이 시점까지 freeze가 + # drained 상태이므로 rollback 전후 어느 root에도 새 DB/file write가 없다. + $freshStoppedTunnelIds = @( + Stop-ProcessesBounded ` + -Processes $priorCloudflaredProcesses ` + -TimeoutSec $ProcessStopTimeoutSeconds ` + -Role "prior cloudflared before upload-root cutover" + ) + } + $apiListenerProcess = Get-ExactLoopbackListenerProcess ` + -Port $ApiPort ` + -Role "public API" + if ($offlineBootstrapMode -and $null -ne $apiListenerProcess) { + throw "Offline bootstrap API listener reappeared after the quiescence receipt; refusing to stop an unpinned process" + } + if ($RequireFreshPublicProvenance) { + if ( + $null -eq $apiListenerProcess -or + [int]$apiListenerProcess.ProcessId -ne [int]$priorApiListenerProof.ListenerPid + ) { + throw "Fresh public API listener changed before the exact stop boundary" + } + } + if ($null -eq $apiListenerProcess) { + # No socket owner means no API process is ours to stop. Command-line decoys + # that merely mention uvicorn/8001 are deliberately left untouched. + $apiStoppedProcessIds = @() + } else { + $apiStoppedProcessIds = @( + Stop-ProcessesBounded ` + -Processes @($apiListenerProcess) ` + -TimeoutSec $ProcessStopTimeoutSeconds ` + -Role "exact public API listener" + ) + } + if (@(Get-ListenerProcessIds -Port $ApiPort).Count -ne 0) { + throw "Public API listener remains on 127.0.0.1:$ApiPort after bounded stop" } - $apiStoppedProcessIds = @( - Stop-UvicornByPort ` - -AppImport "app.main:app" ` - -Port $ApiPort ` - -TimeoutSec $ProcessStopTimeoutSeconds - ) $proc = Start-Process -WindowStyle Hidden -FilePath $Python ` -ArgumentList @( "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", "$ApiPort", + "--workers", "1", "--ws", "websockets", "--ws-max-queue", "4" ) ` @@ -1522,9 +2660,9 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) { -RedirectStandardError $ErrLog ` -PassThru - if ($RequireFreshPublicProvenance) { + if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) { if ($apiStoppedProcessIds -contains $proc.Id) { - throw "Fresh public API did not receive a replacement PID" + throw "Public API cutover did not receive a replacement PID" } $apiLaunchIdentity = Wait-ProcessIdentity ` -ProcessId $proc.Id ` @@ -1532,11 +2670,11 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) { -ExpectedCwd $ApiDir ` -TimeoutSec $ProcessStopTimeoutSeconds if ($apiLaunchIdentity.executable_sha256 -ne $ExpectedPythonSha256.ToLowerInvariant()) { - throw "Fresh public API executable SHA256 does not match the pinned Python" + throw "Public API executable SHA256 does not match the pinned Python" } - foreach ($requiredArgument in @("uvicorn", "app.main:app", "--port", "$ApiPort", "--ws-max-queue", "4")) { + foreach ($requiredArgument in @("uvicorn", "app.main:app", "--port", "$ApiPort", "--workers", "1", "--ws-max-queue", "4")) { if ($apiLaunchIdentity.command_line.IndexOf($requiredArgument, [System.StringComparison]::Ordinal) -lt 0) { - throw "Fresh public API command line is missing required argument: $requiredArgument" + throw "Public API command line is missing required argument: $requiredArgument" } } } @@ -1548,17 +2686,51 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) { param($health) $health.environment -eq "prod" -and $health.db -eq $true -and - $health.engine -eq $true + $health.engine -eq $true -and + (Test-PublicUploadManifestHealth ` + -Health $health ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256) } ` -TimeoutSec $ApiReadySeconds $voiceHealth = Wait-JsonHealth ` -Uri "http://127.0.0.1:$ApiPort/voice/health" ` -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` -TimeoutSec $VoiceApiReadySeconds + if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) { + $null = Assert-PublicUploadWriteFreezeReady ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 ` + -TimeoutSec 30 + } } -if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) { +if ( + $health.environment -ne "prod" -or + -not $health.db -or + -not $health.engine -or + -not (Test-PublicUploadManifestHealth ` + -Health $health ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256) +) { throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)" } +$apiUploadRootHealth = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd $ApiDir ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort +if (-not $apiUploadRootHealth.Ok) { + throw "Public API does not use the pinned persistent upload root: $($apiUploadRootHealth.Detail)" +} +if (-not $RequireFreshPublicProvenance -and -not $offlineBootstrapMode) { + Assert-PublicUploadWritesAvailable ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec 30 +} if (-not (Test-VoiceApiReady -Health $voiceHealth)) { throw "Public voice API does not match the exact local provider/model contract: $($voiceHealth | ConvertTo-Json -Compress)" } @@ -1581,7 +2753,7 @@ if (!$SkipWebRestart) { if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) { $null = @( Stop-ProcessTreeBounded ` - -RootProcessId $build.Id ` + -RootProcess $build ` -TimeoutSec $ProcessStopTimeoutSeconds ` -Role "web build" ) @@ -1607,7 +2779,7 @@ if (!$SkipWebRestart) { $cloudflaredProcess = $null $cloudflaredLaunchIdentity = $null -$cloudflaredStoppedProcessIds = @() +$cloudflaredStoppedProcessIds = @($freshStoppedTunnelIds) if (!$SkipCloudflaredRestart) { if ($RequireFreshPublicProvenance) { $freshFailureStage = "cloudflared_cutover" @@ -1644,16 +2816,24 @@ if (!$SkipCloudflaredRestart) { -ConfigPath $resolvedCloudflaredConfig ` -ExactPath ) + if ($existingCloudflaredProcesses.Count -ne 0) { + throw "Fresh public tunnel config was reacquired by an unpinned process before launch" + } + $additionalCloudflaredStoppedProcessIds = @() } else { $existingCloudflaredProcesses = @( Get-CloudflaredProcessesForConfig -ConfigPath $resolvedCloudflaredConfig ) + $additionalCloudflaredStoppedProcessIds = @( + Stop-ProcessesBounded ` + -Processes $existingCloudflaredProcesses ` + -TimeoutSec $ProcessStopTimeoutSeconds ` + -Role "cloudflared for $resolvedCloudflaredConfig" + ) } $cloudflaredStoppedProcessIds = @( - Stop-ProcessesBounded ` - -Processes $existingCloudflaredProcesses ` - -TimeoutSec $ProcessStopTimeoutSeconds ` - -Role "cloudflared for $resolvedCloudflaredConfig" + @($cloudflaredStoppedProcessIds) + @($additionalCloudflaredStoppedProcessIds) | + Sort-Object -Unique ) $cloudflaredProcess = Start-Process -WindowStyle Hidden -FilePath $Cloudflared ` @@ -1687,8 +2867,24 @@ if ($RequireFreshPublicProvenance) { throw "Fresh public promotion did not produce both API and cloudflared identities" } + $apiFinalListenerProof = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd $ApiDir ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort + if ( + -not $apiFinalListenerProof.Ok -or + [int]$apiFinalListenerProof.ListenerPid -ne [int]$apiLaunchIdentity.pid + ) { + throw "Fresh public API launch PID is not the exact validated 127.0.0.1:$ApiPort listener" + } $apiFinalIdentity = Wait-ProcessIdentity ` - -ProcessId $apiLaunchIdentity.pid ` + -ProcessId $apiFinalListenerProof.ListenerPid ` -Role "api" ` -ExpectedCwd $ApiDir ` -TimeoutSec $ProcessStopTimeoutSeconds @@ -1697,6 +2893,17 @@ if ($RequireFreshPublicProvenance) { -Role "cloudflared" ` -ExpectedCwd $Workspace ` -TimeoutSec $ProcessStopTimeoutSeconds + $finalCloudflaredOwners = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath $resolvedCloudflaredConfig ` + -ExactPath + ) + if ( + $finalCloudflaredOwners.Count -ne 1 -or + [int]$finalCloudflaredOwners[0].ProcessId -ne [int]$cloudflaredFinalIdentity.pid + ) { + throw "Fresh public tunnel config is not owned by the one pinned cloudflared PID" + } foreach ($identityPair in @( [pscustomobject]@{ Role = "api"; Launch = $apiLaunchIdentity; Final = $apiFinalIdentity }, [pscustomobject]@{ Role = "cloudflared"; Launch = $cloudflaredLaunchIdentity; Final = $cloudflaredFinalIdentity } @@ -1738,6 +2945,69 @@ if ($RequireFreshPublicProvenance) { if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) { throw "Exact local voice sidecars changed before provenance receipt" } + $receiptApiListenerProof = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd $ApiDir ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort + if ( + -not $receiptApiListenerProof.Ok -or + [int]$receiptApiListenerProof.ListenerPid -ne [int]$apiLaunchIdentity.pid + ) { + throw "Fresh public API listener drifted before provenance receipt" + } + $apiFinalIdentity = Wait-ProcessIdentity ` + -ProcessId $receiptApiListenerProof.ListenerPid ` + -Role "api before receipt" ` + -ExpectedCwd $ApiDir ` + -TimeoutSec $ProcessStopTimeoutSeconds + foreach ($field in @( + "pid", + "started_at_utc", + "executable_sha256", + "command_line_sha256", + "cwd" + )) { + if ($apiLaunchIdentity[$field].ToString() -cne $apiFinalIdentity[$field].ToString()) { + throw "Fresh API listener provenance drifted before receipt: $field" + } + } + $receiptTunnelOwners = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath $resolvedCloudflaredConfig ` + -ExactPath + ) + if ( + $receiptTunnelOwners.Count -ne 1 -or + [int]$receiptTunnelOwners[0].ProcessId -ne [int]$cloudflaredLaunchIdentity.pid + ) { + throw "Fresh public tunnel config ownership drifted before provenance receipt" + } + $cloudflaredReceiptIdentity = Wait-ProcessIdentity ` + -ProcessId $cloudflaredLaunchIdentity.pid ` + -Role "cloudflared before receipt" ` + -ExpectedCwd $Workspace ` + -TimeoutSec $ProcessStopTimeoutSeconds + foreach ($field in @( + "pid", + "started_at_utc", + "executable_sha256", + "command_line_sha256", + "cwd" + )) { + if ( + $cloudflaredLaunchIdentity[$field].ToString() -cne + $cloudflaredReceiptIdentity[$field].ToString() + ) { + throw "Fresh cloudflared provenance drifted before receipt: $field" + } + } + $cloudflaredFinalIdentity = $cloudflaredReceiptIdentity $psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))") $psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim() if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) { @@ -1745,10 +3015,16 @@ if ($RequireFreshPublicProvenance) { } $safeApiIdentity = ConvertTo-SafeProcessIdentity -Identity $apiFinalIdentity $safeCloudflaredIdentity = ConvertTo-SafeProcessIdentity -Identity $cloudflaredFinalIdentity + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $freshDisabledOriginalTaskDefinitionSnapshot $provenance = [ordered]@{ schema_version = "vignette.public-runtime-launch-provenance.v1" status = "passed" + scope = "runtime_storage_commit_only" + operational_success = $false + task_recovery_receipt_path_sha256 = Get-Utf8Sha256 ` + -Value $resolvedTaskRecoveryReceiptPath captured_at_utc = (Get-Date).ToUniversalTime().ToString("o") source = [ordered]@{ repo_root = (Resolve-Path -LiteralPath $Workspace).Path @@ -1761,6 +3037,27 @@ if ($RequireFreshPublicProvenance) { path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path sha256 = $finalConfigSha256 } + storage = [ordered]@{ + user_upload_root = $resolvedUserUploadDir + migration_manifest_sha256 = $ExpectedUserUploadManifestSha256 + initialized_object_count = [int]$uploadManifestProof.Payload.required_object_count + initialized_reference_count = [int]$uploadManifestProof.Payload.required_reference_count + initialized_reference_set_sha256 = [string]$uploadManifestProof.Payload.reference_set_sha256 + preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount + preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount + required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount + required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount + current_object_count = [int]$uploadManifestProof.Payload.current_object_count + current_reference_count = [int]$uploadManifestProof.Payload.current_reference_count + current_reference_set_sha256 = [string]$uploadManifestProof.Payload.current_reference_set_sha256 + current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount + current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount + write_freeze_path_sha256 = [string]$health.upload_write_freeze.path_sha256 + } + task_definitions = [ordered]@{ + original_set_sha256 = [string]$freshOriginalTaskDefinitionSnapshot.set_sha256 + disabled_pre_cutover_set_sha256 = [string]$freshDisabledOriginalTaskDefinitionSnapshot.set_sha256 + } public_validation = [ordered]@{ health_url = $CanonicalPublicHealthUrl health = $true @@ -1801,18 +3098,194 @@ if ($RequireFreshPublicProvenance) { } } $provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8 + $boundaryTunnelOwners = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath $resolvedCloudflaredConfig ` + -ExactPath + ) + if ( + $boundaryTunnelOwners.Count -ne 1 -or + [int]$boundaryTunnelOwners[0].ProcessId -ne [int]$cloudflaredFinalIdentity.pid + ) { + throw "Fresh public tunnel config ownership drifted at the write-release boundary" + } + $freshFailureStage = "upload_write_release" + # 이 지점부터는 새 API/root를 외부에 유지한다. Exit 함수는 센티널을 지운 뒤 + # health에서 write-unfrozen을 확인하므로, 호출 중 오류가 나도 새 root에 쓰기가 + # 시작됐을 수 있다. 따라서 release 시도 전에 rollback 금지 경계를 확정한다. + $freshNoRollback = $true + Exit-PublicUploadWriteFreeze ` + -FreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 ` + -HealthUri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec 30 + $null = Wait-JsonHealth ` + -Uri $CanonicalPublicHealthUrl ` + -IsHealthy { + param($health) + $freeze = $health.upload_write_freeze + $manifest = $health.upload_manifest + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true -and + $null -ne $freeze -and + $freeze.capable -eq $true -and + $freeze.active -eq $false -and + $freeze.valid -eq $true -and + [int]$freeze.in_flight -eq 0 -and + $null -ne $manifest -and + $manifest.required -eq $true -and + $manifest.validated -eq $true -and + [string]$manifest.manifest_sha256 -ceq $ExpectedUserUploadManifestSha256 + } ` + -TimeoutSec 60 $freshFailureStage = "receipt_publish" try { + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $freshDisabledOriginalTaskDefinitionSnapshot Write-Utf8TextAtomically ` -OutputPath $resolvedRuntimeProvenancePath ` -Value ($provenanceJson + [Environment]::NewLine) } catch { - # 새 PID들은 이미 health/identity gate를 통과했지만, atomic receipt가 없으면 - # 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다. - throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)" + # write release 뒤에는 prior root로 되돌릴 수 없다. passed receipt 없이 새 + # runtime/root와 disabled tasks를 유지하고 trap이 별도 failed evidence를 쓴다. + throw "Fresh public promotion retained the new runtime after write release, but no atomic passed receipt was published. $($_.Exception.Message)" } + $freshFailureStage = "task_definition_cutover" + Assert-FreshPublicProvenanceContract ` + -SourceRoot $resolvedWorkspace ` + -SourceCommit $ExpectedSourceCommit ` + -SourceTree $ExpectedSourceTree ` + -PythonPath $Python ` + -PythonSha256 $ExpectedPythonSha256 ` + -CloudflaredPath $Cloudflared ` + -CloudflaredSha256 $ExpectedCloudflaredSha256 ` + -ConfigPath $resolvedCloudflaredConfig ` + -ConfigSha256 $ExpectedCloudflaredConfigSha256 + $bootInstallAction = { + & $bootTaskInstaller ` + -StableSourceRoot $resolvedWorkspace ` + -Python $Python ` + -Cloudflared $Cloudflared ` + -CloudflaredConfig $resolvedCloudflaredConfig ` + -UserUploadDir $resolvedUserUploadDir ` + -UserUploadManifestPath $resolvedUserUploadManifestPath ` + -ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 ` + -UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -TaskName "VignettePublicRuntime" ` + -InitiallyDisabled + }.GetNewClosure() + $watchdogInstallAction = { + & $watchdogTaskInstaller ` + -StableSourceRoot $resolvedWorkspace ` + -TaskName "VignettePublicRuntimeWatchdog" ` + -Python $Python ` + -UserUploadDir $resolvedUserUploadDir ` + -UserUploadManifestPath $resolvedUserUploadManifestPath ` + -ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 ` + -UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -Cloudflared $Cloudflared ` + -CloudflaredConfig $resolvedCloudflaredConfig ` + -PublicHealthUrl $CanonicalPublicHealthUrl ` + -InitiallyDisabled + }.GetNewClosure() + Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled ` + -BootInstaller $bootInstallAction ` + -WatchdogInstaller $watchdogInstallAction ` + -MaintenanceSnapshot $freshTaskMaintenanceSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + Assert-PublicRuntimeTasksDisabledAndIdle ` + -Snapshot $freshTaskMaintenanceSnapshot ` + -TimeoutSec $ProcessStopTimeoutSeconds + $freshNewDisabledTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot + $null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $freshNewDisabledTaskDefinitionSnapshot ` + -StableSourceRoot $resolvedWorkspace ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -PythonPath $Python ` + -UserUploadDir $resolvedUserUploadDir ` + -UserUploadManifestPath $resolvedUserUploadManifestPath ` + -ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 ` + -UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -CloudflaredPath $Cloudflared ` + -CloudflaredConfigPath $resolvedCloudflaredConfig ` + -PublicHealthUrl $CanonicalPublicHealthUrl + Assert-FreshPublicProvenanceContract ` + -SourceRoot $resolvedWorkspace ` + -SourceCommit $ExpectedSourceCommit ` + -SourceTree $ExpectedSourceTree ` + -PythonPath $Python ` + -PythonSha256 $ExpectedPythonSha256 ` + -CloudflaredPath $Cloudflared ` + -CloudflaredSha256 $ExpectedCloudflaredSha256 ` + -ConfigPath $resolvedCloudflaredConfig ` + -ConfigSha256 $ExpectedCloudflaredConfigSha256 + + $freshFailureStage = "task_maintenance_exit" + $freshOperationalTaskDefinitionSnapshot = Enable-NewPublicRuntimeTaskDefinitions ` + -DisabledSnapshot $freshNewDisabledTaskDefinitionSnapshot + $null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $freshOperationalTaskDefinitionSnapshot ` + -StableSourceRoot $resolvedWorkspace ` + -ExpectedSourceCommit $ExpectedSourceCommit ` + -ExpectedSourceTree $ExpectedSourceTree ` + -PythonPath $Python ` + -UserUploadDir $resolvedUserUploadDir ` + -UserUploadManifestPath $resolvedUserUploadManifestPath ` + -ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 ` + -UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -CloudflaredPath $Cloudflared ` + -CloudflaredConfigPath $resolvedCloudflaredConfig ` + -PublicHealthUrl $CanonicalPublicHealthUrl ` + -AllowEnabled + $restoredTaskTruth = Get-PublicRuntimeTaskMaintenanceState ` + -Snapshot $freshTaskMaintenanceSnapshot + if ( + -not [bool]$restoredTaskTruth.verified -or + -not [bool]$restoredTaskTruth.restored_to_snapshot + ) { + throw "Public runtime task maintenance exit could not prove the original task state" + } + $freshTasksRestored = $true + $freshFailureStage = "task_recovery_receipt_publish" + $runtimeReceiptSha256 = ( + Get-FileHash ` + -LiteralPath $resolvedRuntimeProvenancePath ` + -Algorithm SHA256 + ).Hash.ToLowerInvariant() + $taskRecoveryReceipt = [ordered]@{ + schema_version = "vignette.public-runtime-task-recovery.v1" + status = "passed" + operational_success = $true + captured_at_utc = (Get-Date).ToUniversalTime().ToString("o") + runtime_receipt_sha256 = $runtimeReceiptSha256 + preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount + preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount + required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount + required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount + current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount + current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount + task_definitions = [ordered]@{ + original_set_sha256 = [string]$freshOriginalTaskDefinitionSnapshot.set_sha256 + installed_disabled_set_sha256 = [string]$freshNewDisabledTaskDefinitionSnapshot.set_sha256 + operational_set_sha256 = [string]$freshOperationalTaskDefinitionSnapshot.set_sha256 + } + task_maintenance = [ordered]@{ + restored = $freshTasksRestored + state_verified = [bool]$restoredTaskTruth.verified + task_names = @($CoordinatedTaskNames | Sort-Object -Unique) + } + } + $null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent ` + -ExpectedSnapshot $freshOperationalTaskDefinitionSnapshot + Write-Utf8TextAtomically ` + -OutputPath $resolvedTaskRecoveryReceiptPath ` + -Value ((ConvertTo-Json -InputObject $taskRecoveryReceipt -Depth 5) + [Environment]::NewLine) + $freshTaskMaintenanceEntered = $false $freshPromotionCommitted = $true Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath" + Write-Output "Fresh public task recovery: $resolvedTaskRecoveryReceiptPath" } if ($engineReady) { @@ -1830,6 +3303,7 @@ if (!$SkipWebRestart) { } Write-Output "Health: $($health | ConvertTo-Json -Compress)" Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)" +Write-Output "User upload root: $resolvedUserUploadDir" } finally { if ($null -ne $recoveryLock) { try { diff --git a/scripts/test_initialize_public_runtime_upload_root.py b/scripts/test_initialize_public_runtime_upload_root.py new file mode 100644 index 0000000..d9d2a00 --- /dev/null +++ b/scripts/test_initialize_public_runtime_upload_root.py @@ -0,0 +1,675 @@ +from __future__ import annotations + +import importlib.util +import base64 +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path +from contextlib import redirect_stdout +from unittest.mock import patch + + +SCRIPT = Path(__file__).with_name("initialize-public-runtime-upload-root.py") +WRAPPER = Path(__file__).with_name("initialize-public-runtime-upload-root.ps1") +DATABASE_IDENTITY = Path(__file__).with_name("public_runtime_database_identity.py") + + +def _load_module(): + spec = importlib.util.spec_from_file_location("upload_initializer", SCRIPT) + if spec is None or spec.loader is None: + raise AssertionError("initializer module is not importable") + module = importlib.util.module_from_spec(spec) + script_parent = str(SCRIPT.parent) + inserted = script_parent not in sys.path + if inserted: + sys.path.insert(0, script_parent) + try: + spec.loader.exec_module(module) + finally: + if inserted: + sys.path.remove(script_parent) + return module + + +def _load_database_identity_module(): + spec = importlib.util.spec_from_file_location( + "public_runtime_database_identity_test", DATABASE_IDENTITY + ) + if spec is None or spec.loader is None: + raise AssertionError("database identity module is not importable") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class PublicRuntimeUploadInitializerTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.module = _load_module() + cls.database_identity = _load_database_identity_module() + + def test_connected_database_identity_is_stable_and_cross_target_distinct( + self, + ) -> None: + first = self.database_identity.database_target_sha256( + database_name="vignette", + database_role="vignette_app", + server_address="10.0.0.8", + server_port=5432, + ) + same = self.database_identity.database_target_sha256( + database_name="vignette", + database_role="vignette_app", + server_address="10.0.0.8", + server_port=5432, + ) + other_database = self.database_identity.database_target_sha256( + database_name="vignette_shadow", + database_role="vignette_app", + server_address="10.0.0.8", + server_port=5432, + ) + other_server = self.database_identity.database_target_sha256( + database_name="vignette", + database_role="vignette_app", + server_address="10.0.0.9", + server_port=5432, + ) + self.assertEqual(first, same) + self.assertRegex(first, r"^[a-f0-9]{64}$") + self.assertNotEqual(first, other_database) + self.assertNotEqual(first, other_server) + + def test_reference_inventory_deduplicates_objects_but_counts_rows(self) -> None: + one = "/uploads/profile-avatars/00000000-0000-0000-0000-000000000001-a.png" + two = "/uploads/profile-avatars/00000000-0000-0000-0000-000000000002-b.webp" + inventory = self.module.build_reference_inventory( + [one, one, two], expected_reference_count=3 + ) + self.assertEqual(3, inventory.reference_count) + self.assertEqual(2, inventory.unique_object_count) + self.assertRegex(inventory.reference_set_sha256, r"^[a-f0-9]{64}$") + + def test_zero_database_references_are_valid_and_digest_bound(self) -> None: + inventory = self.module.build_reference_inventory( + [], expected_reference_count=0 + ) + self.assertEqual(0, inventory.reference_count) + self.assertEqual(0, inventory.unique_object_count) + self.assertRegex(inventory.reference_set_sha256, r"^[a-f0-9]{64}$") + + def test_reference_validation_rejects_encoded_control_and_bad_extensions( + self, + ) -> None: + invalid = ( + "/uploads/profile-avatars/name%2Fescape.png", + "/uploads/profile-avatars/name.exe", + "/uploads/profile-avatars/name\x1f.png", + "/uploads/profile-avatars/nested/name.png", + "/uploads/profile-avatars/name.png/", + "/uploads//profile-avatars/name.png", + "/uploads/other/name.png", + ) + for value in invalid: + with self.subTest(value=repr(value)): + with self.assertRaises(self.module.InitializationError): + self.module.normalize_avatar_reference(value) + + def test_copy_is_create_only_hash_verified_and_conflicts_fail(self) -> None: + name = "profile-avatars/00000000-0000-0000-0000-000000000001-a.png" + url = f"/uploads/{name}" + with tempfile.TemporaryDirectory(prefix="vignette-upload-init-") as raw: + root = Path(raw) + source = root / "source" + target = root / "target" + (source / "profile-avatars").mkdir(parents=True) + target.mkdir() + (source / name).write_bytes(b"source-avatar") + inventory = self.module.scan_preserved_inventory([source]) + + first = self.module.copy_required_objects( + inventory=inventory, source_roots=[source], upload_root=target + ) + self.assertEqual(1, first.copied_count) + self.assertEqual(b"source-avatar", (target / name).read_bytes()) + + second = self.module.copy_required_objects( + inventory=inventory, source_roots=[source], upload_root=target + ) + self.assertEqual(1, second.reused_exact_count) + + (target / name).write_bytes(b"different") + with self.assertRaisesRegex( + self.module.InitializationError, + "target_inventory_not_empty_or_exact", + ): + self.module.copy_required_objects( + inventory=inventory, + source_roots=[source], + upload_root=target, + ) + + def test_source_conflict_and_missing_are_fail_closed(self) -> None: + name = "profile-avatars/00000000-0000-0000-0000-000000000001-a.jpg" + url = f"/uploads/{name}" + with tempfile.TemporaryDirectory(prefix="vignette-upload-source-") as raw: + root = Path(raw) + first = root / "first" + second = root / "second" + target = root / "target" + for directory in (first, second, target): + directory.mkdir() + (first / "profile-avatars").mkdir() + (second / "profile-avatars").mkdir() + (first / name).write_bytes(b"one") + (second / name).write_bytes(b"two") + with self.assertRaisesRegex( + self.module.InitializationError, "source_object_conflict" + ): + self.module.scan_preserved_inventory([first, second]) + + (first / name).unlink() + (second / name).unlink() + inventory = self.module.build_reference_inventory( + [url], expected_reference_count=1 + ) + preserved = self.module.scan_preserved_inventory([first, second]) + with self.assertRaisesRegex( + self.module.InitializationError, "source_object_missing" + ): + self.module.assert_database_references_preserved( + references=inventory, + preserved=preserved, + ) + + def test_explicit_source_union_preserves_93_objects_for_8_db_refs(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-union-") as raw: + root = Path(raw) + sources = [root / f"source-{index}" for index in range(3)] + for source in sources: + (source / "profile-avatars").mkdir(parents=True) + names = [f"avatar-{index:03d}.png" for index in range(93)] + for index, name in enumerate(names): + destination = sources[index % len(sources)] / "profile-avatars" / name + destination.write_bytes(f"avatar-{index}".encode("ascii")) + # 동일 상대 경로/동일 내용은 여러 명시 root에 있어도 하나로 보존한다. + duplicate = sources[1] / "profile-avatars" / names[0] + duplicate.write_bytes(b"avatar-0") + references = self.module.build_reference_inventory( + [f"/uploads/profile-avatars/{name}" for name in names[:8]], + expected_reference_count=8, + ) + preserved = self.module.scan_preserved_inventory(sources) + self.module.assert_database_references_preserved( + references=references, + preserved=preserved, + ) + self.assertEqual(93, preserved.object_count) + self.assertEqual( + sum(len(f"avatar-{index}".encode("ascii")) for index in range(93)), + preserved.total_size_bytes, + ) + self.assertEqual(8, references.reference_count) + self.assertRegex(preserved.inventory_sha256, r"^[a-f0-9]{64}$") + self.module.assert_expected_preserved_inventory( + inventory=preserved, + expected_object_count=93, + expected_total_size_bytes=preserved.total_size_bytes, + expected_inventory_sha256=preserved.inventory_sha256, + ) + with self.assertRaisesRegex( + self.module.InitializationError, + "preserved_inventory_pin_mismatch", + ): + self.module.assert_expected_preserved_inventory( + inventory=preserved, + expected_object_count=92, + expected_total_size_bytes=preserved.total_size_bytes, + expected_inventory_sha256=preserved.inventory_sha256, + ) + + def test_preserved_inventory_rejects_invalid_nested_and_changed_sources(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-stability-") as raw: + source = Path(raw) / "source" + avatar_root = source / "profile-avatars" + avatar_root.mkdir(parents=True) + avatar = avatar_root / "valid.png" + avatar.write_bytes(b"before") + expected = self.module.scan_preserved_inventory([source]) + + avatar.write_bytes(b"after") + actual = self.module.scan_preserved_inventory([source]) + with self.assertRaisesRegex( + self.module.InitializationError, + "source_inventory_changed_during_copy", + ): + self.module.assert_preserved_inventory_stable( + expected=expected, + actual=actual, + ) + + avatar.unlink() + (avatar_root / "nested").mkdir() + with self.assertRaisesRegex( + self.module.InitializationError, + "source_inventory_entry_invalid", + ): + self.module.scan_preserved_inventory([source]) + + def test_preserved_inventory_probe_requires_caller_pin_without_raw_paths(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-probe-") as raw: + source = Path(raw) / "private-user-source" + avatar_root = source / "profile-avatars" + avatar_root.mkdir(parents=True) + (avatar_root / "private-avatar-name.png").write_bytes(b"avatar") + inventory = self.module.scan_preserved_inventory([source]) + output = io.StringIO() + with redirect_stdout(output): + exit_code = self.module._verify_preserved_inventory_cli( + [ + "--source-root", + str(source), + "--expected-preserved-object-count", + "1", + "--expected-preserved-total-size-bytes", + str(inventory.total_size_bytes), + "--expected-preserved-inventory-sha256", + inventory.inventory_sha256, + ] + ) + payload = json.loads(output.getvalue()) + self.assertEqual(0, exit_code) + self.assertEqual("verified", payload["status"]) + self.assertEqual( + inventory.total_size_bytes, + payload["preserved_total_size_bytes"], + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn(str(source), serialized) + self.assertNotIn("private-avatar-name.png", serialized) + + def test_failed_copy_cleanup_preserves_a_replacement_after_handle_close(self) -> None: + with tempfile.TemporaryDirectory(prefix="vignette-upload-cleanup-race-") as raw: + root = Path(raw) + source = root / "source.bin" + target = root / "target.bin" + source.write_bytes(b"source-avatar") + expected_size, expected_sha256 = self.module._hash_regular_file(source) + + def replace_after_close(*_args, **_kwargs) -> bool: + target.unlink() + target.write_bytes(b"replacement-owned-by-other-process") + return False + + with patch.object( + self.module, "_target_matches", side_effect=replace_after_close + ): + with self.assertRaisesRegex( + self.module.InitializationError, + "target_object_verification_failed", + ): + self.module._copy_one_create_only( + source=source, + target=target, + expected_size=expected_size, + expected_sha256=expected_sha256, + ) + + self.assertEqual(b"replacement-owned-by-other-process", target.read_bytes()) + + def test_active_private_audio_always_requires_a_separate_migration(self) -> None: + self.module.assert_no_active_private_audio(0) + for count in (1, 5): + with self.subTest(count=count): + with self.assertRaisesRegex( + self.module.InitializationError, + "active_private_audio_requires_separate_migration", + ): + self.module.assert_no_active_private_audio(count) + + def test_offline_capture_rejects_forged_legacy_source_pins(self) -> None: + commit = "a" * 40 + tree = "b" * 40 + identity = { + "pid": 42, + "started_at_utc": "2026-08-29T00:00:00Z", + "executable_sha256": "c" * 64, + "command_line_sha256": "d" * 64, + "cwd_sha256": "e" * 64, + } + with tempfile.TemporaryDirectory(prefix="vignette-offline-source-") as raw: + source = Path(raw).resolve() + capture = { + "schema_version": self.module.OFFLINE_CAPTURE_SCHEMA_VERSION, + "status": "quiesced", + "captured_at_utc": "2026-08-29T00:00:01Z", + "source_commit": commit, + "source_tree": tree, + "source_root_sha256s": [ + self.module.canonical_path_sha256(source) + ], + "source_root_set_sha256": self.module._source_root_bindings( + [source] + )[1], + "api_identity": identity, + "tunnel_identity": identity, + "listener_absent": True, + "tunnel_absent": True, + "listener_endpoint_sha256": "f" * 64, + "tunnel_config_sha256": "1" * 64, + } + encoded = base64.b64encode( + json.dumps(capture, separators=(",", ":")).encode("utf-8") + ).decode("ascii") + decoded = self.module.decode_offline_quiescence_capture( + encoded, + legacy_source_roots=[source], + expected_source_commit=commit, + expected_source_tree=tree, + ) + self.assertEqual(commit, decoded["source_commit"]) + with self.assertRaisesRegex( + self.module.InitializationError, + "offline_quiescence_capture_invalid", + ): + self.module.decode_offline_quiescence_capture( + encoded, + legacy_source_roots=[source], + expected_source_commit="9" * 40, + expected_source_tree=tree, + ) + + def test_offline_receipt_is_private_hash_bound_and_path_free(self) -> None: + commit = "a" * 40 + tree = "b" * 40 + identity = { + "pid": 42, + "started_at_utc": "2026-08-29T00:00:00Z", + "executable_sha256": "c" * 64, + "command_line_sha256": "d" * 64, + "cwd_sha256": "e" * 64, + } + with tempfile.TemporaryDirectory(prefix="vignette-offline-receipt-") as raw: + root = Path(raw).resolve() + source = root / "legacy-secret-user" + source_two = root / "legacy-secret-user-two" + source_three = root / "legacy-secret-user-three" + upload = root / "public" + state = root / "private" + for directory in ( + source, + source_two, + source_three, + upload, + state, + ): + directory.mkdir() + (source / "profile-avatars").mkdir() + (source / "profile-avatars" / "private-name.png").write_bytes( + b"avatar" + ) + inventory = self.module.build_reference_inventory( + ["/uploads/profile-avatars/private-name.png"], + expected_reference_count=1, + ) + source_roots = [source, source_two, source_three] + source_root_sha256s, source_root_set_sha256 = ( + self.module._source_root_bindings(source_roots) + ) + capture = { + "captured_at_utc": "2026-08-29T00:00:01Z", + "source_commit": commit, + "source_tree": tree, + "source_root_sha256s": list(source_root_sha256s), + "source_root_set_sha256": source_root_set_sha256, + "api_identity": identity, + "tunnel_identity": identity, + "listener_endpoint_sha256": "f" * 64, + "tunnel_config_sha256": "1" * 64, + } + preserved = self.module.scan_preserved_inventory(source_roots) + invalid_objects, invalid_references = ( + self.module.required_decode_invalid_counts( + references=inventory, + preserved=preserved, + ) + ) + payload = self.module.build_offline_quiescence_receipt( + capture=capture, + inventory=inventory, + preserved=preserved, + database_target_digest="2" * 64, + ) + receipt, digest = self.module.write_offline_quiescence_receipt_create_only( + manifest_state_dir=state, + payload=payload, + ) + proof = self.module.validate_offline_quiescence_receipt( + receipt_path=receipt, + expected_receipt_sha256=digest, + stable_source_root=source, + upload_root=upload, + expected_source_commit=commit, + expected_source_tree=tree, + expected_source_roots=source_roots, + expected_database_target_sha256="2" * 64, + expected_preserved_object_count=1, + expected_preserved_total_size_bytes=preserved.total_size_bytes, + expected_preserved_inventory_sha256=preserved.inventory_sha256, + expected_preserved_decode_valid_count=preserved.decode_valid_count, + expected_preserved_decode_invalid_count=( + preserved.decode_invalid_count + ), + expected_required_decode_invalid_object_count=invalid_objects, + expected_required_decode_invalid_reference_count=( + invalid_references + ), + expected_reference_count=1, + expected_unique_object_count=1, + expected_reference_set_sha256=inventory.reference_set_sha256, + ) + hash_only_proof = self.module.validate_offline_quiescence_receipt( + receipt_path=receipt, + expected_receipt_sha256=digest, + stable_source_root=source, + upload_root=upload, + expected_source_commit=commit, + expected_source_tree=tree, + expected_source_roots=None, + expected_source_root_sha256s=source_root_sha256s, + expected_source_root_set_sha256=source_root_set_sha256, + expected_database_target_sha256="2" * 64, + expected_preserved_object_count=1, + expected_preserved_total_size_bytes=preserved.total_size_bytes, + expected_preserved_inventory_sha256=preserved.inventory_sha256, + expected_preserved_decode_valid_count=preserved.decode_valid_count, + expected_preserved_decode_invalid_count=( + preserved.decode_invalid_count + ), + expected_required_decode_invalid_object_count=invalid_objects, + expected_required_decode_invalid_reference_count=( + invalid_references + ), + expected_reference_count=1, + expected_unique_object_count=1, + expected_reference_set_sha256=inventory.reference_set_sha256, + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertRegex(proof.receipt_sha256, r"^[a-f0-9]{64}$") + self.assertEqual(tuple(source_root_sha256s), proof.source_root_sha256s) + self.assertEqual(source_root_set_sha256, proof.source_root_set_sha256) + self.assertEqual(proof.receipt_sha256, hash_only_proof.receipt_sha256) + self.assertNotIn(str(root), serialized) + self.assertNotIn("private-name.png", serialized) + self.assertNotIn("/uploads/", serialized) + + def test_runtime_freeze_requires_exact_loopback_health_and_token(self) -> None: + token_sha256 = "b" * 64 + health = json.dumps( + { + "upload_write_freeze": { + "capable": True, + "active": True, + "valid": True, + "in_flight": 0, + "token_sha256": token_sha256, + } + } + ).encode("utf-8") + + class Response: + def __enter__(self): + return self + + def __exit__(self, *unused): + return False + + def read(self): + return health + + class Opener: + def open(self, request, timeout): + return Response() + + with patch.object( + self.module.urllib.request, + "build_opener", + return_value=Opener(), + ): + self.module.wait_for_drained_runtime_freeze( + health_url="http://127.0.0.1:8001/health", + token_sha256=token_sha256, + timeout_seconds=0.1, + ) + with self.assertRaisesRegex( + self.module.InitializationError, "runtime_health_url_invalid" + ): + self.module.wait_for_drained_runtime_freeze( + health_url="https://example.invalid/health", + token_sha256=token_sha256, + timeout_seconds=0.1, + ) + + def test_manifest_and_result_are_privacy_safe(self) -> None: + name = "profile-avatars/00000000-0000-0000-0000-000000000001-secret.png" + url = f"/uploads/{name}" + with tempfile.TemporaryDirectory(prefix="vignette-upload-manifest-") as raw: + root = Path(raw) + upload = root / "upload" + state = root / "state" + freeze = state / "freeze.json" + (upload / "profile-avatars").mkdir(parents=True) + state.mkdir() + (upload / name).write_bytes(b"avatar") + freeze.write_text("{}", encoding="utf-8") + inventory = self.module.build_reference_inventory( + [url, url], expected_reference_count=2 + ) + preserved = self.module.scan_preserved_inventory([upload]) + payload = self.module.privacy_safe_manifest( + upload_root=upload, + inventory=inventory, + preserved=preserved, + database_target_sha256="b" * 64, + freeze_token_sha256="a" * 64, + freeze_path=freeze, + ) + _, digest = self.module.write_manifest_create_only( + manifest_state_dir=state, payload=payload + ) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn("secret.png", serialized) + self.assertNotIn("00000000-0000", serialized) + self.assertNotIn("/uploads/", serialized) + self.assertNotIn(str(root), serialized) + self.assertRegex(digest, r"^[a-f0-9]{64}$") + contract = payload["database_reference_contract"] + self.assertEqual(2, contract["reference_count"]) + self.assertEqual(1, contract["unique_object_count"]) + self.assertEqual(2, payload["required_reference_count"]) + self.assertEqual( + 2, payload["preserved_objects"][0]["reference_count"] + ) + self.assertEqual(1, payload["preserved_object_count"]) + self.assertEqual( + preserved.inventory_sha256, + payload["preserved_object_set_sha256"], + ) + self.assertEqual("b" * 64, payload["database_target_sha256"]) + self.assertEqual( + inventory.reference_set_sha256, payload["reference_set_sha256"] + ) + self.assertEqual( + payload["reference_set_sha256"], contract["reference_set_sha256"] + ) + self.assertEqual( + [ + { + "path_sha256": payload["preserved_objects"][0]["path_sha256"], + "reference_count": 2, + } + ], + contract["objects"], + ) + result = self.module.privacy_safe_result( + manifest_path=state / f"public-avatar-upload-{digest}.json", + manifest_sha256=digest, + upload_root=upload, + inventory=inventory, + preserved=preserved, + copy_proof=self.module.CopyProof(1, 1, 0), + freeze_token_sha256="a" * 64, + ) + serialized_result = json.dumps(result, sort_keys=True) + self.assertNotIn("secret.png", serialized_result) + self.assertNotIn(str(root), serialized_result) + self.assertEqual( + { + "status", + "manifest_sha256", + "manifest_path_sha256", + "root_path_sha256", + "preserved_object_count", + "preserved_total_size_bytes", + "preserved_inventory_sha256", + "preserved_decode_valid_count", + "preserved_decode_invalid_count", + "required_decode_invalid_object_count", + "required_decode_invalid_reference_count", + "required_object_count", + "database_reference_count", + "database_reference_set_sha256", + "copied_object_count", + "reused_exact_object_count", + "write_freeze_token_sha256", + }, + set(result), + ) + + def test_wrapper_owns_creation_and_freeze_create_new(self) -> None: + source = WRAPPER.read_text(encoding="utf-8") + self.assertIn("-CreateIfMissing", source) + self.assertIn("[System.IO.FileMode]::CreateNew", source) + self.assertIn("ExpectedReferenceCount", source) + self.assertIn("ExpectedPreservedObjectCount", source) + self.assertIn("ExpectedPreservedTotalSizeBytes", source) + self.assertIn("ExpectedPreservedInventorySha256", source) + self.assertIn('"--expected-preserved-object-count"', source) + self.assertIn('"--expected-preserved-total-size-bytes"', source) + self.assertIn('"--expected-preserved-inventory-sha256"', source) + self.assertIn("parsedOutput.preserved_total_size_bytes", source) + self.assertIn("Remove-OwnedWriteFreeze", source) + self.assertIn("Assert-StableInitializerSourceProvenance", source) + self.assertIn("symbolic-ref -q HEAD", source) + self.assertIn("status --porcelain=v1 --untracked-files=all", source) + worker = SCRIPT.read_text(encoding="utf-8") + self.assertIn('open("xb")', worker) + self.assertNotIn("shutil.move", worker) + self.assertNotIn("os.replace", worker) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_legacy_public_runtime_upload_bootstrap.py b/scripts/test_legacy_public_runtime_upload_bootstrap.py new file mode 100644 index 0000000..300f441 --- /dev/null +++ b/scripts/test_legacy_public_runtime_upload_bootstrap.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import os +import subprocess +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +BOOTSTRAP = SCRIPTS / "bootstrap-legacy-public-runtime-upload-root.ps1" +START = SCRIPTS / "start-public-runtime.ps1" +INITIALIZER = SCRIPTS / "initialize-public-runtime-upload-root.ps1" + + +class LegacyPublicRuntimeUploadBootstrapContractTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.bootstrap = BOOTSTRAP.read_text(encoding="utf-8") + cls.start = START.read_text(encoding="utf-8") + cls.initializer = INITIALIZER.read_text(encoding="utf-8") + + def test_success_state_machine_has_exact_irreversible_order(self) -> None: + markers = ( + "LEGACY_BOOTSTRAP_STAGE:task_maintenance_enter", + "LEGACY_BOOTSTRAP_STAGE:exact_runtime_capture", + "LEGACY_BOOTSTRAP_STAGE:tunnel_quiescence", + "LEGACY_BOOTSTRAP_STAGE:listener_quiescence", + "LEGACY_BOOTSTRAP_STAGE:offline_initializer", + "LEGACY_BOOTSTRAP_STAGE:new_api_frozen", + "LEGACY_BOOTSTRAP_STAGE:new_tunnel_public_frozen", + "LEGACY_BOOTSTRAP_STAGE:no_rollback_boundary", + "LEGACY_BOOTSTRAP_STAGE:write_release", + "LEGACY_BOOTSTRAP_STAGE:cutover_receipt_publish", + "LEGACY_BOOTSTRAP_STAGE:task_maintenance_exit", + "LEGACY_BOOTSTRAP_STAGE:task_recovery_receipt_publish", + ) + positions = [self.bootstrap.index(marker) for marker in markers] + self.assertEqual(sorted(positions), positions) + boundary = self.bootstrap.index("$noRollback = $true", positions[6]) + release = self.bootstrap.index("Remove-OwnedFreezeByHash", boundary) + self.assertLess(boundary, release) + + def test_bootstrap_holds_one_inherited_lock_and_competing_start_cannot_enter( + self, + ) -> None: + acquire = self.bootstrap.index("Enter-InheritedBootstrapRecoveryLock") + task_enter = self.bootstrap.index("LEGACY_BOOTSTRAP_STAGE:task_maintenance_enter") + start_call = self.bootstrap.index("& $startScript") + dispose = self.bootstrap.rindex("$bootstrapLock.Stream.Dispose()") + self.assertLess(acquire, task_enter) + self.assertLess(start_call, dispose) + for marker in ( + "[System.IO.FileShare]::Read", + "-InheritedRecoveryLockReceiptPath $bootstrapLock.Path", + "-ExpectedInheritedRecoveryLockReceiptSha256 $bootstrapLock.ReceiptSha256", + ): + self.assertIn(marker, self.bootstrap) + self.assertLess( + self.bootstrap.index("Legacy preserved source inventory preflight failed"), + self.bootstrap.index("LEGACY_BOOTSTRAP_STAGE:task_maintenance_enter"), + ) + self.assertIn("Inherited recovery lock is not held", self.start) + self.assertIn("[System.IO.FileShare]::None", self.start) + self.assertIn("[int]$payload.owner_pid -ne $PID", self.start) + + def test_quiescence_race_is_rechecked_around_initializer_and_api_start(self) -> None: + self.assertGreaterEqual( + self.bootstrap.count("Assert-OfflinedRuntimeRaceGate"), 4 + ) + initializer = self.bootstrap.index("& $initializer") + api_start = self.bootstrap.index("& $startScript") + race_positions = [] + offset = 0 + while True: + found = self.bootstrap.find("Assert-OfflinedRuntimeRaceGate", offset) + if found < 0: + break + race_positions.append(found) + offset = found + 1 + self.assertTrue(any(position < initializer for position in race_positions)) + self.assertTrue(any(initializer < position < api_start for position in race_positions)) + self.assertIn("Assert-PublicRuntimeTasksDisabledAndIdle", self.bootstrap) + self.assertIn("Assert-LoopbackListenerAbsent", self.bootstrap) + self.assertIn("Assert-TunnelAbsent", self.bootstrap) + + def test_rollback_stops_only_exact_owned_replacements_tunnel_first(self) -> None: + restore = self.bootstrap[ + self.bootstrap.index("function Restore-LegacyRuntime") : + self.bootstrap.index("$resolvedStableSourceRoot =", self.bootstrap.index("function Restore-LegacyRuntime")) + ] + tunnel = restore.index('Role "owned new tunnel"') + api = restore.index('Role "owned new API"') + self.assertLess(tunnel, api) + self.assertIn("Rollback refuses to stop an unowned tunnel", restore) + self.assertIn("Rollback refuses to stop an unowned API listener", restore) + self.assertNotIn("foreach ($tunnel in $tunnelMatches)", restore) + self.assertIn("-OwnedNewApi $newApiIdentity", self.bootstrap) + self.assertIn("-OwnedNewTunnel $newTunnelIdentity", self.bootstrap) + + def test_pre_boundary_restores_then_enables_post_boundary_disables(self) -> None: + catch = self.bootstrap[ + self.bootstrap.index("} catch {\n $rollbackSucceeded") : + ] + restore = catch.index("Restore-LegacyRuntime") + task_exit = catch.index("Exit-PublicRuntimeTaskMaintenance") + self.assertLess(restore, task_exit) + self.assertIn("if (-not $noRollback)", catch) + self.assertIn("Suspend-PublicRuntimeTasks", catch) + self.assertIn("tasks_remain_disabled = [bool]$taskTruth.all_disabled_and_idle", catch) + self.assertIn("cutover_receipt_published", catch) + + def test_prior_effective_dotenv_and_new_pid_environment_are_hash_bound(self) -> None: + for marker in ( + 'Join-Path ([string]$Identity.cwd) ".env"', + 'check-ignore --quiet -- "apps/api/.env"', + "Set-CompleteProcessEnvironment -Environment $Identity.environment", + "from app.config import settings as s", + "Assert-NewApiEnvironmentMatches", + "Get-ConnectedDatabaseTargetSha256", + "manifestProof.Payload.database_target_sha256", + "required_environment_sha256 = $requiredEnvironmentDigest", + ): + self.assertIn(marker, self.bootstrap) + receipt = self.bootstrap[ + self.bootstrap.index("$cutoverReceipt =") : + self.bootstrap.index("Write-PrivacySafeReceiptCreateOnly", self.bootstrap.index("$cutoverReceipt =")) + ] + self.assertNotIn("DATABASE_URL =", receipt) + self.assertNotIn("OAUTH_GOOGLE_CLIENT_SECRET =", receipt) + self.assertNotIn("SESSION_SECRET =", receipt) + + def test_google_only_contract_is_checked_local_public_frozen_and_unfrozen(self) -> None: + self.assertGreaterEqual(self.bootstrap.count("Wait-GoogleAuthContract"), 5) + self.assertIn('$enabledProviders[0] -eq "google"', self.bootstrap) + self.assertIn("$config.dev_login_enabled -eq $false", self.bootstrap) + self.assertGreaterEqual(self.bootstrap.count("Assert-CurrentTunnelIdentity"), 3) + self.assertIn("public_unfrozen = $true", self.bootstrap) + + def test_legacy_upload_root_falls_back_only_when_variable_is_absent(self) -> None: + source_root = self.bootstrap[ + self.bootstrap.index("function Get-ValidatedLegacySourceRoots") : + self.bootstrap.index("function Restore-LegacyRuntime") + ] + present = source_root.index("$declaredPresent = $true") + empty_abort = source_root.index( + 'throw "Legacy USER_UPLOAD_DIR is present but empty"', present + ) + fallback = source_root.index("if (-not $declaredPresent)", empty_abort) + self.assertLess(present, empty_abort) + self.assertLess(empty_abort, fallback) + for marker in ( + "$resolvedAllowedRoots += $resolvedAllowed", + 'throw "Allowed legacy upload roots contain a duplicate"', + "source_root_sha256s = @($sourceRootSha256s)", + "source_root_set_sha256 = Get-Utf8Sha256", + "vignette.public-upload-offline-quiescence-capture.v2", + "-SourceUploadDir $legacySourceRoots", + "Get-ValidatedExplicitLegacySourceRoots", + "Legacy bootstrap requires exactly three explicit source roots", + '"probe-preserved-inventory"', + "-ExpectedPreservedObjectCount $ExpectedPreservedObjectCount", + "-ExpectedPreservedInventorySha256 $ExpectedPreservedInventorySha256", + "Legacy preserved source inventory preflight failed", + 'Test-ArgumentPair -Arguments $arguments -Name "--workers" -Value "1"', + "initializerPayload.preserved_inventory_sha256", + "initializerPayload.preserved_total_size_bytes", + "preservedProbeDecodeCounts.ValidCount", + "initializerPreservedDecodeCounts.ValidCount", + "initializerRequiredDecodeCounts.ObjectCount", + "manifestProof.Payload.preserved_object_set_sha256", + "manifestProof.Payload.preserved_total_size_bytes", + "manifestPreservedDecodeCounts.ValidCount", + "manifestRequiredDecodeCounts.ObjectCount", + "manifestCurrentDecodeCounts.ObjectCount", + "Offline initializer preserved inventory proof drifted", + "Offline initializer decode proof drifted across preflight, manifest, or current DB", + '"--expected-preserved-total-size-bytes"', + "preservedProbePayload.preserved_total_size_bytes", + "-ExpectedPreservedTotalSizeBytes $ExpectedPreservedTotalSizeBytes", + ): + self.assertIn(marker, self.bootstrap) + self.assertNotIn( + "Offline quiescence initialization requires exactly one validated source root", + self.initializer, + ) + + def test_total_size_pin_is_recorded_in_all_success_receipts(self) -> None: + total_size_field = ( + "preserved_total_size_bytes = " + "[long]$manifestProof.Payload.preserved_total_size_bytes" + ) + cutover = self.bootstrap[ + self.bootstrap.index("$cutoverReceipt =") : + self.bootstrap.index( + "Write-PrivacySafeReceiptCreateOnly", + self.bootstrap.index("$cutoverReceipt ="), + ) + ] + task_recovery = self.bootstrap[ + self.bootstrap.index("$taskRecoveryReceipt =") : + self.bootstrap.index( + "Write-PrivacySafeReceiptCreateOnly", + self.bootstrap.index("$taskRecoveryReceipt ="), + ) + ] + final_result = self.bootstrap[ + self.bootstrap.rindex("$result = [ordered]@{") : + ] + self.assertIn(total_size_field, cutover) + self.assertIn(total_size_field, task_recovery) + self.assertIn(total_size_field, final_result) + + def test_decode_counts_are_validated_and_recorded_in_all_success_receipts( + self, + ) -> None: + for expected in ( + "function Get-RequiredPrivacySafeCount", + "function Get-PreservedDecodeCountProof", + "function Get-RequiredDecodeInvalidCountProof", + "function Get-CurrentDecodeInvalidCountProof", + 'Name "preserved_decode_valid_count"', + 'Name "preserved_decode_invalid_count"', + 'Name "required_decode_invalid_object_count"', + 'Name "required_decode_invalid_reference_count"', + 'Name "current_decode_invalid_object_count"', + 'Name "current_decode_invalid_reference_count"', + ): + with self.subTest(expected=expected): + self.assertIn(expected, self.bootstrap) + + proof_compare = self.bootstrap[ + self.bootstrap.index("$initializerRequiredObjectCount =") : + self.bootstrap.index("LEGACY_BOOTSTRAP_STAGE:new_api_frozen") + ] + for expected in ( + "$preservedProbeDecodeCounts.ValidCount", + "$initializerPreservedDecodeCounts.ValidCount", + "$initializerRequiredDecodeCounts.ObjectCount", + "$manifestPreservedDecodeCounts.ValidCount", + "$manifestRequiredDecodeCounts.ObjectCount", + "$manifestCurrentDecodeCounts.ObjectCount", + ): + with self.subTest(proof=expected): + self.assertIn(expected, proof_compare) + + cutover = self.bootstrap[ + self.bootstrap.index("$cutoverReceipt =") : + self.bootstrap.index( + "Write-PrivacySafeReceiptCreateOnly", + self.bootstrap.index("$cutoverReceipt ="), + ) + ] + task_recovery = self.bootstrap[ + self.bootstrap.index("$taskRecoveryReceipt =") : + self.bootstrap.index( + "Write-PrivacySafeReceiptCreateOnly", + self.bootstrap.index("$taskRecoveryReceipt ="), + ) + ] + final_result = self.bootstrap[self.bootstrap.rindex("$result = [ordered]@{") :] + receipt_fields = { + "preserved_decode_valid_count": "manifestPreservedDecodeCounts.ValidCount", + "preserved_decode_invalid_count": "manifestPreservedDecodeCounts.InvalidCount", + "required_decode_invalid_object_count": "manifestRequiredDecodeCounts.ObjectCount", + "required_decode_invalid_reference_count": "manifestRequiredDecodeCounts.ReferenceCount", + "current_decode_invalid_object_count": "manifestCurrentDecodeCounts.ObjectCount", + "current_decode_invalid_reference_count": "manifestCurrentDecodeCounts.ReferenceCount", + } + for receipt_name, receipt in ( + ("cutover", cutover), + ("task_recovery", task_recovery), + ("final", final_result), + ): + for field, source in receipt_fields.items(): + with self.subTest(receipt=receipt_name, field=field): + self.assertIn(f"{field} = [int]${source}", receipt) + + def test_initializer_online_cleanup_and_offline_cleanup_are_separate(self) -> None: + self.assertIn("Assert-OnlineUploadWritesRecovered", self.initializer) + self.assertIn("$freeze.active -eq $false", self.initializer) + self.assertIn("$freeze.valid -eq $true", self.initializer) + self.assertIn("[int]$freeze.in_flight -eq 0", self.initializer) + self.assertIn("if (-not $offlineQuiescenceMode)", self.initializer) + + @unittest.skipUnless(os.name == "nt", "PowerShell 5.1 is Windows-only") + def test_all_bootstrap_contract_scripts_parse_in_windows_powershell_51(self) -> None: + powershell = Path(os.environ["SystemRoot"]) / "System32/WindowsPowerShell/v1.0/powershell.exe" + files = (BOOTSTRAP, START, INITIALIZER) + quoted = ",".join("'" + str(path).replace("'", "''") + "'" for path in files) + command = ( + "$ErrorActionPreference='Stop';" + f"$files=@({quoted});" + "foreach($file in $files){$tokens=$null;$errors=$null;" + "[void][System.Management.Automation.Language.Parser]::ParseFile($file,[ref]$tokens,[ref]$errors);" + "if($errors.Count -gt 0){exit 7}};exit 0" + ) + result = subprocess.run( + [str(powershell), "-NoProfile", "-Command", command], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(0, result.returncode, msg=result.stderr) + self.assertNotIn("$pid =", self.bootstrap.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_environment_handoff.py b/scripts/test_public_runtime_environment_handoff.py new file mode 100644 index 0000000..79dc799 --- /dev/null +++ b/scripts/test_public_runtime_environment_handoff.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +BOOTSTRAP = SCRIPTS / "bootstrap-legacy-public-runtime-upload-root.ps1" + + +def _function(source: str, name: str, next_name: str) -> str: + start = source.index(f"function {name}") + end = source.index(f"function {next_name}", start) + return source[start:end] + + +class PublicRuntimeEnvironmentHandoffTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.source = BOOTSTRAP.read_text(encoding="utf-8") + cls.functions = "\n".join( + ( + _function( + cls.source, + "Save-CompleteProcessEnvironment", + "Get-IdentityEnvironmentValue", + ), + _function( + cls.source, + "Invoke-EffectiveApiSettingsProbe", + "Set-RequiredApiEnvironmentFromIdentity", + ), + _function( + cls.source, + "Get-FutureLauncherRequiredApiSettings", + "Assert-RequiredApiSettingsEqual", + ), + _function( + cls.source, + "Assert-RequiredApiSettingsEqual", + "Assert-EnvironmentFilePinned", + ), + _function( + cls.source, + "Set-CompleteProcessEnvironment", + "ConvertTo-WindowsCommandLineArgument", + ), + ) + ) + + def _powershell(self) -> str: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + return powershell + + @staticmethod + def _write_fake_api(api_root: Path, *, client_secret: str) -> None: + package = api_root / "app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "config.py").write_text( + """import json +import os +from pathlib import Path + + +values = {} +for line in Path('.env').read_text(encoding='utf-8').splitlines(): + if line and not line.lstrip().startswith('#') and '=' in line: + key, value = line.split('=', 1) + values[key.strip()] = value.strip() + + +def get(name, default=''): + return os.environ.get(name, values.get(name, default)) + + +class Secret: + def __init__(self, value): self.value = value + def get_secret_value(self): return self.value + + +class Settings: + database_url = get('DATABASE_URL') + session_secret = get('SESSION_SECRET') + engine_gateway_shared_secret = Secret(get('ENGINE_GATEWAY_SHARED_SECRET')) + engine_url = get('ENGINE_URL') + oauth_google_client_id = get('OAUTH_GOOGLE_CLIENT_ID') + oauth_google_client_secret = get('OAUTH_GOOGLE_CLIENT_SECRET') + oauth_redirect_uri = get('OAUTH_REDIRECT_URI') + auth_allowed_email_domains = json.loads(get('AUTH_ALLOWED_EMAIL_DOMAINS', '[]')) + auth_teacher_emails = json.loads(get('AUTH_TEACHER_EMAILS', '[]')) + auth_admin_emails = json.loads(get('AUTH_ADMIN_EMAILS', '[]')) + auth_super_admin_emails = json.loads(get('AUTH_SUPER_ADMIN_EMAILS', '[]')) + auth_approved_emails = json.loads(get('AUTH_APPROVED_EMAILS', '[]')) + auth_new_user_default_status = get('AUTH_NEW_USER_DEFAULT_STATUS', 'pending') + auth_email_cohort_map = json.loads(get('AUTH_EMAIL_COHORT_MAP', '{}')) + auth_domain_cohort_map = json.loads(get('AUTH_DOMAIN_COHORT_MAP', '{}')) + default_affiliation = get('DEFAULT_AFFILIATION') + + +settings = Settings() +""", + encoding="utf-8", + ) + (api_root / ".env").write_text( + "\n".join( + ( + "DATABASE_URL=postgresql://unit:pw@127.0.0.1:5432/vignette", + "SESSION_SECRET=unit-session-secret-that-is-long-enough", + "ENGINE_GATEWAY_SHARED_SECRET=unit-engine-secret-that-is-long-enough", + "ENGINE_URL=http://127.0.0.1:9099", + "OAUTH_GOOGLE_CLIENT_ID=unit-client.apps.googleusercontent.com", + f"OAUTH_GOOGLE_CLIENT_SECRET={client_secret}", + "OAUTH_REDIRECT_URI=https://api-vignette.chanpaca.net/auth/callback", + 'AUTH_ALLOWED_EMAIL_DOMAINS=["example.test"]', + 'AUTH_SUPER_ADMIN_EMAILS=["owner@example.test"]', + "AUTH_NEW_USER_DEFAULT_STATUS=approved", + "DEFAULT_AFFILIATION=unit", + "", + ) + ), + encoding="utf-8", + ) + + def _run_handoff(self, *, mismatched_target: bool) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory(prefix="vignette-env-handoff-") as raw: + root = Path(raw) + prior_api = root / "prior" / "apps" / "api" + stable_root = root / "stable" + target_api = stable_root / "apps" / "api" + self._write_fake_api(prior_api, client_secret="unit-prior-secret") + self._write_fake_api( + target_api, + client_secret=( + "unit-target-drift" if mismatched_target else "unit-prior-secret" + ), + ) + harness = root / "handoff.ps1" + escaped_prior = str(prior_api).replace("'", "''") + escaped_stable = str(stable_root).replace("'", "''") + escaped_python = sys.executable.replace("'", "''") + harness.write_text( + self.functions + + f""" +$resolvedPythonPath = '{escaped_python}' +$resolvedStableSourceRoot = '{escaped_stable}' +$required = @( + 'DATABASE_URL','SESSION_SECRET','ENGINE_URL','ENGINE_GATEWAY_SHARED_SECRET', + 'OAUTH_GOOGLE_CLIENT_ID','OAUTH_GOOGLE_CLIENT_SECRET','OAUTH_REDIRECT_URI', + 'AUTH_ALLOWED_EMAIL_DOMAINS','AUTH_TEACHER_EMAILS','AUTH_ADMIN_EMAILS', + 'AUTH_SUPER_ADMIN_EMAILS','AUTH_APPROVED_EMAILS','AUTH_NEW_USER_DEFAULT_STATUS', + 'AUTH_EMAIL_COHORT_MAP','AUTH_DOMAIN_COHORT_MAP','DEFAULT_AFFILIATION' +) +$completeEnvironment = Save-CompleteProcessEnvironment +$priorEnvironment = [ordered]@{{}} +foreach ($key in $completeEnvironment.Keys) {{ + $isRequired = $false + foreach ($requiredName in $required) {{ + if ([string]$key -ieq $requiredName) {{ + $isRequired = $true + break + }} + }} + if (-not $isRequired) {{ + $priorEnvironment[[string]$key] = [string]$completeEnvironment[$key] + }} +}} +$priorIdentity = [ordered]@{{ + cwd = '{escaped_prior}' + executable_path = '{escaped_python}' + environment = $priorEnvironment +}} +foreach ($requiredName in $required) {{ + if ($priorIdentity.environment.Contains($requiredName)) {{ + throw 'Test fixture unexpectedly contains required PID environment' + }} +}} +$prior = Invoke-EffectiveApiSettingsProbe ` + -PythonPath $resolvedPythonPath ` + -ApiCwd $priorIdentity.cwd ` + -Environment $priorIdentity.environment +$future = Get-FutureLauncherRequiredApiSettings ` + -PriorApiIdentity $priorIdentity ` + -RequiredNames $required ` + -EnginePort 9099 +Assert-RequiredApiSettingsEqual ` + -Expected $prior ` + -Actual $future ` + -Role 'future launcher' +Write-Output 'PASS' +""", + encoding="utf-8-sig", + ) + return subprocess.run( + [ + self._powershell(), + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + + def test_pid_environment_empty_dotenv_only_reboots_to_same_effective_settings( + self, + ) -> None: + completed = self._run_handoff(mismatched_target=False) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + self.assertEqual("PASS", completed.stdout.strip()) + self.assertNotIn("unit-prior-secret", completed.stdout + completed.stderr) + + def test_future_dotenv_drift_fails_without_printing_secret(self) -> None: + completed = self._run_handoff(mismatched_target=True) + self.assertNotEqual(0, completed.returncode) + self.assertNotIn("unit-prior-secret", completed.stdout + completed.stderr) + self.assertNotIn("unit-target-drift", completed.stdout + completed.stderr) + self.assertIn("effective settings digest drift", completed.stderr) + + def test_source_hash_is_pinned_before_settings_copy_and_after_copy(self) -> None: + main = self.source[ + self.source.index("$priorEnvironmentFilePath =") : + self.source.index("$priorTunnelIdentity =", self.source.index("$priorEnvironmentFilePath =")) + ] + source_hash = main.index("$priorEnvironmentFileSha256 =") + settings = main.index("Set-RequiredApiEnvironmentFromIdentity") + copy = main.index("Ensure-ReleaseEnvironmentFile") + future = main.index("Get-FutureLauncherRequiredApiSettings") + equality = main.index("Assert-RequiredApiSettingsEqual") + self.assertLess(source_hash, settings) + self.assertLess(settings, copy) + self.assertLess(copy, future) + self.assertLess(future, equality) + ensure = self.source[ + self.source.index("function Ensure-ReleaseEnvironmentFile") : + self.source.index("function Get-FutureLauncherRequiredApiSettings") + ] + self.assertIn('check-ignore --quiet -- "apps/api/.env"', ensure) + self.assertGreaterEqual(ensure.count("Assert-EnvironmentFilePinned `"), 3) + + def test_future_launcher_uses_the_actual_engine_port(self) -> None: + function = self.source[ + self.source.index("function Get-FutureLauncherRequiredApiSettings") : + self.source.index("function Assert-RequiredApiSettingsEqual") + ] + self.assertIn("[int]$EnginePort", function) + self.assertIn( + '$futureEnvironment["ENGINE_URL"] = "http://127.0.0.1:$EnginePort"', + function, + ) + self.assertNotIn("127.0.0.1:3001", function) + main = self.source[ + self.source.index("$priorEnvironmentFilePath =") : + self.source.index( + "$priorTunnelIdentity =", + self.source.index("$priorEnvironmentFilePath ="), + ) + ] + self.assertIn("-EnginePort $EnginePort", main) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_listener_pid_probe.py b/scripts/test_public_runtime_listener_pid_probe.py new file mode 100644 index 0000000..863e089 --- /dev/null +++ b/scripts/test_public_runtime_listener_pid_probe.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + + +PROBE = Path(__file__).resolve().parent / "probe-public-runtime-upload-root.py" +EXPECTED_MANIFEST = r"C:\private\vignette-upload-manifest.json" +EXPECTED_MANIFEST_SHA256 = "a" * 64 +EXPECTED_FREEZE = r"C:\private\vignette-upload-freeze.json" +EXPECTED_DB_TARGET_SHA256 = "b" * 64 + + +def _load_probe(): + spec = importlib.util.spec_from_file_location("listener_pid_probe", PROBE) + if spec is None or spec.loader is None: + raise AssertionError(f"probe module unavailable: {PROBE}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _argv(*, host: str = "127.0.0.1", port: int = 8001) -> list[str]: + return [ + "python.exe", + "-m", + "uvicorn", + "app.main:app", + "--host", + host, + "--port", + str(port), + "--workers", + "1", + ] + + +def _snapshot( + pid: int, + *, + argv: list[str] | None = None, + cwd: str = r"D:\release\apps\api", + root: str | None = r"C:\stable-uploads", + manifest_required: str | None = "true", + manifest_path: str | None = EXPECTED_MANIFEST, + manifest_sha256: str | None = EXPECTED_MANIFEST_SHA256, + freeze_path: str | None = EXPECTED_FREEZE, + database_target_sha256: str | None = EXPECTED_DB_TARGET_SHA256, +) -> dict[str, object]: + return { + "pid": pid, + "argv": _argv() if argv is None else argv, + "cwd": cwd, + "environment": { + "USER_UPLOAD_DIR": root, + "USER_UPLOAD_MANIFEST_REQUIRED": manifest_required, + "USER_UPLOAD_MANIFEST_PATH": manifest_path, + "USER_UPLOAD_MANIFEST_SHA256": manifest_sha256, + "USER_UPLOAD_WRITE_FREEZE_PATH": freeze_path, + "PUBLIC_RUNTIME_DB_TARGET_SHA256": database_target_sha256, + }, + } + + +class ListenerPidEvaluationTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.probe = _load_probe() + + def evaluate(self, listener_pids, snapshots): + return self.probe.evaluate_listener_binding( + listener_pids=listener_pids, + process_snapshots=snapshots, + expected_root=r"C:\stable-uploads", + expected_api_cwd=r"D:\release\apps\api", + expected_manifest_path=EXPECTED_MANIFEST, + expected_manifest_sha256=EXPECTED_MANIFEST_SHA256, + expected_write_freeze_path=EXPECTED_FREEZE, + expected_database_target_sha256=EXPECTED_DB_TARGET_SHA256, + api_port=8001, + ) + + def test_only_the_listener_snapshot_can_pass(self) -> None: + decoy = _snapshot(20) + listener = _snapshot(10, argv=["python.exe", "decoy.py"]) + result = self.evaluate([10], [decoy, listener]) + self.assertNotEqual(0, result[0]) + self.assertEqual("listener_command_mismatch", result[1]["reason"]) + self.assertEqual(10, result[1]["pid"]) + + def test_zero_and_multiple_listener_pids_fail_closed(self) -> None: + for listener_pids in ([], [10, 20]): + with self.subTest(listener_pids=listener_pids): + result = self.evaluate(listener_pids, [_snapshot(10), _snapshot(20)]) + self.assertNotEqual(0, result[0]) + self.assertEqual("api_listener_count_mismatch", result[1]["reason"]) + + def test_duplicate_socket_rows_for_one_pid_are_not_ambiguous(self) -> None: + result = self.evaluate([10, 10], [_snapshot(10)]) + self.assertEqual(0, result[0]) + self.assertEqual(10, result[1]["pid"]) + + def test_missing_listener_snapshot_fails_closed(self) -> None: + result = self.evaluate([10], [_snapshot(20)]) + self.assertNotEqual(0, result[0]) + self.assertEqual("listener_process_unavailable", result[1]["reason"]) + + def test_cwd_root_host_app_and_port_are_exact(self) -> None: + cases = [ + (_snapshot(10, cwd=r"D:\other\apps\api"), "listener_cwd_drift"), + (_snapshot(10, root=r"D:\other-uploads"), "user_upload_dir_drift"), + (_snapshot(10, argv=_argv(host="0.0.0.0")), "listener_command_mismatch"), + (_snapshot(10, argv=_argv(port=9001)), "listener_command_mismatch"), + ( + _snapshot( + 10, + argv=[ + "python.exe", + "-m", + "uvicorn", + "other.main:app", + "--host", + "127.0.0.1", + "--port", + "8001", + ], + ), + "listener_command_mismatch", + ), + ( + _snapshot(10, argv=_argv()[:-2]), + "listener_command_mismatch", + ), + ( + _snapshot(10, argv=[*_argv()[:-1], "2"]), + "listener_command_mismatch", + ), + ( + _snapshot(10, argv=[*_argv(), "--workers", "1"]), + "listener_command_mismatch", + ), + ] + for snapshot, expected_reason in cases: + with self.subTest(expected_reason=expected_reason): + result = self.evaluate([10], [snapshot]) + self.assertNotEqual(0, result[0]) + self.assertEqual(expected_reason, result[1]["reason"]) + + def test_listener_upload_receipt_and_database_target_are_exact(self) -> None: + cases = [ + ( + _snapshot(10, manifest_required="TRUE"), + "upload_manifest_required_mismatch", + ), + ( + _snapshot(10, manifest_path=r"C:\private\other.json"), + "upload_manifest_path_drift", + ), + ( + _snapshot(10, manifest_sha256="A" * 64), + "upload_manifest_sha256_drift", + ), + ( + _snapshot(10, freeze_path=r"C:\private\other-freeze.json"), + "upload_write_freeze_path_drift", + ), + ( + _snapshot(10, database_target_sha256="c" * 64), + "database_target_sha256_drift", + ), + ] + for snapshot, expected_reason in cases: + with self.subTest(expected_reason=expected_reason): + result = self.evaluate([10], [snapshot]) + self.assertNotEqual(0, result[0]) + self.assertEqual(expected_reason, result[1]["reason"]) + + +class ListenerPidCollectionTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.probe = _load_probe() + + def test_listener_collection_uses_exact_ipv4_loopback_and_deduplicates_pid( + self, + ) -> None: + listen = "LISTEN" + connections = [ + SimpleNamespace(status=listen, pid=10, laddr=("127.0.0.1", 8001)), + SimpleNamespace(status=listen, pid=10, laddr=("127.0.0.1", 8001)), + SimpleNamespace(status=listen, pid=20, laddr=("0.0.0.0", 8001)), + SimpleNamespace(status=listen, pid=30, laddr=("::1", 8001)), + SimpleNamespace(status=listen, pid=40, laddr=("127.0.0.1", 9001)), + SimpleNamespace(status="ESTABLISHED", pid=50, laddr=("127.0.0.1", 8001)), + SimpleNamespace(status=listen, pid=None, laddr=("127.0.0.1", 8001)), + ] + fake_psutil = SimpleNamespace( + CONN_LISTEN=listen, + net_connections=lambda *, kind: connections, + ) + with patch.dict(sys.modules, {"psutil": fake_psutil}): + self.assertEqual([10], self.probe._collect_listener_pids(8001)) + + def test_snapshot_collection_inspects_only_supplied_listener_pid(self) -> None: + requested: list[int] = [] + + class FakeProcess: + def __init__(self, pid: int) -> None: + requested.append(pid) + self.pid = pid + + def cmdline(self): + return _argv() + + def cwd(self): + return r"D:\release\apps\api" + + def environ(self): + return { + "USER_UPLOAD_DIR": r"C:\stable-uploads", + "USER_UPLOAD_MANIFEST_REQUIRED": "true", + "USER_UPLOAD_MANIFEST_PATH": EXPECTED_MANIFEST, + "USER_UPLOAD_MANIFEST_SHA256": EXPECTED_MANIFEST_SHA256, + "USER_UPLOAD_WRITE_FREEZE_PATH": EXPECTED_FREEZE, + "PUBLIC_RUNTIME_DB_TARGET_SHA256": EXPECTED_DB_TARGET_SHA256, + "DATABASE_URL": "must-not-be-retained", + } + + fake_psutil = SimpleNamespace( + AccessDenied=type("AccessDenied", (Exception,), {}), + NoSuchProcess=type("NoSuchProcess", (Exception,), {}), + ZombieProcess=type("ZombieProcess", (Exception,), {}), + Process=FakeProcess, + process_iter=lambda *args, **kwargs: self.fail("process_iter must not run"), + ) + with patch.dict(sys.modules, {"psutil": fake_psutil}): + snapshots = self.probe._collect_process_snapshots([10, 10]) + + self.assertEqual([10], requested) + self.assertEqual(1, len(snapshots)) + self.assertEqual( + { + "USER_UPLOAD_DIR": r"C:\stable-uploads", + "USER_UPLOAD_MANIFEST_REQUIRED": "true", + "USER_UPLOAD_MANIFEST_PATH": EXPECTED_MANIFEST, + "USER_UPLOAD_MANIFEST_SHA256": EXPECTED_MANIFEST_SHA256, + "USER_UPLOAD_WRITE_FREEZE_PATH": EXPECTED_FREEZE, + "PUBLIC_RUNTIME_DB_TARGET_SHA256": EXPECTED_DB_TARGET_SHA256, + }, + snapshots[0]["environment"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_task_definition_cutover.py b/scripts/test_public_runtime_task_definition_cutover.py new file mode 100644 index 0000000..bd7f503 --- /dev/null +++ b/scripts/test_public_runtime_task_definition_cutover.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +CONTRACT = SCRIPTS / "public-runtime-task-definition-cutover.ps1" +REPO_ROOT = SCRIPTS.parent + + +class PublicRuntimeTaskDefinitionCutoverTest(unittest.TestCase): + def _run_powershell(self, body: str) -> subprocess.CompletedProcess[str]: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + with tempfile.TemporaryDirectory(prefix="vignette-task-cutover-") as raw: + harness = Path(raw) / "task-cutover.ps1" + harness.write_text(body, encoding="utf-8-sig") + return subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + + def _assert_harness_passes(self, body: str) -> None: + completed = self._run_powershell(body) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_contract_digest_ignores_only_enabled_state(self) -> None: + contract = str(CONTRACT).replace("'", "''") + self._assert_harness_passes( + f""" +. '{contract}' +$enabled = 'trueone' +$disabled = 'falseone' +$drifted = 'falsetwo' +$enabledDigest = Get-PublicRuntimeTaskXmlContractSha256 -Xml $enabled +$disabledDigest = Get-PublicRuntimeTaskXmlContractSha256 -Xml $disabled +$driftedDigest = Get-PublicRuntimeTaskXmlContractSha256 -Xml $drifted +if ($enabledDigest -cne $disabledDigest) {{ exit 2 }} +if ($enabledDigest -ceq $driftedDigest) {{ exit 3 }} +exit 0 +""" + ) + + def test_exact_action_contract_accepts_disabled_and_operational_snapshots(self) -> None: + contract = str(CONTRACT).replace("'", "''") + root = str(REPO_ROOT).replace("'", "''") + self._assert_harness_passes( + f""" +. '{contract}' +$contracts = @(Get-PublicRuntimeExpectedTaskActionContracts ` + -StableSourceRoot '{root}' ` + -ExpectedSourceCommit ('a' * 40) ` + -ExpectedSourceTree ('b' * 40) ` + -PythonPath 'C:\\Python311\\python.exe' ` + -UserUploadDir 'C:\\Runtime\\uploads' ` + -UserUploadManifestPath 'C:\\RuntimeState\\manifest.json' ` + -ExpectedUserUploadManifestSha256 ('c' * 64) ` + -UserUploadWriteFreezePath 'C:\\RuntimeState\\freeze.json' ` + -CloudflaredPath 'C:\\Tools\\cloudflared.exe' ` + -CloudflaredConfigPath 'C:\\RuntimeState\\cloudflared.yml' ` + -PublicHealthUrl 'https://api-vignette.chanpaca.net/health') +$entries = @() +foreach ($item in $contracts) {{ + $entries += [pscustomobject]@{{ + role = $item.role + enabled = $false + action_execute = $item.execute + action_arguments = $item.arguments + action_working_directory = $item.working_directory + }} +}} +$snapshot = [pscustomobject]@{{ entries = $entries }} +$null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $snapshot ` + -StableSourceRoot '{root}' ` + -ExpectedSourceCommit ('a' * 40) ` + -ExpectedSourceTree ('b' * 40) ` + -PythonPath 'C:\\Python311\\python.exe' ` + -UserUploadDir 'C:\\Runtime\\uploads' ` + -UserUploadManifestPath 'C:\\RuntimeState\\manifest.json' ` + -ExpectedUserUploadManifestSha256 ('c' * 64) ` + -UserUploadWriteFreezePath 'C:\\RuntimeState\\freeze.json' ` + -CloudflaredPath 'C:\\Tools\\cloudflared.exe' ` + -CloudflaredConfigPath 'C:\\RuntimeState\\cloudflared.yml' ` + -PublicHealthUrl 'https://api-vignette.chanpaca.net/health' +foreach ($entry in $entries) {{ $entry.enabled = $true }} +$null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $snapshot ` + -StableSourceRoot '{root}' ` + -ExpectedSourceCommit ('a' * 40) ` + -ExpectedSourceTree ('b' * 40) ` + -PythonPath 'C:\\Python311\\python.exe' ` + -UserUploadDir 'C:\\Runtime\\uploads' ` + -UserUploadManifestPath 'C:\\RuntimeState\\manifest.json' ` + -ExpectedUserUploadManifestSha256 ('c' * 64) ` + -UserUploadWriteFreezePath 'C:\\RuntimeState\\freeze.json' ` + -CloudflaredPath 'C:\\Tools\\cloudflared.exe' ` + -CloudflaredConfigPath 'C:\\RuntimeState\\cloudflared.yml' ` + -PublicHealthUrl 'https://api-vignette.chanpaca.net/health' ` + -AllowEnabled +$entries[0].action_arguments += ' -InjectedDrift' +try {{ + $null = Assert-NewPublicRuntimeTaskDefinitionsPinned ` + -Snapshot $snapshot ` + -StableSourceRoot '{root}' ` + -ExpectedSourceCommit ('a' * 40) ` + -ExpectedSourceTree ('b' * 40) ` + -PythonPath 'C:\\Python311\\python.exe' ` + -UserUploadDir 'C:\\Runtime\\uploads' ` + -UserUploadManifestPath 'C:\\RuntimeState\\manifest.json' ` + -ExpectedUserUploadManifestSha256 ('c' * 64) ` + -UserUploadWriteFreezePath 'C:\\RuntimeState\\freeze.json' ` + -CloudflaredPath 'C:\\Tools\\cloudflared.exe' ` + -CloudflaredConfigPath 'C:\\RuntimeState\\cloudflared.yml' ` + -PublicHealthUrl 'https://api-vignette.chanpaca.net/health' ` + -AllowEnabled + exit 2 +}} catch {{ + exit 0 +}} +""" + ) + + def test_second_installer_failure_compensates_both_tasks_to_disabled(self) -> None: + contract = str(CONTRACT).replace("'", "''") + self._assert_harness_passes( + f""" +. '{contract}' +$script:states = @{{ Boot = $false; Watchdog = $false }} +$script:bootCalls = 0 +$script:watchdogCalls = 0 +$script:suspendCalls = 0 +$script:assertCalls = 0 +function Suspend-PublicRuntimeTasks {{ + param($Snapshot, $TimeoutSec) + $script:suspendCalls += 1 + $script:states.Boot = $false + $script:states.Watchdog = $false +}} +function Assert-PublicRuntimeTasksDisabledAndIdle {{ + param($Snapshot, $TimeoutSec) + $script:assertCalls += 1 + if ($script:states.Boot -or $script:states.Watchdog) {{ + throw 'task was not compensated to disabled' + }} +}} +$bootInstaller = {{ + $script:bootCalls += 1 + $script:states.Boot = $true +}} +$watchdogInstaller = {{ + $script:watchdogCalls += 1 + throw 'synthetic second installer failure' +}} +$maintenance = @( + [pscustomobject]@{{ role='boot'; task_name='Boot' }}, + [pscustomobject]@{{ role='watchdog'; task_name='Watchdog' }} +) +try {{ + Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled ` + -BootInstaller $bootInstaller ` + -WatchdogInstaller $watchdogInstaller ` + -MaintenanceSnapshot $maintenance ` + -TimeoutSec 3 + exit 2 +}} catch {{ + if ($script:bootCalls -ne 1 -or $script:watchdogCalls -ne 1) {{ exit 3 }} + if ($script:suspendCalls -ne 1 -or $script:assertCalls -ne 1) {{ exit 4 }} + if ($script:states.Boot -or $script:states.Watchdog) {{ exit 5 }} + exit 0 +}} +""" + ) + + def test_second_enable_failure_compensates_both_tasks_to_disabled(self) -> None: + contract = str(CONTRACT).replace("'", "''") + self._assert_harness_passes( + f""" +. '{contract}' +$script:states = @{{ Boot = $false; Watchdog = $false }} +$script:taskPaths = @() +function Assert-PublicRuntimeTaskDefinitionSnapshotCurrent {{ + param($ExpectedSnapshot) + return $ExpectedSnapshot +}} +function Enable-ScheduledTask {{ + param($TaskName, $TaskPath, $ErrorAction) + $script:taskPaths += $TaskPath + if ($TaskPath -cne '\\') {{ throw 'wrong task path' }} + if ($TaskName -eq 'Watchdog') {{ throw 'synthetic second enable failure' }} + $script:states[$TaskName] = $true +}} +function Disable-ScheduledTask {{ + param($TaskName, $TaskPath, $ErrorAction) + $script:taskPaths += $TaskPath + if ($TaskPath -cne '\\') {{ throw 'wrong task path' }} + $script:states[$TaskName] = $false +}} +function Get-PublicRuntimeTaskDefinitionSnapshot {{ + param($BootTaskName, $WatchdogTaskName, [switch]$RequireEnabled) + return [pscustomobject]@{{ entries = @( + [pscustomobject]@{{ role='boot'; task_name='Boot'; enabled=$script:states.Boot }}, + [pscustomobject]@{{ role='watchdog'; task_name='Watchdog'; enabled=$script:states.Watchdog }} + ) }} +}} +$snapshot = [pscustomobject]@{{ entries = @( + [pscustomobject]@{{ role='boot'; task_name='Boot'; contract_sha256=('a' * 64); action_execute='one'; action_arguments='one'; action_working_directory='one' }}, + [pscustomobject]@{{ role='watchdog'; task_name='Watchdog'; contract_sha256=('b' * 64); action_execute='two'; action_arguments='two'; action_working_directory='two' }} +) }} +try {{ + Enable-NewPublicRuntimeTaskDefinitions -DisabledSnapshot $snapshot | Out-Null + exit 2 +}} catch {{ + if ($script:states.Boot -or $script:states.Watchdog) {{ exit 3 }} + if ($script:taskPaths.Count -lt 3) {{ exit 4 }} + if (@($script:taskPaths | Where-Object {{ $_ -cne '\\' }}).Count -ne 0) {{ exit 5 }} + exit 0 +}} +""" + ) + + def test_operational_definition_drift_disables_both_tasks(self) -> None: + contract = str(CONTRACT).replace("'", "''") + self._assert_harness_passes( + f""" +. '{contract}' +$script:states = @{{ Boot = $false; Watchdog = $false }} +function Assert-PublicRuntimeTaskDefinitionSnapshotCurrent {{ param($ExpectedSnapshot); return $ExpectedSnapshot }} +function Enable-ScheduledTask {{ + param($TaskName, $TaskPath, $ErrorAction) + if ($TaskPath -cne '\\') {{ throw 'wrong task path' }} + $script:states[$TaskName] = $true +}} +function Disable-ScheduledTask {{ + param($TaskName, $TaskPath, $ErrorAction) + if ($TaskPath -cne '\\') {{ throw 'wrong task path' }} + $script:states[$TaskName] = $false +}} +function Get-PublicRuntimeTaskDefinitionSnapshot {{ + param($BootTaskName, $WatchdogTaskName, [switch]$RequireEnabled) + return [pscustomobject]@{{ entries = @( + [pscustomobject]@{{ role='boot'; task_name='Boot'; enabled=$true; contract_sha256=('a' * 64); action_execute='one'; action_arguments='one'; action_working_directory='one' }}, + [pscustomobject]@{{ role='watchdog'; task_name='Watchdog'; enabled=$true; contract_sha256=('c' * 64); action_execute='two'; action_arguments='two'; action_working_directory='two' }} + ); set_sha256=('d' * 64) }} +}} +$snapshot = [pscustomobject]@{{ entries = @( + [pscustomobject]@{{ role='boot'; task_name='Boot'; contract_sha256=('a' * 64); action_execute='one'; action_arguments='one'; action_working_directory='one' }}, + [pscustomobject]@{{ role='watchdog'; task_name='Watchdog'; contract_sha256=('b' * 64); action_execute='two'; action_arguments='two'; action_working_directory='two' }} +) }} +try {{ + Enable-NewPublicRuntimeTaskDefinitions -DisabledSnapshot $snapshot | Out-Null + exit 2 +}} catch {{ + if ($script:states.Boot -or $script:states.Watchdog) {{ exit 3 }} + exit 0 +}} +""" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_task_maintenance.py b/scripts/test_public_runtime_task_maintenance.py new file mode 100644 index 0000000..f741bb3 --- /dev/null +++ b/scripts/test_public_runtime_task_maintenance.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +CONTRACT = SCRIPTS / "public-runtime-task-maintenance.ps1" +START = SCRIPTS / "start-public-runtime.ps1" + + +class PublicRuntimeTaskMaintenanceTest(unittest.TestCase): + def _run_powershell(self, body: str) -> subprocess.CompletedProcess[str]: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + with tempfile.TemporaryDirectory(prefix="vignette-task-maintenance-") as raw: + harness = Path(raw) / "task-maintenance.ps1" + harness.write_text(body, encoding="utf-8-sig") + return subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + + def test_enter_disables_and_exit_restores_only_previously_enabled_tasks( + self, + ) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{ + Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }} + Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }} +}} +function Get-ScheduledTask {{ + param($TaskName, $ErrorAction) + return $script:states[$TaskName] +}} +function Disable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $false +}} +function Enable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $true +}} +. '{contract}' +$snapshot = @(Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 1) +if ($script:states.Watchdog.Settings.Enabled) {{ exit 2 }} +if ($script:states.Boot.Settings.Enabled) {{ exit 3 }} +Exit-PublicRuntimeTaskMaintenance -Snapshot $snapshot +if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 4 }} +if ($script:states.Boot.Settings.Enabled) {{ exit 5 }} +exit 0 +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_failed_reenable_returns_every_owned_task_to_disabled_idle(self) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{ + Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }} + Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }} +}} +function Get-ScheduledTask {{ + param($TaskName, $ErrorAction) + return $script:states[$TaskName] +}} +function Disable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $false +}} +function Enable-ScheduledTask {{ + param($TaskName, $ErrorAction) + if ($TaskName -eq 'Boot') {{ throw 'synthetic enable failure' }} + $script:states[$TaskName].Settings.Enabled = $true +}} +. '{contract}' +$snapshot = @( + [pscustomobject]@{{ task_name = 'Watchdog'; exists = $true; was_enabled = $true }}, + [pscustomobject]@{{ task_name = 'Boot'; exists = $true; was_enabled = $true }} +) +try {{ + Exit-PublicRuntimeTaskMaintenance -Snapshot $snapshot + exit 2 +}} catch {{ + if ($script:states.Watchdog.Settings.Enabled) {{ exit 3 }} + if ($script:states.Boot.Settings.Enabled) {{ exit 4 }} + exit 0 +}} +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_state_probe_reports_actual_restored_and_disabled_truth(self) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{ + Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }} + Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }} +}} +function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }} +. '{contract}' +$snapshot = @( + [pscustomobject]@{{ task_name = 'Watchdog'; exists = $true; was_enabled = $true }}, + [pscustomobject]@{{ task_name = 'Boot'; exists = $true; was_enabled = $false }} +) +$restored = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot +if (-not $restored.verified) {{ exit 2 }} +if (-not $restored.restored_to_snapshot) {{ exit 3 }} +if ($restored.all_disabled_and_idle) {{ exit 4 }} +$script:states.Watchdog.Settings.Enabled = $false +$disabled = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot +if (-not $disabled.verified) {{ exit 5 }} +if (-not $disabled.all_disabled_and_idle) {{ exit 6 }} +if ($disabled.restored_to_snapshot) {{ exit 7 }} +exit 0 +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_second_disable_failure_restores_full_preflight_snapshot(self) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{ + Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }} + Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }} +}} +function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }} +function Disable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $false + if ($TaskName -eq 'Watchdog') {{ throw 'synthetic second disable failure' }} +}} +function Enable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $true +}} +. '{contract}' +try {{ + Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 1 | Out-Null + exit 2 +}} catch {{ + if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 3 }} + if (-not $script:states.Boot.Settings.Enabled) {{ exit 4 }} + exit 0 +}} +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_idle_timeout_restores_full_preflight_snapshot(self) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{ + Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Running' }} + Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }} +}} +function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }} +function Disable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $false +}} +function Enable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:states[$TaskName].Settings.Enabled = $true +}} +. '{contract}' +try {{ + Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 0 | Out-Null + exit 2 +}} catch {{ + if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 3 }} + if (-not $script:states.Boot.Settings.Enabled) {{ exit 4 }} + exit 0 +}} +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_task_created_after_absent_snapshot_fails_truth_without_mutating_it( + self, + ) -> None: + contract = str(CONTRACT).replace("'", "''") + completed = self._run_powershell( + f""" +$script:states = @{{}} +$script:disableCalls = @() +function Get-ScheduledTask {{ + param($TaskName, $ErrorAction) + if ($script:states.ContainsKey($TaskName)) {{ return $script:states[$TaskName] }} + return $null +}} +function Disable-ScheduledTask {{ + param($TaskName, $ErrorAction) + $script:disableCalls += $TaskName + $script:states[$TaskName].Settings.Enabled = $false +}} +function Enable-ScheduledTask {{ param($TaskName, $ErrorAction) }} +. '{contract}' +$snapshot = @(Enter-PublicRuntimeTaskMaintenance -TaskNames @('Boot') -TimeoutSec 1) +$script:states.Boot = [pscustomobject]@{{ + Settings = [pscustomobject]@{{ Enabled = $true }} + State = 'Ready' +}} +$truth = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot +if (-not $truth.verified) {{ exit 2 }} +if ($truth.all_disabled_and_idle) {{ exit 3 }} +if ($truth.restored_to_snapshot) {{ exit 4 }} +try {{ + Assert-PublicRuntimeTasksDisabledAndIdle -Snapshot $snapshot -TimeoutSec 1 + exit 5 +}} catch {{ + if ($script:disableCalls -contains 'Boot') {{ exit 6 }} + exit 0 +}} +""" + ) + self.assertEqual( + 0, + completed.returncode, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_start_reenables_tasks_only_after_unfrozen_passed_receipt(self) -> None: + source = START.read_text(encoding="utf-8") + maintenance = source.index("Enter-PublicRuntimeTaskMaintenance `") + mutation = source.index('$freshFailureStage = "api_cutover"') + no_rollback = source.index("$freshNoRollback = $true") + release = source.index("Exit-PublicUploadWriteFreeze `", no_rollback) + receipt_stage = source.index('$freshFailureStage = "receipt_publish"') + receipt_write = source.index("Write-Utf8TextAtomically `", receipt_stage) + install_tasks = source.index('$freshFailureStage = "task_definition_cutover"', receipt_write) + verify_disabled = source.index( + "Assert-NewPublicRuntimeTaskDefinitionsPinned `", + install_tasks, + ) + restore_tasks = source.index( + "Enable-NewPublicRuntimeTaskDefinitions `", + verify_disabled, + ) + actual_task_probe = source.index( + "Get-PublicRuntimeTaskMaintenanceState `", + restore_tasks, + ) + task_entered_clear = source.index( + "$freshTaskMaintenanceEntered = $false", + actual_task_probe, + ) + task_receipt_stage = source.index( + '$freshFailureStage = "task_recovery_receipt_publish"', + restore_tasks, + ) + task_receipt_write = source.index( + "Write-Utf8TextAtomically `", + task_receipt_stage, + ) + fully_committed = source.index( + "$freshPromotionCommitted = $true", + task_receipt_write, + ) + self.assertLess(maintenance, mutation) + self.assertLess(no_rollback, release) + self.assertLess(release, receipt_stage) + self.assertLess(receipt_stage, receipt_write) + self.assertLess(receipt_write, install_tasks) + self.assertLess(install_tasks, verify_disabled) + self.assertLess(verify_disabled, restore_tasks) + self.assertLess(restore_tasks, actual_task_probe) + self.assertLess(actual_task_probe, task_receipt_stage) + self.assertLess(restore_tasks, task_receipt_stage) + self.assertLess(task_receipt_stage, task_receipt_write) + self.assertLess(task_receipt_write, task_entered_clear) + self.assertLess(task_entered_clear, fully_committed) + self.assertIn('scope = "runtime_storage_commit_only"', source) + self.assertIn('operational_success = $false', source) + self.assertIn('schema_version = "vignette.public-runtime-task-recovery.v1"', source) + self.assertIn("$freshTaskMaintenanceWasEntered", source) + self.assertIn("tasks_restored_to_snapshot = $TasksRestored", source) + self.assertIn("task_state_verified = $TaskStateVerified", source) + self.assertIn("Suspend-PublicRuntimeTasks `", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_upload_release_safety.py b/scripts/test_public_runtime_upload_release_safety.py new file mode 100644 index 0000000..cfa0a12 --- /dev/null +++ b/scripts/test_public_runtime_upload_release_safety.py @@ -0,0 +1,460 @@ +from __future__ import annotations + +import importlib.util +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +REPO = SCRIPTS.parent +HELPER = SCRIPTS / "public-runtime-upload-root.ps1" +MANIFEST_PROBE = SCRIPTS / "validate-public-runtime-upload-manifest.py" +OFFLINE_QUIESCENCE_PROBE = SCRIPTS / "validate-public-runtime-offline-quiescence.py" +DATABASE_IDENTITY = SCRIPTS / "public_runtime_database_identity.py" +INITIALIZER = SCRIPTS / "initialize-public-runtime-upload-root.ps1" +INITIALIZER_WORKER = SCRIPTS / "initialize-public-runtime-upload-root.py" +PROCESS_PROBE = SCRIPTS / "probe-public-runtime-upload-root.py" +START = SCRIPTS / "start-public-runtime.ps1" +BOOT = SCRIPTS / "boot-public-runtime.ps1" +WATCH = SCRIPTS / "watch-public-runtime.ps1" +INSTALL = SCRIPTS / "install-public-runtime-task.ps1" +REGISTER = SCRIPTS / "register-boot-task.ps1" + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise AssertionError(f"could not import {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run_windows_powershell(script: Path) -> subprocess.CompletedProcess[str]: + powershell = shutil.which("powershell.exe") + if powershell is None: + raise unittest.SkipTest("Windows PowerShell 5.1 is unavailable") + return subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script), + ], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + ) + + +class PublicRuntimeUploadReleaseSafetyTest(unittest.TestCase): + @unittest.skipUnless(shutil.which("git.exe"), "git.exe is unavailable") + def test_drive_and_unc_share_roots_are_rejected_before_path_trimming(self) -> None: + with tempfile.TemporaryDirectory( + prefix="vignette-upload-root-boundary-" + ) as raw: + root = Path(raw) + source = root / "source" + source.mkdir() + subprocess.run( + [shutil.which("git.exe") or "git.exe", "-C", str(source), "init"], + check=True, + capture_output=True, + ) + harness = root / "root-boundary.ps1" + quoted_helper = str(HELPER).replace("'", "''") + quoted_source = str(source).replace("'", "''") + harness.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quoted_helper}' +function Assert-Rejected([string]$Candidate) {{ + $rejected = $false + try {{ + Resolve-PublicRuntimeUploadRoot ` + -SourceRoot '{quoted_source}' ` + -UploadRoot $Candidate ` + -CreateIfMissing | Out-Null + }} catch {{ + $rejected = $true + }} + if (-not $rejected) {{ throw "filesystem root accepted: $Candidate" }} +}} +Assert-Rejected 'D:\\' +Assert-Rejected '\\\\server\\share\\' +""", + encoding="utf-8-sig", + ) + result = _run_windows_powershell(harness) + self.assertEqual( + result.returncode, + 0, + msg=f"stdout={result.stdout}\nstderr={result.stderr}", + ) + + def test_manifest_and_freeze_paths_must_stay_outside_public_and_source_roots( + self, + ) -> None: + with tempfile.TemporaryDirectory( + prefix="vignette-upload-private-boundary-" + ) as raw: + root = Path(raw) + source = root / "source" + upload = root / "uploads" + private = root / "private" + source.mkdir() + upload.mkdir() + private.mkdir() + inside_upload = upload / "manifest.json" + inside_source = source / "manifest.json" + outside = private / "manifest.json" + for path in (inside_upload, inside_source, outside): + path.write_text("{}", encoding="utf-8") + harness = root / "private-boundary.ps1" + + def quote(value: Path) -> str: + return str(value).replace("'", "''") + + harness.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quote(HELPER)}' +function Assert-Rejected([string]$Candidate) {{ + $rejected = $false + try {{ + Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot '{quote(source)}' ` + -UploadRoot '{quote(upload)}' ` + -StatePath $Candidate ` + -RequireFile | Out-Null + }} catch {{ + $rejected = $true + }} + if (-not $rejected) {{ throw "public state path accepted: $Candidate" }} +}} +Assert-Rejected '{quote(inside_upload)}' +Assert-Rejected '{quote(inside_source)}' +$accepted = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot '{quote(source)}' ` + -UploadRoot '{quote(upload)}' ` + -StatePath '{quote(outside)}' ` + -RequireFile +if ($accepted -cne (Resolve-Path -LiteralPath '{quote(outside)}').Path) {{ + throw 'external private manifest was not preserved' +}} +""", + encoding="utf-8-sig", + ) + result = _run_windows_powershell(harness) + self.assertEqual( + result.returncode, + 0, + msg=f"stdout={result.stdout}\nstderr={result.stderr}", + ) + + def test_every_consumer_requires_one_pinned_validated_manifest(self) -> None: + for path in (START, BOOT, WATCH, INSTALL, REGISTER): + source = _read(path) + with self.subTest(path=path.name): + self.assertIn("[string]$UserUploadManifestPath", source) + self.assertIn("[string]$ExpectedUserUploadManifestSha256", source) + self.assertIn("[string]$UserUploadWriteFreezePath", source) + self.assertIn("Test-PublicRuntimeUploadManifest", source) + self.assertIn("validate-public-runtime-upload-manifest.py", source) + + for path in (START, INSTALL, REGISTER): + source = _read(path) + call = source.index( + "$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot" + ) + call_window = source[call : call + 360] + with self.subTest(no_implicit_create=path.name): + self.assertNotIn("-CreateIfMissing", call_window) + + hidden = _read(SCRIPTS / "watch-public-runtime-hidden.vbs") + for marker in ( + "-UserUploadManifestPath", + "-ExpectedUserUploadManifestSha256", + "-UserUploadWriteFreezePath", + ): + self.assertIn(marker, hidden) + + validator = _read(MANIFEST_PROBE) + offline_validator = _read(OFFLINE_QUIESCENCE_PROBE) + database_identity = _read(DATABASE_IDENTITY) + self.assertIn("SELECT avatar_url", validator) + self.assertIn("validate_current_avatar_references", validator) + self.assertIn("verify_preserved_objects=True", validator) + self.assertIn("reject_unbound_extras=False", validator) + self.assertIn('"current_reference_set_sha256"', validator) + for field in ( + "preserved_decode_valid_count", + "preserved_decode_invalid_count", + "required_decode_invalid_object_count", + "required_decode_invalid_reference_count", + ): + with self.subTest(validator_field=field): + self.assertIn(f'"{field}"', validator) + self.assertIn(f'"{field}"', offline_validator) + for field in ( + "current_decode_invalid_object_count", + "current_decode_invalid_reference_count", + ): + with self.subTest(current_validator_field=field): + self.assertIn(f'"{field}"', validator) + + start_contract = _read(START) + self.assertIn("$offlinePreservedDecodeCounts.ValidCount", start_contract) + self.assertIn("$manifestPreservedDecodeCounts.ValidCount", start_contract) + self.assertIn("$offlineRequiredDecodeCounts.ObjectCount", start_contract) + self.assertIn("$manifestRequiredDecodeCounts.ObjectCount", start_contract) + self.assertIn("$manifestCurrentDecodeCounts.ObjectCount", start_contract) + + def test_initializer_is_the_only_explicit_root_creator_and_is_copy_only( + self, + ) -> None: + self.assertTrue(INITIALIZER.is_file()) + self.assertTrue(INITIALIZER_WORKER.is_file()) + self.assertTrue(MANIFEST_PROBE.is_file()) + wrapper = _read(INITIALIZER) + worker = _read(INITIALIZER_WORKER) + self.assertIn("-CreateIfMissing", wrapper) + self.assertIn("ExpectedReferenceCount", wrapper) + self.assertIn("ExpectedPreservedObjectCount", wrapper) + self.assertIn("ExpectedPreservedTotalSizeBytes", wrapper) + self.assertIn("ExpectedPreservedInventorySha256", wrapper) + self.assertIn("scan_preserved_inventory(source_roots)", worker) + self.assertGreaterEqual(worker.count("assert_expected_preserved_inventory("), 3) + self.assertIn("FileMode]::CreateNew", wrapper) + self.assertIn('open("xb")', worker) + self.assertNotIn("shutil.move", worker) + self.assertNotIn("os.replace", worker) + self.assertNotIn( + 'relative_path"', worker[worker.index("def privacy_safe_manifest") :] + ) + + def test_process_proof_is_bound_to_the_unique_loopback_listener_pid(self) -> None: + module = _load(PROCESS_PROBE, "public_runtime_listener_probe") + expected_root = r"C:\stable-uploads" + expected_cwd = r"D:\release\apps\api" + expected_manifest = r"C:\private\upload-manifest.json" + expected_manifest_sha256 = "a" * 64 + expected_freeze = r"C:\private\upload-freeze.json" + expected_database_target_sha256 = "b" * 64 + argv = [ + "python.exe", + "-m", + "uvicorn", + "app.main:app", + "--host", + "127.0.0.1", + "--port", + "8001", + "--workers", + "1", + ] + snapshots = [ + { + "pid": 10, + "argv": argv, + "cwd": expected_cwd, + "environment": { + "USER_UPLOAD_DIR": expected_root, + "USER_UPLOAD_MANIFEST_REQUIRED": "true", + "USER_UPLOAD_MANIFEST_PATH": expected_manifest, + "USER_UPLOAD_MANIFEST_SHA256": expected_manifest_sha256, + "USER_UPLOAD_WRITE_FREEZE_PATH": expected_freeze, + "PUBLIC_RUNTIME_DB_TARGET_SHA256": ( + expected_database_target_sha256 + ), + }, + }, + { + "pid": 20, + "argv": argv, + "cwd": expected_cwd, + "environment": { + "USER_UPLOAD_DIR": expected_root, + "USER_UPLOAD_MANIFEST_REQUIRED": "true", + "USER_UPLOAD_MANIFEST_PATH": expected_manifest, + "USER_UPLOAD_MANIFEST_SHA256": expected_manifest_sha256, + "USER_UPLOAD_WRITE_FREEZE_PATH": expected_freeze, + "PUBLIC_RUNTIME_DB_TARGET_SHA256": ( + expected_database_target_sha256 + ), + }, + }, + ] + passed = module.evaluate_listener_binding( + listener_pids=[10], + process_snapshots=snapshots, + expected_root=expected_root, + expected_api_cwd=expected_cwd, + expected_manifest_path=expected_manifest, + expected_manifest_sha256=expected_manifest_sha256, + expected_write_freeze_path=expected_freeze, + expected_database_target_sha256=expected_database_target_sha256, + api_port=8001, + ) + self.assertEqual(0, passed[0]) + self.assertEqual(10, passed[1]["pid"]) + + decoy_only = module.evaluate_listener_binding( + listener_pids=[30], + process_snapshots=snapshots, + expected_root=expected_root, + expected_api_cwd=expected_cwd, + expected_manifest_path=expected_manifest, + expected_manifest_sha256=expected_manifest_sha256, + expected_write_freeze_path=expected_freeze, + expected_database_target_sha256=expected_database_target_sha256, + api_port=8001, + ) + self.assertNotEqual(0, decoy_only[0]) + self.assertEqual("listener_process_unavailable", decoy_only[1]["reason"]) + + ambiguous = module.evaluate_listener_binding( + listener_pids=[10, 20], + process_snapshots=snapshots, + expected_root=expected_root, + expected_api_cwd=expected_cwd, + expected_manifest_path=expected_manifest, + expected_manifest_sha256=expected_manifest_sha256, + expected_write_freeze_path=expected_freeze, + expected_database_target_sha256=expected_database_target_sha256, + api_port=8001, + ) + self.assertNotEqual(0, ambiguous[0]) + self.assertEqual("api_listener_count_mismatch", ambiguous[1]["reason"]) + + def test_listener_identity_call_chain_pins_receipt_and_database_target( + self, + ) -> None: + contract = _read(HELPER) + start = _read(START) + validator = _read(MANIFEST_PROBE) + database_identity = _read(DATABASE_IDENTITY) + for expected in ( + '"--expected-manifest-path"', + '"--expected-manifest-sha256"', + '"--expected-write-freeze-path"', + '"--expected-database-target-sha256"', + "ListenerPid = $listenerPid", + ): + with self.subTest(source="contract", expected=expected): + self.assertIn(expected, contract) + + prior_capture = start[ + start.index( + "$priorApiListenerProof = Test-PublicRuntimeApiUploadRoot" + ) : start.index("$priorCloudflaredProcesses = @(") + ] + self.assertIn("$priorApiListenerProof.ListenerPid", prior_capture) + self.assertNotIn("Get-UvicornProcessesByPort", prior_capture) + self.assertIn( + "$env:PUBLIC_RUNTIME_DB_TARGET_SHA256 = $expectedDatabaseTargetSha256", + start, + ) + self.assertIn("connected_database_target_sha256", validator) + self.assertIn("current_database()", database_identity) + self.assertIn("current_user::text", database_identity) + self.assertIn("inet_server_addr()", database_identity) + self.assertIn('"database_target_sha256"', validator) + + def test_fresh_cutover_requires_drained_freeze_and_restores_it(self) -> None: + source = _read(START) + for expected in ( + "Assert-PublicUploadWriteFreezeReady", + "Exit-PublicUploadWriteFreeze", + "upload_write_freeze", + '"USER_UPLOAD_MANIFEST_PATH"', + '"USER_UPLOAD_MANIFEST_SHA256"', + '"USER_UPLOAD_WRITE_FREEZE_PATH"', + "Fresh public promotion requires a drained upload-write freeze", + "Fresh public rollback did not restore upload-write availability", + ): + with self.subTest(expected=expected): + self.assertIn(expected, source) + + def test_write_release_is_an_irreversible_cutover_boundary(self) -> None: + source = _read(START) + release_stage = source.index('$freshFailureStage = "upload_write_release"') + release_call = source.index( + "Exit-PublicUploadWriteFreeze `", + release_stage, + ) + no_rollback = source.index( + "$freshNoRollback = $true", + release_stage, + ) + receipt_publish = source.index( + '$freshFailureStage = "receipt_publish"', + release_stage, + ) + passed_receipt_write = source.index( + "Write-Utf8TextAtomically `", + receipt_publish, + ) + committed = source.index( + "$freshPromotionCommitted = $true", + receipt_publish, + ) + self.assertLess(no_rollback, release_call) + self.assertLess(release_call, receipt_publish) + self.assertLess(receipt_publish, passed_receipt_write) + self.assertLess(passed_receipt_write, committed) + + trap = source[ + source.index("trap {") : source.index("if (!(Test-Path $Python))") + ] + rollback_guard = trap.index("-not $freshNoRollback") + rollback_call = trap.index("Restore-PriorPublicRuntime `") + self.assertLess(rollback_guard, rollback_call) + + # Exit deletes the sentinel before polling health. A timeout after that + # deletion must not make the trap restore the prior upload root. + exit_function = source[ + source.index("function Exit-PublicUploadWriteFreeze") : source.index( + "function Assert-PublicUploadWritesAvailable" + ) + ] + self.assertLess( + exit_function.index("[System.IO.File]::Delete($FreezePath)"), + exit_function.index("Get-JsonHealth"), + ) + + def test_boot_and_watchdog_skip_mutation_during_valid_active_freeze(self) -> None: + for path in (BOOT, WATCH): + source = _read(path) + with self.subTest(path=path.name): + freeze_guard = source.index( + "promotion-in-progress: valid drained upload freeze is active" + ) + self.assertIn("$promotionFreeze.active -eq $true", source) + self.assertIn("$promotionFreeze.valid -eq $true", source) + self.assertIn("[int]$promotionFreeze.in_flight -eq 0", source) + self.assertIn( + "[string]$promotionFreeze.token_sha256 -ceq", + source, + ) + if path.name == "boot-public-runtime.ps1": + mutation = source.index("& $startScript @startArgs") + else: + mutation = source.index("& $startScript @startArgs") + self.assertLess(freeze_guard, mutation) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_upload_root.py b/scripts/test_public_runtime_upload_root.py new file mode 100644 index 0000000..e0db612 --- /dev/null +++ b/scripts/test_public_runtime_upload_root.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import importlib.util +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS = Path(__file__).resolve().parent +HELPER = SCRIPTS / "public-runtime-upload-root.ps1" +PROBE = SCRIPTS / "probe-public-runtime-upload-root.py" +START = SCRIPTS / "start-public-runtime.ps1" +BOOT = SCRIPTS / "boot-public-runtime.ps1" +WATCH = SCRIPTS / "watch-public-runtime.ps1" +INSTALL = SCRIPTS / "install-public-runtime-task.ps1" +REGISTER = SCRIPTS / "register-boot-task.ps1" +HIDDEN = SCRIPTS / "watch-public-runtime-hidden.vbs" + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _run_powershell(script: Path) -> subprocess.CompletedProcess[str]: + powershell = shutil.which("powershell.exe") + if powershell is None: + raise unittest.SkipTest("Windows PowerShell 5.1 is unavailable") + return subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script), + ], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + ) + + +class PublicRuntimeUploadRootContractTest(unittest.TestCase): + def test_call_chain_owns_one_explicit_upload_root(self) -> None: + self.assertTrue(HELPER.is_file()) + self.assertTrue(PROBE.is_file()) + start = _read(START) + boot = _read(BOOT) + watch = _read(WATCH) + install = _read(INSTALL) + register = _read(REGISTER) + hidden = _read(HIDDEN) + + for source in (start, boot, watch, install, register): + with self.subTest(source=source[:24]): + self.assertIn("[string]$UserUploadDir", source) + self.assertIn("Resolve-PublicRuntimeUploadRoot", source) + self.assertIn("scripts\\public-runtime-upload-root.ps1", source) + self.assertIn("scripts\\probe-public-runtime-upload-root.py", source) + + self.assertIn("$env:USER_UPLOAD_DIR = $resolvedUserUploadDir", start) + self.assertIn('"USER_UPLOAD_DIR"', start) + self.assertIn("user_upload_root = $resolvedUserUploadDir", start) + self.assertIn('"-UserUploadDir", $resolvedUserUploadDir', boot) + self.assertIn("UserUploadDir = $resolvedUserUploadDir", watch) + self.assertIn('"-UserUploadDir `"$resolvedUserUploadDir`""', install) + self.assertIn('"-UserUploadDir `"$resolvedUserUploadDir`""', register) + self.assertIn("'-UserUploadDir'", hidden) + + def test_watchdog_check_only_proves_running_api_storage_contract(self) -> None: + watch = _read(WATCH) + provenance_comment = watch.index( + "# health probe, failcount 기록, 프로세스 재기동보다 먼저" + ) + source_gate = watch.index("Assert-StableSourceProvenance", provenance_comment) + helper_load = watch.index(". $uploadRootContract", source_gate) + upload_preflight = watch.index( + "$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot", helper_load + ) + health_checks = watch.index("$checks = @(") + runtime_probe = watch.index("Test-PublicRuntimeApiUploadRoot", health_checks) + check_only = watch.index("if ($CheckOnly)", health_checks) + restart = watch.index("& $startScript @startArgs") + self.assertLess(source_gate, helper_load) + self.assertLess(helper_load, upload_preflight) + self.assertLess(upload_preflight, health_checks) + self.assertLess(health_checks, runtime_probe) + self.assertLess(runtime_probe, check_only) + self.assertLess(check_only, restart) + self.assertIn('(Test-PublicRuntimeApiUploadRoot `', watch[health_checks:check_only]) + self.assertIn('($FailedNames -contains "api-upload-root")', watch) + + def test_boot_and_registrars_verify_source_before_loading_upload_helper(self) -> None: + start = _read(START) + fresh_gate = start.index("Assert-FreshPublicProvenanceContract `", start.index("if ($RequireFreshPublicProvenance)")) + start_helper = start.index(". $uploadRootContract", fresh_gate) + self.assertLess(fresh_gate, start_helper) + + boot = _read(BOOT) + boot_comment = boot.index( + "# Docker/DB/process mutation보다 먼저 stable source를 매 실행 재검증한다." + ) + boot_gate = boot.index("Assert-StableSourceProvenance", boot_comment) + self.assertLess(boot_gate, boot.index(". $uploadRootContract", boot_gate)) + + for path, helper_name in ( + (INSTALL, "$uploadRootContract"), + (REGISTER, "$UploadRootContract"), + ): + source = _read(path) + clean_gate = source.index('$dirty = Invoke-GitText -Arguments @("status"') + helper_load = source.index(f". {helper_name}") + task_mutation = source.index("Register-ScheduledTask") + with self.subTest(path=path.name): + self.assertLess(clean_gate, helper_load) + self.assertLess(helper_load, task_mutation) + + @unittest.skipUnless(shutil.which("git.exe"), "git.exe is unavailable") + def test_path_preflight_rejects_relative_and_source_overlap_in_powershell_51( + self, + ) -> None: + self.assertTrue(HELPER.is_file()) + with tempfile.TemporaryDirectory(prefix="vignette-upload-contract-") as temp: + root = Path(temp) + source = root / "source" + source.mkdir() + subprocess.run( + [shutil.which("git.exe") or "git.exe", "-C", str(source), "init"], + check=True, + capture_output=True, + ) + upload = root / "stable-uploads" + file_target = root / "not-a-directory" + file_target.write_text("x", encoding="utf-8") + harness = root / "upload-root-harness.ps1" + quoted_helper = str(HELPER).replace("'", "''") + quoted_root = str(root).replace("'", "''") + quoted_source = str(source).replace("'", "''") + quoted_upload = str(upload).replace("'", "''") + quoted_file = str(file_target).replace("'", "''") + harness.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quoted_helper}' +function Assert-Rejected([string]$Candidate) {{ + $rejected = $false + try {{ + Resolve-PublicRuntimeUploadRoot ` + -SourceRoot '{quoted_source}' ` + -UploadRoot $Candidate ` + -CreateIfMissing ` + -ProbeWritable | Out-Null + }} catch {{ + $rejected = $true + }} + if (-not $rejected) {{ throw "unsafe upload root accepted: $Candidate" }} +}} +Assert-Rejected 'relative\\uploads' +Assert-Rejected (Join-Path '{quoted_source}' 'uploads') +Assert-Rejected '{quoted_source}' +Assert-Rejected '{quoted_file}' +$junctionTarget = Join-Path '{quoted_root}' 'junction-target' +$junction = Join-Path '{quoted_root}' 'upload-junction' +[System.IO.Directory]::CreateDirectory($junctionTarget) | Out-Null +try {{ + New-Item -ItemType Junction -Path $junction -Target $junctionTarget | Out-Null + Assert-Rejected (Join-Path $junction 'uploads') +}} finally {{ + if ([System.IO.Directory]::Exists($junction)) {{ + [System.IO.Directory]::Delete($junction) + }} +}} +$resolved = Resolve-PublicRuntimeUploadRoot ` + -SourceRoot '{quoted_source}' ` + -UploadRoot '{quoted_upload}' ` + -CreateIfMissing ` + -ProbeWritable +$expected = (Resolve-Path -LiteralPath '{quoted_upload}').Path +if ($resolved -cne $expected) {{ throw "resolved root mismatch: $resolved" }} +if (@(Get-ChildItem -LiteralPath $resolved -Force -Filter '.vignette-write-probe-*.tmp').Count -ne 0) {{ + throw 'writability probe was not cleaned' +}} +""", + encoding="utf-8-sig", + ) + completed = _run_powershell(harness) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_user_upload_environment_is_in_fresh_rollback_snapshot(self) -> None: + start = _read(START) + functions = start[ + start.index("function Save-ManagedEnvironment") : + start.index("function Save-CompleteProcessEnvironment") + ] + with tempfile.TemporaryDirectory(prefix="vignette-upload-rollback-") as temp: + root = Path(temp) + function_file = root / "environment-functions.ps1" + harness = root / "environment-rollback.ps1" + function_file.write_text(functions, encoding="utf-8-sig") + quoted_functions = str(function_file).replace("'", "''") + harness.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quoted_functions}' +$original = [Environment]::GetEnvironmentVariable('USER_UPLOAD_DIR', 'Process') +try {{ + [Environment]::SetEnvironmentVariable('USER_UPLOAD_DIR', 'C:\\prior-uploads', 'Process') + $snapshot = Save-ManagedEnvironment -Names @('USER_UPLOAD_DIR') + [Environment]::SetEnvironmentVariable('USER_UPLOAD_DIR', 'C:\\fresh-uploads', 'Process') + Restore-ManagedEnvironment -Snapshot $snapshot + if ($env:USER_UPLOAD_DIR -cne 'C:\\prior-uploads') {{ throw 'prior upload root was not restored' }} + + [Environment]::SetEnvironmentVariable('USER_UPLOAD_DIR', $null, 'Process') + $missing = Save-ManagedEnvironment -Names @('USER_UPLOAD_DIR') + [Environment]::SetEnvironmentVariable('USER_UPLOAD_DIR', 'C:\\fresh-uploads', 'Process') + Restore-ManagedEnvironment -Snapshot $missing + if ($null -ne [Environment]::GetEnvironmentVariable('USER_UPLOAD_DIR', 'Process')) {{ + throw 'missing upload root was not restored as missing' + }} +}} finally {{ + [Environment]::SetEnvironmentVariable('USER_UPLOAD_DIR', $original, 'Process') +}} +""", + encoding="utf-8-sig", + ) + completed = _run_powershell(harness) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_process_probe_reports_missing_and_mismatched_upload_roots(self) -> None: + self.assertTrue(PROBE.is_file()) + spec = importlib.util.spec_from_file_location("upload_probe", PROBE) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader if spec else None) + module = importlib.util.module_from_spec(spec) + assert spec is not None and spec.loader is not None + spec.loader.exec_module(module) + + expected = r"C:\Users\tester\AppData\Local\Vignette\public-runtime\uploads" + argv = [ + "python.exe", + "-m", + "uvicorn", + "app.main:app", + "--host", + "127.0.0.1", + "--port", + "8001", + "--workers", + "1", + ] + passed = module.evaluate_snapshots( + [{"pid": 10, "argv": argv, "user_upload_root": expected}], + expected_root=expected, + api_port=8001, + ) + self.assertEqual(passed[0], 0) + self.assertEqual(passed[1]["status"], "passed") + + missing = module.evaluate_snapshots( + [{"pid": 10, "argv": argv, "user_upload_root": None}], + expected_root=expected, + api_port=8001, + ) + self.assertNotEqual(missing[0], 0) + self.assertEqual(missing[1]["reason"], "user_upload_dir_missing") + + mismatched = module.evaluate_snapshots( + [{"pid": 10, "argv": argv, "user_upload_root": r"D:\\release\\uploads"}], + expected_root=expected, + api_port=8001, + ) + self.assertNotEqual(mismatched[0], 0) + self.assertEqual(mismatched[1]["reason"], "user_upload_dir_drift") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_public_runtime_watchdog_provenance.py b/scripts/test_public_runtime_watchdog_provenance.py index db8b877..2599de4 100644 --- a/scripts/test_public_runtime_watchdog_provenance.py +++ b/scripts/test_public_runtime_watchdog_provenance.py @@ -17,6 +17,10 @@ HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs" TASK_LAUNCHER = SCRIPTS / "watch-public-runtime-task.vbs" START = SCRIPTS / "start-public-runtime.ps1" VOICE_PROBE = SCRIPTS / "probe-public-voice-sidecars.py" +UPLOAD_ROOT_CONTRACT = SCRIPTS / "public-runtime-upload-root.ps1" +UPLOAD_ROOT_PROBE = SCRIPTS / "probe-public-runtime-upload-root.py" +UPLOAD_MANIFEST_PROBE = SCRIPTS / "validate-public-runtime-upload-manifest.py" +DATABASE_IDENTITY = SCRIPTS / "public_runtime_database_identity.py" REPO_ROOT = SCRIPTS.parent RUNBOOK = REPO_ROOT / "docs" / "ops" / "public-runtime-watchdog.md" LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md" @@ -72,6 +76,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): '"-ExpectedSourceTree $sourceTree"', '"-ExpectedWatchdogSha256 $watchdogSha256"', '"-ExpectedStartScriptSha256 $startScriptSha256"', + '"-UserUploadDir `"$resolvedUserUploadDir`""', "-WorkingDirectory $resolvedSourceRoot", "Watchdog installer is not executing from the pinned stable source root", ): @@ -111,6 +116,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): '"-ExpectedSourceTree $sourceTree"', '"-ExpectedBootScriptSha256 $bootScriptSha256"', '"-ExpectedStartScriptSha256 $startScriptSha256"', + '"-UserUploadDir `"$resolvedUserUploadDir`""', "-WorkingDirectory $resolvedSourceRoot", "Boot task registrar is not executing from the pinned stable source root", ): @@ -247,10 +253,19 @@ exit 0 self.assertIn("$voiceSidecarsAfter = Test-VoiceSidecarStack", WATCHDOG_SOURCE) self.assertIn('-Name "voice-api"', WATCHDOG_SOURCE) self.assertIn("$voiceApiAfter = Test-JsonHealth", WATCHDOG_SOURCE) - self.assertIn( - "(Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)", - BOOT_SOURCE, - ) + boot_skip_gate = BOOT_SOURCE[ + BOOT_SOURCE.index("$webHealthy = Test-WebPreviewHealthy") : + BOOT_SOURCE.index("$startArgs = @(") + ] + for expected in ( + "(Test-ApiControlPlaneHealthy)", + "(Test-EngineHealthy)", + "(Test-VoiceApiHealthy)", + "(Test-VoiceSidecarStack)", + "(Test-PublicRuntimeApiUploadRoot `", + ): + with self.subTest(expected=expected): + self.assertIn(expected, boot_skip_gate) self.assertGreaterEqual(BOOT_SOURCE.count("Test-VoiceApiHealthy"), 3) self.assertIn('"-WhisperPort", $WhisperPort', BOOT_SOURCE) self.assertIn('"-MeloTtsPort", $MeloTtsPort', BOOT_SOURCE) @@ -460,6 +475,7 @@ if ($result.Detail -ne 'exact readiness probe failed') { "-ExpectedSourceTree", "-ExpectedWatchdogSha256", "-ExpectedStartScriptSha256", + "-UserUploadDir", "WorkingDirectory", "Pinned watchdog script path does not match its working directory", "Pinned watchdog source root does not match its working directory", @@ -494,6 +510,7 @@ if ($result.Detail -ne 'exact readiness probe failed') { "-ExpectedSourceTree", "-ExpectedWatchdogSha256", "-ExpectedStartScriptSha256", + "-UserUploadDir", "--porcelain=v1 --untracked-files=normal", ): with self.subTest(expected=expected): @@ -514,6 +531,10 @@ if ($result.Detail -ne 'exact readiness probe failed') { copied_boot_register = scripts / BOOT_REGISTER.name copied_start = scripts / START.name copied_voice_probe = scripts / VOICE_PROBE.name + copied_upload_contract = scripts / UPLOAD_ROOT_CONTRACT.name + copied_upload_probe = scripts / UPLOAD_ROOT_PROBE.name + copied_upload_manifest_probe = scripts / UPLOAD_MANIFEST_PROBE.name + copied_database_identity = scripts / DATABASE_IDENTITY.name copied_task_launcher = scripts / TASK_LAUNCHER.name shutil.copy2(WATCHDOG, copied_watchdog) shutil.copy2(INSTALLER, copied_installer) @@ -521,6 +542,10 @@ if ($result.Detail -ne 'exact readiness probe failed') { shutil.copy2(BOOT_REGISTER, copied_boot_register) shutil.copy2(START, copied_start) shutil.copy2(VOICE_PROBE, copied_voice_probe) + shutil.copy2(UPLOAD_ROOT_CONTRACT, copied_upload_contract) + shutil.copy2(UPLOAD_ROOT_PROBE, copied_upload_probe) + shutil.copy2(UPLOAD_MANIFEST_PROBE, copied_upload_manifest_probe) + shutil.copy2(DATABASE_IDENTITY, copied_database_identity) shutil.copy2(TASK_LAUNCHER, copied_task_launcher) self._git(root, "init") @@ -535,6 +560,10 @@ if ($result.Detail -ne 'exact readiness probe failed') { "scripts/register-boot-task.ps1", "scripts/start-public-runtime.ps1", "scripts/probe-public-voice-sidecars.py", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", "scripts/watch-public-runtime-task.vbs", ) self._git(root, "commit", "-m", "watchdog fixture") @@ -544,6 +573,10 @@ if ($result.Detail -ne 'exact readiness probe failed') { watchdog_hash = sha256(copied_watchdog) boot_hash = sha256(copied_boot) start_hash = sha256(copied_start) + upload_root = Path( + tempfile.mkdtemp(prefix="vignette-watchdog-uploads-") + ) + self.addCleanup(shutil.rmtree, upload_root, True) with copied_start.open("ab") as stream: stream.write(b"\n# dirty fixture\n") @@ -566,6 +599,14 @@ if ($result.Detail -ne 'exact readiness probe failed') { watchdog_hash, "-ExpectedStartScriptSha256", start_hash, + "-UserUploadDir", + str(upload_root), + "-UserUploadManifestPath", + str(root / "private-manifest.json"), + "-ExpectedUserUploadManifestSha256", + "e" * 64, + "-UserUploadWriteFreezePath", + str(root / "private-freeze.json"), "-CheckOnly", "-SkipPublicHealth", "-SkipCloudflaredRestart", @@ -601,6 +642,14 @@ if ($result.Detail -ne 'exact readiness probe failed') { boot_hash, "-ExpectedStartScriptSha256", start_hash, + "-UserUploadDir", + str(upload_root), + "-UserUploadManifestPath", + str(root / "private-manifest.json"), + "-ExpectedUserUploadManifestSha256", + "e" * 64, + "-UserUploadWriteFreezePath", + str(root / "private-freeze.json"), ], check=False, capture_output=True, @@ -625,6 +674,14 @@ if ($result.Detail -ne 'exact readiness probe failed') { str(registrar), "-StableSourceRoot", str(root), + "-UserUploadDir", + str(upload_root), + "-UserUploadManifestPath", + str(root / "private-manifest.json"), + "-ExpectedUserUploadManifestSha256", + "e" * 64, + "-UserUploadWriteFreezePath", + str(root / "private-freeze.json"), ], check=False, capture_output=True, @@ -644,7 +701,7 @@ if ($result.Detail -ne 'exact readiness probe failed') { powershell = shutil.which("powershell.exe") if powershell is None: self.skipTest("Windows PowerShell 5.1 is unavailable") - for script in (WATCHDOG, INSTALLER, BOOT, BOOT_REGISTER): + for script in (WATCHDOG, INSTALLER, BOOT, BOOT_REGISTER, UPLOAD_ROOT_CONTRACT): escaped = str(script.resolve()).replace("'", "''") command = ( "$tokens=$null; $errors=$null; " @@ -710,7 +767,11 @@ if ($result.Detail -ne 'exact readiness probe failed') { f"-ExpectedSourceCommit {'a' * 40} " f"-ExpectedSourceTree {'b' * 40} " f"-ExpectedWatchdogSha256 {'c' * 64} " - f"-ExpectedStartScriptSha256 {'d' * 64}" + f"-ExpectedStartScriptSha256 {'d' * 64} " + f'-UserUploadDir "C:\\Persistent Vignette Uploads" ' + f'-UserUploadManifestPath "C:\\Private Vignette State\\manifest.json" ' + f"-ExpectedUserUploadManifestSha256 {'e' * 64} " + f'-UserUploadWriteFreezePath "C:\\Private Vignette State\\freeze.json"' ) fixture = ( "$script:watchdogTriggered=$false;" diff --git a/scripts/test_start_public_runtime_contract.py b/scripts/test_start_public_runtime_contract.py index ba562b4..14294c3 100644 --- a/scripts/test_start_public_runtime_contract.py +++ b/scripts/test_start_public_runtime_contract.py @@ -16,6 +16,25 @@ WHISPER_START = (SCRIPTS / "start-local-whisper-stt.ps1").read_text( class PublicRuntimeVoiceContractTest(unittest.TestCase): + def test_public_api_is_forced_to_one_uvicorn_worker(self) -> None: + launch_start = PUBLIC_RUNTIME.index( + '$proc = Start-Process -WindowStyle Hidden -FilePath $Python' + ) + api_launch = PUBLIC_RUNTIME[ + launch_start : PUBLIC_RUNTIME.index( + 'Start-Sleep -Seconds 3', launch_start + ) + ] + self.assertIn('"--workers", "1"', api_launch) + self.assertIn( + '"--port", "$ApiPort", "--workers", "1"', + " ".join(api_launch.split()), + ) + self.assertIn( + '@("uvicorn", "app.main:app", "--port", "$ApiPort", "--workers", "1"', + api_launch, + ) + def test_recovery_is_serialized_and_lock_is_always_released(self) -> None: lock = PUBLIC_RUNTIME.index("$recoveryLock = Enter-RecoveryLock") main_try = PUBLIC_RUNTIME.index("try {", lock) @@ -125,10 +144,16 @@ exit 0 if powershell is None: self.skipTest("Windows PowerShell 5.1 is not available") + environment_repair = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index( + "function Repair-CaseInsensitiveProcessEnvironment" + ) : PUBLIC_RUNTIME.index("$resolvedWorkspace") + ].strip() with tempfile.TemporaryDirectory() as temporary_directory: harness = Path(temporary_directory) / "process-exit-code.ps1" harness.write_text( - """ + environment_repair + + """ $process = Start-Process -FilePath 'cmd.exe' ` -ArgumentList @('/c', 'exit 0') ` -NoNewWindow ` @@ -201,7 +226,7 @@ try {{ }} if (-not (Test-Path -LiteralPath '{quoted_child_pid_for_harness}')) {{ exit 2 }} $childProcessId = [int](Get-Content -LiteralPath '{quoted_child_pid_for_harness}' -Raw) - $stopped = @(Stop-ProcessTreeBounded -RootProcessId $root.Id -TimeoutSec 10 -Role 'test tree') + $stopped = @(Stop-ProcessTreeBounded -RootProcess $root -TimeoutSec 10 -Role 'test tree') $exitDeadline = (Get-Date).AddSeconds(10) do {{ $rootAlive = $null -ne (Get-Process -Id $root.Id -ErrorAction SilentlyContinue) @@ -349,7 +374,7 @@ class FreshPublicProvenanceContractTest(unittest.TestCase): "output preflight failed before runtime mutation", "[System.IO.FileMode]::Open", "[System.IO.FileAccess]::ReadWrite", - "[System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup)", + "[System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup, $true)", "[System.IO.File]::Delete($probeTarget)", ): with self.subTest(expected=expected): @@ -363,7 +388,7 @@ class FreshPublicProvenanceContractTest(unittest.TestCase): for expected in ( "[System.IO.FileMode]::CreateNew", "$stream.Flush($true)", - "[System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath)", + "[System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath, $true)", "[System.IO.File]::Move($temporaryPath, $OutputPath)", "[System.IO.File]::Delete($temporaryPath)", ): @@ -411,12 +436,14 @@ $failedPath = Write-FailedFreshPromotionEvidence ` -RollbackSucceeded $true ` -RollbackResult @{{local_health=$true;public_health=$true}} ` -SourceCommit ('a' * 40) ` - -SourceTree ('b' * 40) + -SourceTree ('b' * 40) ` + -UserUploadRoot 'C:\\stable-uploads' $failed = Get-Content -LiteralPath $failedPath -Raw -Encoding UTF8 | ConvertFrom-Json if ((Split-Path -Leaf $failedPath) -notlike '*.failed.log') {{ throw 'failure evidence suffix mismatch' }} if ($failed.status -ne 'failed_rolled_back') {{ throw 'failure evidence status mismatch' }} if ($failed.failure_stage -ne 'receipt_publish') {{ throw 'failure evidence stage mismatch' }} if (-not $failed.rollback.succeeded) {{ throw 'failure evidence rollback mismatch' }} +if ($failed.storage.user_upload_root -cne 'C:\\stable-uploads') {{ throw 'failure evidence upload root mismatch' }} if (@(Get-ChildItem -LiteralPath (Split-Path -Parent $resolved) -Filter '*.tmp').Count -ne 0) {{ throw 'temporary receipt files were not cleaned' }} @@ -478,7 +505,9 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri receipt = PUBLIC_RUNTIME.index("$provenance = [ordered]@{") api_section = PUBLIC_RUNTIME[api:cloud] cloud_section = PUBLIC_RUNTIME[cloud:receipt] - self.assertIn("Stop-UvicornByPort `", api_section) + self.assertIn("Get-ExactLoopbackListenerProcess `", api_section) + self.assertIn('Where-Object { $_.LocalAddress -eq "127.0.0.1" }', PUBLIC_RUNTIME) + self.assertNotIn("Stop-UvicornByPort `", api_section) self.assertIn("-TimeoutSec $ProcessStopTimeoutSeconds", api_section) self.assertIn("-WorkingDirectory $ApiDir", api_section) self.assertIn("did not receive a replacement PID", api_section) @@ -507,6 +536,15 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri self.assertIn("if ($RequireFreshPublicProvenance)", cloud_section) self.assertIn("-ConfigPath $resolvedCloudflaredConfig `", cloud_section) self.assertIn("-ExactPath", cloud_section) + self.assertIn( + "Fresh public tunnel config was reacquired by an unpinned process before launch", + cloud_section, + ) + fresh_branch = cloud_section[ + cloud_section.index("if ($RequireFreshPublicProvenance)") : + cloud_section.index("} else {") + ] + self.assertNotIn("Stop-ProcessesBounded `", fresh_branch) matcher = PUBLIC_RUNTIME[ PUBLIC_RUNTIME.index("function Get-CloudflaredProcessesForConfig") : PUBLIC_RUNTIME.index("function Wait-ProcessIdentity") @@ -537,6 +575,58 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri self.assertNotIn("credential", config_projection) self.assertNotIn("contents", config_projection) + def test_decode_proofs_are_cross_checked_and_bound_to_success_receipts( + self, + ) -> None: + manifest_validation = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("$uploadManifestProof =") : + PUBLIC_RUNTIME.index("if ($RequireFreshPublicProvenance) {", PUBLIC_RUNTIME.index("$uploadManifestProof =") + 1) + ] + for expected in ( + "Get-PreservedDecodeCountProof", + "Get-RequiredDecodeInvalidCountProof", + "Get-CurrentDecodeInvalidCountProof", + "$offlinePreservedDecodeCounts.ValidCount", + "$manifestPreservedDecodeCounts.ValidCount", + "$offlineRequiredDecodeCounts.ObjectCount", + "$manifestRequiredDecodeCounts.ObjectCount", + "$manifestCurrentDecodeCounts.ObjectCount", + "preserved_inventory_sha256", + "preserved_object_set_sha256", + ): + with self.subTest(validation=expected): + self.assertIn(expected, manifest_validation) + + runtime_receipt = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("$provenance = [ordered]@{") : + PUBLIC_RUNTIME.index( + '$freshFailureStage = "receipt_publish"', + PUBLIC_RUNTIME.index("$provenance = [ordered]@{"), + ) + ] + task_recovery_receipt = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("$taskRecoveryReceipt = [ordered]@{") : + PUBLIC_RUNTIME.index( + "Write-Utf8TextAtomically `", + PUBLIC_RUNTIME.index("$taskRecoveryReceipt = [ordered]@{"), + ) + ] + receipt_fields = { + "preserved_decode_valid_count": "manifestPreservedDecodeCounts.ValidCount", + "preserved_decode_invalid_count": "manifestPreservedDecodeCounts.InvalidCount", + "required_decode_invalid_object_count": "manifestRequiredDecodeCounts.ObjectCount", + "required_decode_invalid_reference_count": "manifestRequiredDecodeCounts.ReferenceCount", + "current_decode_invalid_object_count": "manifestCurrentDecodeCounts.ObjectCount", + "current_decode_invalid_reference_count": "manifestCurrentDecodeCounts.ReferenceCount", + } + for receipt_name, receipt in ( + ("runtime", runtime_receipt), + ("task_recovery", task_recovery_receipt), + ): + for field, source in receipt_fields.items(): + with self.subTest(receipt=receipt_name, field=field): + self.assertIn(f"{field} = [int]${source}", receipt) + def test_process_start_and_command_hash_match_topology_psutil_algorithm(self) -> None: identity = PUBLIC_RUNTIME[ PUBLIC_RUNTIME.index("function Wait-ProcessIdentity") : @@ -678,13 +768,20 @@ try {{ f"""$ErrorActionPreference = 'Stop' . '{quoted_functions}' $events = New-Object System.Collections.ArrayList +$script:apiRestored = $false $root = '{quoted_root}' -function Stop-UvicornByPort {{ param($AppImport,$Port,$TimeoutSec); $null=$events.Add('stop-api'); 11 }} +function Get-ExactLoopbackListenerProcess {{ param($Port,$Role); [pscustomobject]@{{ProcessId=404}} }} +function Get-VerifiedProcessFromIdentity {{ + param($Identity,$Role,$TimeoutSec) + if ($null -eq $Identity) {{ return $null }} + [pscustomobject]@{{ProcessId=[int]$Identity.pid}} +}} +function Get-ListenerProcessIds {{ param($Port); if ($script:apiRestored) {{ @(101) }} else {{ @() }} }} function Restore-ManagedEnvironment {{ param($Snapshot); $null=$events.Add('restore-env') }} function Start-PinnedPriorProcess {{ param($Identity,$Role,$StdoutLog,$StderrLog) $null=$events.Add("start-$Role") - if ($Role -eq 'api') {{ [pscustomobject]@{{Id=101}} }} else {{ [pscustomobject]@{{Id=202}} }} + if ($Role -eq 'api') {{ $script:apiRestored=$true; [pscustomobject]@{{Id=101}} }} else {{ [pscustomobject]@{{Id=202}} }} }} function Wait-ProcessIdentity {{ param($ProcessId,$Role,$ExpectedCwd,$TimeoutSec) @@ -712,8 +809,11 @@ function Wait-JsonHealth {{ $payload }} function Test-VoiceSidecarReady {{ param($Component); $true }} -function Get-CloudflaredProcessesForConfig {{ param($ConfigPath,[switch]$ExactPath); @([pscustomobject]@{{ProcessId=303}}) }} -function Stop-ProcessesBounded {{ param($Processes,$TimeoutSec,$Role); $null=$events.Add('stop-cloudflared'); 303 }} +function Get-CloudflaredProcessesForConfig {{ param($ConfigPath,[switch]$ExactPath); @() }} +function Stop-ProcessesBounded {{ + param($Processes,$TimeoutSec,$Role) + if ($Role -like '*API*') {{ $null=$events.Add('stop-api'); 404 }} else {{ $null=$events.Add('stop-cloudflared'); 303 }} +}} function ConvertTo-SafeProcessIdentity {{ param($Identity); $Identity }} $configPath = Join-Path $root 'cloudflared.yml' @@ -723,11 +823,15 @@ $priorVoice = [ordered]@{{ stt_provider='openai';stt_model='gpt-4o-mini-transcribe'; tts_provider='openai';tts_model='gpt-4o-mini-tts';uvicorn_ws_max_queue=4 }} -$priorApi=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} -$priorCloud=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$priorApi=[ordered]@{{pid=11;started_at_utc='2026-08-09T00:00:00Z';executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$priorCloud=[ordered]@{{pid=22;started_at_utc='2026-08-09T00:00:00Z';executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$replacementApi=[ordered]@{{pid=404;started_at_utc='2026-08-09T00:00:00Z';executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$replacementCloud=[ordered]@{{pid=303;started_at_utc='2026-08-09T00:00:00Z';executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} $result = Restore-PriorPublicRuntime ` -PriorApi $priorApi ` -PriorCloudflared $priorCloud ` + -ReplacementApi $replacementApi ` + -ReplacementCloudflared $replacementCloud ` -PriorLocalVoiceContract $priorVoice ` -PriorPublicVoiceContract $priorVoice ` -EnvironmentSnapshot @{{}} ` @@ -821,11 +925,11 @@ if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'v self.assertEqual(status.stdout, b"") def test_fresh_cutover_requires_and_restores_a_pinned_prior_runtime(self) -> None: - prior_capture = PUBLIC_RUNTIME.index("$priorApiProcesses = @(") + prior_capture = PUBLIC_RUNTIME.index("$priorApiListenerProof =") mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"') self.assertLess(prior_capture, mutation) for expected in ( - "requires exactly one prior API process for transactional rollback", + "requires one exact prior API listener identity", "requires exactly one prior cloudflared process for transactional rollback", "$freshPriorApiIdentity = Wait-ProcessIdentity", "$freshPriorCloudflaredIdentity = Wait-ProcessIdentity", @@ -863,13 +967,36 @@ if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'v stt_start = PUBLIC_RUNTIME.index("& $WhisperStartScript `") tts_start = PUBLIC_RUNTIME.index("& $MeloTtsStartScript `") self.assertLess( - PUBLIC_RUNTIME.index("Fresh public promotion will not mutate local_whisper"), + PUBLIC_RUNTIME.index("Public upload-root promotion will not mutate local_whisper"), stt_start, ) self.assertLess( - PUBLIC_RUNTIME.index("Fresh public promotion will not mutate MeloTTS"), + PUBLIC_RUNTIME.index("Public upload-root promotion will not mutate MeloTTS"), tts_start, ) + self.assertGreaterEqual( + PUBLIC_RUNTIME.count("$RequireFreshPublicProvenance -or $offlineBootstrapMode"), + 2, + ) + + def test_offline_bootstrap_never_stops_a_reappeared_api_listener(self) -> None: + listener_probe = PUBLIC_RUNTIME.index( + '$apiListenerProcess = Get-ExactLoopbackListenerProcess `' + ) + offline_abort = PUBLIC_RUNTIME.index( + "if ($offlineBootstrapMode -and $null -ne $apiListenerProcess)", + listener_probe, + ) + exact_stop = PUBLIC_RUNTIME.index( + "Stop-ProcessesBounded `", + offline_abort, + ) + self.assertLess(listener_probe, offline_abort) + self.assertLess(offline_abort, exact_stop) + self.assertIn( + "Offline bootstrap API listener reappeared after the quiescence receipt", + PUBLIC_RUNTIME[offline_abort:exact_stop], + ) def test_success_and_rollback_require_canonical_public_engine_and_voice(self) -> None: rollback = PUBLIC_RUNTIME[ @@ -894,6 +1021,42 @@ if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'v self.assertIn("$health.engine -eq $true", success_section) self.assertIn("Test-VoiceSidecarReady", success_section) + def test_fresh_tunnel_identity_is_unique_at_release_and_rollback_closes_ingress_first( + self, + ) -> None: + rollback = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") : + PUBLIC_RUNTIME.index("function Stop-NodeByPortHint") + ] + tunnel_stop = rollback.index("$verifiedReplacementCloudflared =") + api_listener = rollback.index("$replacementApiListener =") + self.assertLess(tunnel_stop, api_listener) + + provenance_json = PUBLIC_RUNTIME.index( + "$provenanceJson = ConvertTo-Json -InputObject $provenance" + ) + boundary_tunnel = PUBLIC_RUNTIME.index( + "$boundaryTunnelOwners = @(", provenance_json + ) + no_rollback = PUBLIC_RUNTIME.index("$freshNoRollback = $true", boundary_tunnel) + release = PUBLIC_RUNTIME.index( + "Exit-PublicUploadWriteFreeze `", no_rollback + ) + public_unfrozen = PUBLIC_RUNTIME.index( + "-Uri $CanonicalPublicHealthUrl `", release + ) + receipt_stage = PUBLIC_RUNTIME.index( + '$freshFailureStage = "receipt_publish"', public_unfrozen + ) + self.assertLess(provenance_json, boundary_tunnel) + self.assertLess(boundary_tunnel, no_rollback) + self.assertLess(no_rollback, release) + self.assertLess(release, public_unfrozen) + self.assertLess(public_unfrozen, receipt_stage) + self.assertIn("$freeze.active -eq $false", PUBLIC_RUNTIME[public_unfrozen:receipt_stage]) + self.assertIn("$freeze.valid -eq $true", PUBLIC_RUNTIME[public_unfrozen:receipt_stage]) + self.assertIn("[int]$freeze.in_flight -eq 0", PUBLIC_RUNTIME[public_unfrozen:receipt_stage]) + def test_failed_cutover_emits_metadata_only_rollback_evidence(self) -> None: failure_writer = PUBLIC_RUNTIME[ PUBLIC_RUNTIME.index("function Write-FailedFreshPromotionEvidence") : @@ -904,7 +1067,7 @@ if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'v '"failed_rolled_back"', '"failed_rollback"', "failure_stage = $FailureStage", - "attempted = $true", + "attempted = $RollbackAttempted", "succeeded = $RollbackSucceeded", "Write-Utf8TextAtomically", ): diff --git a/scripts/validate-public-runtime-offline-quiescence.py b/scripts/validate-public-runtime-offline-quiescence.py new file mode 100644 index 0000000..1a9869d --- /dev/null +++ b/scripts/validate-public-runtime-offline-quiescence.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Validate a privacy-safe legacy bootstrap quiescence receipt.""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib.util +import json +import sys +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +INITIALIZER = SCRIPT_DIR / "initialize-public-runtime-upload-root.py" + + +async def _current_database_target_sha256() -> str: + import asyncpg + + from app.config import settings + from public_runtime_database_identity import connected_database_target_sha256 + + connection = await asyncpg.connect( + settings.database_url, command_timeout=settings.db_command_timeout + ) + try: + async with connection.transaction(isolation="repeatable_read", readonly=True): + return await connected_database_target_sha256(connection) + finally: + await connection.close() + + +def _load_initializer(): + spec = importlib.util.spec_from_file_location( + "vignette_upload_initializer_contract", INITIALIZER + ) + if spec is None or spec.loader is None: + raise RuntimeError("offline_quiescence_validator_unavailable") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--receipt-path", required=True) + parser.add_argument("--expected-receipt-sha256", required=True) + parser.add_argument("--manifest-path", required=True) + parser.add_argument("--expected-manifest-sha256", required=True) + parser.add_argument("--stable-source-root", required=True) + parser.add_argument("--upload-root", required=True) + parser.add_argument("--expected-legacy-source-commit", required=True) + parser.add_argument("--expected-legacy-source-tree", required=True) + args = parser.parse_args() + module = _load_initializer() + try: + manifest_path = module._validated_regular_file(Path(args.manifest_path)) + if module.sha256_file(manifest_path) != args.expected_manifest_sha256: + raise module.InitializationError("manifest_hash_drift") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + offline = manifest.get("offline_quiescence") if isinstance(manifest, dict) else None + if not isinstance(offline, dict) or set(offline) != { + "receipt_sha256", + "database_target_sha256", + "source_root_sha256s", + "source_root_set_sha256", + "preserved_object_count", + "preserved_total_size_bytes", + "preserved_inventory_sha256", + "preserved_decode_valid_count", + "preserved_decode_invalid_count", + "required_decode_invalid_object_count", + "required_decode_invalid_reference_count", + "reference_count", + "unique_object_count", + "reference_set_sha256", + }: + raise module.InitializationError("manifest_offline_quiescence_missing") + if offline.get("receipt_sha256") != args.expected_receipt_sha256: + raise module.InitializationError("manifest_quiescence_receipt_drift") + proof = module.validate_offline_quiescence_receipt( + receipt_path=Path(args.receipt_path), + expected_receipt_sha256=args.expected_receipt_sha256, + stable_source_root=Path(args.stable_source_root), + upload_root=Path(args.upload_root), + expected_source_commit=args.expected_legacy_source_commit, + expected_source_tree=args.expected_legacy_source_tree, + expected_source_roots=None, + expected_source_root_sha256s=offline.get("source_root_sha256s"), + expected_source_root_set_sha256=offline.get( + "source_root_set_sha256" + ), + expected_database_target_sha256=asyncio.run( + _current_database_target_sha256() + ), + expected_preserved_object_count=offline.get( + "preserved_object_count" + ), + expected_preserved_total_size_bytes=offline.get( + "preserved_total_size_bytes" + ), + expected_preserved_inventory_sha256=offline.get( + "preserved_inventory_sha256" + ), + expected_preserved_decode_valid_count=offline.get( + "preserved_decode_valid_count" + ), + expected_preserved_decode_invalid_count=offline.get( + "preserved_decode_invalid_count" + ), + expected_required_decode_invalid_object_count=offline.get( + "required_decode_invalid_object_count" + ), + expected_required_decode_invalid_reference_count=offline.get( + "required_decode_invalid_reference_count" + ), + expected_reference_count=offline.get("reference_count"), + expected_unique_object_count=offline.get("unique_object_count"), + expected_reference_set_sha256=offline.get("reference_set_sha256"), + ) + if list(proof.source_root_sha256s) != offline.get( + "source_root_sha256s" + ) or proof.source_root_set_sha256 != offline.get( + "source_root_set_sha256" + ): + raise module.InitializationError("manifest_legacy_source_root_drift") + except (OSError, ValueError, module.InitializationError): + print( + json.dumps( + {"status": "failed", "reason": "offline_quiescence_contract_failed"}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 1 + except Exception: + print( + json.dumps( + {"status": "failed", "reason": "offline_quiescence_validation_unavailable"}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 1 + print( + json.dumps( + { + "status": "passed", + "receipt_sha256": proof.receipt_sha256, + "database_target_sha256": proof.database_target_sha256, + "source_root_set_sha256": proof.source_root_set_sha256, + "preserved_object_count": proof.preserved_object_count, + "preserved_total_size_bytes": proof.preserved_total_size_bytes, + "preserved_inventory_sha256": proof.preserved_inventory_sha256, + "preserved_decode_valid_count": ( + proof.preserved_decode_valid_count + ), + "preserved_decode_invalid_count": ( + proof.preserved_decode_invalid_count + ), + "required_decode_invalid_object_count": ( + proof.required_decode_invalid_object_count + ), + "required_decode_invalid_reference_count": ( + proof.required_decode_invalid_reference_count + ), + "reference_count": proof.reference_count, + "unique_object_count": proof.unique_object_count, + "reference_set_sha256": proof.reference_set_sha256, + "listener_absent": True, + "tunnel_absent": True, + }, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate-public-runtime-upload-manifest.py b/scripts/validate-public-runtime-upload-manifest.py new file mode 100644 index 0000000..67b7371 --- /dev/null +++ b/scripts/validate-public-runtime-upload-manifest.py @@ -0,0 +1,136 @@ +"""Validate a privacy-safe public-avatar migration manifest.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parent.parent +API_ROOT = REPO_ROOT / "apps" / "api" +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +os.chdir(API_ROOT) + +from app.config import settings # noqa: E402 +from app.upload_storage import ( # noqa: E402 + validate_current_avatar_references, + validate_upload_manifest, +) +from public_runtime_database_identity import ( # noqa: E402 + connected_database_target_sha256, +) + + +async def _load_current_runtime_db_state() -> tuple[list[str], str]: + import asyncpg + + connection = await asyncpg.connect(settings.database_url) + try: + async with connection.transaction(isolation="repeatable_read", readonly=True): + target_sha256 = await connected_database_target_sha256(connection) + rows = await connection.fetch( + """ + SELECT avatar_url + FROM app.app_user + WHERE btrim(avatar_url) LIKE '/uploads/%' + ORDER BY avatar_url + """ + ) + return [str(row["avatar_url"]) for row in rows], target_sha256 + finally: + await connection.close() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--upload-root", required=True) + parser.add_argument("--manifest-path", required=True) + parser.add_argument("--expected-manifest-sha256", required=True) + parser.add_argument("--expected-write-freeze-path", required=True) + args = parser.parse_args() + try: + current_urls, database_target = asyncio.run(_load_current_runtime_db_state()) + proof = validate_upload_manifest( + upload_root=Path(args.upload_root), + manifest_path=Path(args.manifest_path), + expected_manifest_sha256=args.expected_manifest_sha256, + expected_write_freeze_path=Path(args.expected_write_freeze_path), + expected_database_target_sha256=database_target, + verify_preserved_objects=True, + reject_unbound_extras=False, + ) + current = validate_current_avatar_references( + upload_root=Path(args.upload_root), + avatar_urls=current_urls, + initial_manifest=proof, + expected_database_target_sha256=database_target, + ) + except (OSError, ValueError): + print( + json.dumps( + {"status": "failed", "reason": "upload_storage_contract_failed"}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 1 + except Exception: + print( + json.dumps( + {"status": "failed", "reason": "current_db_inventory_unavailable"}, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 1 + print( + json.dumps( + { + "status": "passed", + "manifest_sha256": proof.manifest_sha256, + "root_path_sha256": proof.root_path_sha256, + "required_object_count": proof.required_object_count, + "preserved_object_count": proof.preserved_object_count, + "preserved_total_size_bytes": proof.preserved_total_size_bytes, + "preserved_decode_valid_count": ( + proof.preserved_decode_valid_count + ), + "preserved_decode_invalid_count": ( + proof.preserved_decode_invalid_count + ), + "required_decode_invalid_object_count": ( + proof.required_decode_invalid_object_count + ), + "required_decode_invalid_reference_count": ( + proof.required_decode_invalid_reference_count + ), + "preserved_object_set_sha256": proof.preserved_object_set_sha256, + "required_reference_count": proof.required_reference_count, + "reference_set_sha256": proof.reference_set_sha256, + "write_freeze_token_sha256": proof.write_freeze_token_sha256, + "current_object_count": current.object_count, + "current_reference_count": current.reference_count, + "current_reference_set_sha256": current.reference_set_sha256, + "current_decode_invalid_object_count": ( + current.decode_invalid_object_count + ), + "current_decode_invalid_reference_count": ( + current.decode_invalid_reference_count + ), + "database_target_sha256": database_target, + }, + ensure_ascii=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/watch-public-runtime-hidden.vbs b/scripts/watch-public-runtime-hidden.vbs index bbac35d..fc1d981 100644 --- a/scripts/watch-public-runtime-hidden.vbs +++ b/scripts/watch-public-runtime-hidden.vbs @@ -22,7 +22,7 @@ command = "$ErrorActionPreference='Stop';" & _ "$expectedFileArg='-File '+$quote+$expectedScript+$quote;$expectedRootArg='-StableSourceRoot '+$quote+$root+$quote;" & _ "if($action.Arguments.IndexOf($expectedFileArg,[StringComparison]::OrdinalIgnoreCase) -lt 0){throw 'Pinned watchdog script path does not match its working directory'};" & _ "if($action.Arguments.IndexOf($expectedRootArg,[StringComparison]::OrdinalIgnoreCase) -lt 0){throw 'Pinned watchdog source root does not match its working directory'};" & _ - "$required=@('-StableSourceRoot','-ExpectedSourceCommit','-ExpectedSourceTree','-ExpectedWatchdogSha256','-ExpectedStartScriptSha256');" & _ + "$required=@('-StableSourceRoot','-ExpectedSourceCommit','-ExpectedSourceTree','-ExpectedWatchdogSha256','-ExpectedStartScriptSha256','-UserUploadDir','-UserUploadManifestPath','-ExpectedUserUploadManifestSha256','-UserUploadWriteFreezePath');" & _ "foreach($marker in $required){if($action.Arguments.IndexOf($marker,[StringComparison]::Ordinal) -lt 0){throw ('Unpinned watchdog task action: missing '+$marker)}};" & _ "if($action.Arguments -match '(?i)(?:^|\s)-Workspace(?:\s|$)'){throw 'Legacy shared-worktree watchdog action is forbidden'};" & _ "Start-ScheduledTask -TaskName 'VignettePublicRuntimeWatchdog'" diff --git a/scripts/watch-public-runtime.ps1 b/scripts/watch-public-runtime.ps1 index 6563cc7..d114a48 100644 --- a/scripts/watch-public-runtime.ps1 +++ b/scripts/watch-public-runtime.ps1 @@ -13,6 +13,12 @@ [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{64}$")] [string]$ExpectedStartScriptSha256, + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedPythonSha256 = "", + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedCloudflaredSha256 = "", + [ValidatePattern("^$|^[0-9a-fA-F]{64}$")] + [string]$ExpectedCloudflaredConfigSha256 = "", [int]$ApiPort = 8001, [int]$WebPort = 5174, [int]$EnginePort = 9099, @@ -23,6 +29,15 @@ [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", + [Parameter(Mandatory = $true)] + [string]$UserUploadDir, + [Parameter(Mandatory = $true)] + [string]$UserUploadManifestPath, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[0-9a-f]{64}$")] + [string]$ExpectedUserUploadManifestSha256, + [Parameter(Mandatory = $true)] + [string]$UserUploadWriteFreezePath, [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health", [string[]]$AdditionalPublicHealthUrls = @(), # 게이트웨이가 shared secret으로 떠 있으면 /ready는 인증이 필요하다(/health만 면제). @@ -41,6 +56,10 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" +$uploadRootContract = Join-Path $resolvedSourceRoot "scripts\public-runtime-upload-root.ps1" +$uploadRootProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-runtime-upload-root.py" +$uploadManifestProbe = Join-Path $resolvedSourceRoot "scripts\validate-public-runtime-upload-manifest.py" +$databaseIdentityHelper = Join-Path $resolvedSourceRoot "scripts\public_runtime_database_identity.py" function Invoke-GitText { param([string[]]$Arguments) @@ -62,6 +81,11 @@ function Assert-StableSourceProvenance { if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) { throw "Pinned voice sidecar probe not found at $voiceSidecarProbe" } + foreach ($uploadContractFile in @($uploadRootContract, $uploadRootProbe, $uploadManifestProbe, $databaseIdentityHelper)) { + if (-not (Test-Path -LiteralPath $uploadContractFile -PathType Leaf)) { + throw "Pinned upload-root contract file not found at $uploadContractFile" + } + } $runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path if (-not [string]::Equals( @@ -108,7 +132,13 @@ function Assert-StableSourceProvenance { foreach ($relativePath in @( "scripts/watch-public-runtime.ps1", "scripts/start-public-runtime.ps1", - "scripts/probe-public-voice-sidecars.py" + "scripts/probe-public-voice-sidecars.py", + "scripts/public-runtime-upload-root.ps1", + "scripts/probe-public-runtime-upload-root.py", + "scripts/validate-public-runtime-upload-manifest.py", + "scripts/public_runtime_database_identity.py", + "apps/api/app/upload_storage.py", + "apps/api/app/upload_runtime.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -121,11 +151,45 @@ function Assert-StableSourceProvenance { if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) { throw "Pinned start script SHA256 drift" } + foreach ($pin in @( + [pscustomobject]@{ Path = $Python; Expected = $ExpectedPythonSha256; Role = "Python" }, + [pscustomobject]@{ Path = $Cloudflared; Expected = $ExpectedCloudflaredSha256; Role = "cloudflared" }, + [pscustomobject]@{ Path = $CloudflaredConfig; Expected = $ExpectedCloudflaredConfigSha256; Role = "cloudflared config" } + )) { + if ([string]::IsNullOrWhiteSpace([string]$pin.Expected)) { + continue + } + if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) { + throw "Pinned $($pin.Role) is unavailable" + } + $actual = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -cne ([string]$pin.Expected).ToLowerInvariant()) { + throw "Pinned $($pin.Role) SHA256 drift" + } + } } # health probe, failcount 기록, 프로세스 재기동보다 먼저 source provenance를 닫는다. # 검증 실패는 운영 프로세스를 그대로 보존한 채 non-zero로 끝난다. Assert-StableSourceProvenance +. $uploadRootContract +$uploadRootResolveArgs = @{ + SourceRoot = $resolvedSourceRoot + UploadRoot = $UserUploadDir +} +if (-not $CheckOnly) { + $uploadRootResolveArgs["ProbeWritable"] = $true +} +$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot @uploadRootResolveArgs +$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadManifestPath ` + -RequireFile +$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath ` + -SourceRoot $resolvedSourceRoot ` + -UploadRoot $resolvedUserUploadDir ` + -StatePath $UserUploadWriteFreezePath if (!$LogPath) { $LogPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.log" @@ -337,7 +401,11 @@ function Test-PublicRuntimeHardDown { ($FailedNames -contains "web-preview") -and ($FailedNames -contains "voice-sidecars") ) - return ($FailedNames -contains "cloudflared") -or $allLocalSurfacesDown + return ( + ($FailedNames -contains "cloudflared") -or + ($FailedNames -contains "api-upload-root") -or + $allLocalSurfacesDown + ) } # engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. @@ -345,6 +413,57 @@ function Test-PublicRuntimeHardDown { # 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0). # /ready는 게이트웨이 readiness 캐시(ENGINE_READY_TTL_SECONDS)를 그대로 쓰므로 # 매 주기 LLM 호출로 이어지지 않는다. 콜드 스폰 여유로 타임아웃만 넉넉히 준다. +$uploadManifestCheck = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $uploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +$expectedDatabaseTargetSha256 = "0000000000000000000000000000000000000000000000000000000000000000" +if ($uploadManifestCheck.Ok) { + $candidateDatabaseTargetSha256 = [string]$uploadManifestCheck.Payload.database_target_sha256 + if ($candidateDatabaseTargetSha256 -match "^[0-9a-f]{64}$") { + $expectedDatabaseTargetSha256 = $candidateDatabaseTargetSha256 + } +} +if (Test-Path -LiteralPath $resolvedUserUploadWriteFreezePath -PathType Leaf) { + $promotionHealth = $null + try { + $promotionHealth = Invoke-RestMethod ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -TimeoutSec 5 + } catch { + $promotionHealth = $null + } + $promotionFreeze = $null + if ($null -ne $promotionHealth) { + $promotionFreeze = $promotionHealth.upload_write_freeze + } + if ( + $uploadManifestCheck.Ok -and + $null -ne $promotionFreeze -and + $promotionFreeze.capable -eq $true -and + $promotionFreeze.active -eq $true -and + $promotionFreeze.valid -eq $true -and + [int]$promotionFreeze.in_flight -eq 0 -and + [string]$promotionFreeze.token_sha256 -ceq + [string]$uploadManifestCheck.Payload.write_freeze_token_sha256 + ) { + if ($CheckOnly) { + Write-Output "promotion-in-progress: valid drained upload freeze is active; runtime mutation skipped" + } else { + Write-WatchdogLog "promotion-in-progress: valid drained upload freeze is active; runtime mutation skipped" + } + exit 0 + } + if ($CheckOnly) { + Write-Output "unhealthy: upload freeze sentinel exists without exact active/drained API proof; runtime mutation refused" + } else { + Write-WatchdogLog "ERROR: upload freeze sentinel exists without exact active/drained API proof; refusing runtime mutation" + } + exit 1 +} $checks = @( (Test-JsonHealth ` -Name "engine" ` @@ -355,8 +474,27 @@ $checks = @( (Test-JsonHealth ` -Name "api" ` -Uri "http://127.0.0.1:$ApiPort/health" ` - -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -and + $health.engine -and + $health.upload_write_freeze.capable -eq $true -and + $health.upload_write_freeze.active -eq $false -and + $health.upload_write_freeze.valid -eq $true + } ` -TimeoutSec 60), + $uploadManifestCheck, + (Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort), (Test-JsonHealth ` -Name "voice-api" ` -Uri "http://127.0.0.1:$ApiPort/voice/health" ` @@ -389,7 +527,7 @@ if (!$SkipPublicHealth) { $failed = @($checks | Where-Object { -not $_.Ok }) if ($CheckOnly) { if ($failed.Count -eq 0) { - Write-Output "healthy: $($checks.Name -join ', ')" + Write-Output "healthy: $($checks.Name -join ', '); user_upload_root=$resolvedUserUploadDir" exit 0 } Write-Output "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))" @@ -418,6 +556,10 @@ if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown) { if ($hardDown -and $failCount -lt $FailuresBeforeRestart) { Write-WatchdogLog "immediate restart: public runtime hard-down detected" } +if ($failedNames -contains "api-upload-manifest") { + Write-WatchdogLog "ERROR: immutable upload migration receipt or current DB inventory is invalid; refusing runtime mutation" + exit 1 +} # DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당). # 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에 @@ -442,6 +584,10 @@ $startArgs = @{ Python = $Python Cloudflared = $Cloudflared CloudflaredConfig = $CloudflaredConfig + UserUploadDir = $resolvedUserUploadDir + UserUploadManifestPath = $resolvedUserUploadManifestPath + ExpectedUserUploadManifestSha256 = $ExpectedUserUploadManifestSha256 + UserUploadWriteFreezePath = $resolvedUserUploadWriteFreezePath } if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) { $startArgs["SkipWebRestart"] = $true @@ -463,11 +609,46 @@ try { $apiAfter = Test-JsonHealth ` -Name "api" ` -Uri "http://127.0.0.1:$ApiPort/health" ` - -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -and + $health.engine -and + $health.upload_write_freeze.capable -eq $true -and + $health.upload_write_freeze.active -eq $false -and + $health.upload_write_freeze.valid -eq $true + } ` -TimeoutSec 90 if (!$apiAfter.Ok) { throw "Public API still unhealthy after restart: $($apiAfter.Detail)" } +$uploadManifestAfter = Test-PublicRuntimeUploadManifest ` + -PythonPath $Python ` + -ProbePath $uploadManifestProbe ` + -UploadRoot $resolvedUserUploadDir ` + -ManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath +if (-not $uploadManifestAfter.Ok) { + throw "Public upload migration receipt or current DB inventory is invalid after restart" +} +$expectedDatabaseTargetSha256 = [string]$uploadManifestAfter.Payload.database_target_sha256 +if ($expectedDatabaseTargetSha256 -notmatch "^[0-9a-f]{64}$") { + throw "Public upload inventory proof did not return a valid database target identity after restart" +} +$apiUploadRootAfter = Test-PublicRuntimeApiUploadRoot ` + -PythonPath $Python ` + -ProbePath $uploadRootProbe ` + -ExpectedUploadRoot $resolvedUserUploadDir ` + -ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") ` + -ExpectedManifestPath $resolvedUserUploadManifestPath ` + -ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 ` + -ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath ` + -ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 ` + -ApiPort $ApiPort +if (-not $apiUploadRootAfter.Ok) { + throw "Public API still uses the wrong upload root after restart: $($apiUploadRootAfter.Detail)" +} $voiceApiAfter = Test-JsonHealth ` -Name "voice-api" `