1664 lines
64 KiB
Python
1664 lines
64 KiB
Python
#!/usr/bin/env python3
|
|
"""공개 프로필 아바타 저장소를 명시적으로 초기화한다.
|
|
|
|
이 작업자만 공개 업로드 루트에 디렉터리와 파일을 만들 수 있다. 원본은 읽기만
|
|
하며 대상은 ``xb`` 모드로만 생성한다. 출력과 매니페스트에는 사용자 식별자,
|
|
원본 URL, 파일명 또는 원시 경로를 기록하지 않는다.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterable, Mapping, Sequence
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
API_ROOT = REPO_ROOT / "apps" / "api"
|
|
if str(API_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(API_ROOT))
|
|
|
|
from app.upload_storage import ( # noqa: E402
|
|
WRITE_FREEZE_SCHEMA_VERSION,
|
|
build_privacy_safe_manifest,
|
|
canonical_path_sha256,
|
|
inspect_public_avatar_image,
|
|
public_avatar_relative_path_sha256,
|
|
public_avatar_url_to_relative_path,
|
|
sha256_file,
|
|
)
|
|
from public_runtime_database_identity import ( # noqa: E402
|
|
connected_database_target_sha256,
|
|
)
|
|
|
|
|
|
_ALLOWED_AVATAR_NAME = re.compile(
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.(?:png|jpg|jpeg|webp)$"
|
|
)
|
|
_SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
_GIT_OBJECT_RE = re.compile(r"^[a-f0-9]{40}$")
|
|
_UTC_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$")
|
|
_COPY_CHUNK_SIZE = 1024 * 1024
|
|
OFFLINE_CAPTURE_SCHEMA_VERSION = (
|
|
"vignette.public-upload-offline-quiescence-capture.v2"
|
|
)
|
|
OFFLINE_RECEIPT_SCHEMA_VERSION = (
|
|
"vignette.public-upload-offline-quiescence.v3"
|
|
)
|
|
|
|
|
|
class InitializationError(RuntimeError):
|
|
"""개인정보를 포함하지 않는 안정적인 초기화 실패 코드."""
|
|
|
|
|
|
class _RejectRedirects(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, request, file_pointer, code, message, headers, new_url):
|
|
raise InitializationError("runtime_health_redirect_rejected")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ReferenceInventory:
|
|
objects: tuple[tuple[str, int], ...]
|
|
reference_count: int
|
|
reference_set_sha256: str
|
|
|
|
@property
|
|
def unique_object_count(self) -> int:
|
|
return len(self.objects)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PreservedObject:
|
|
relative_path: str
|
|
source_path: Path
|
|
size_bytes: int
|
|
content_sha256: str
|
|
decode_valid: bool
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PreservedInventory:
|
|
objects: tuple[PreservedObject, ...]
|
|
inventory_sha256: str
|
|
|
|
@property
|
|
def object_count(self) -> int:
|
|
return len(self.objects)
|
|
|
|
@property
|
|
def total_size_bytes(self) -> int:
|
|
return sum(item.size_bytes for item in self.objects)
|
|
|
|
@property
|
|
def decode_valid_count(self) -> int:
|
|
return sum(1 for item in self.objects if item.decode_valid)
|
|
|
|
@property
|
|
def decode_invalid_count(self) -> int:
|
|
return self.object_count - self.decode_valid_count
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CopyProof:
|
|
object_count: int
|
|
copied_count: int
|
|
reused_exact_count: int
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class OfflineQuiescenceProof:
|
|
receipt_path: Path
|
|
receipt_sha256: str
|
|
database_target_sha256: str
|
|
source_root_sha256s: tuple[str, ...]
|
|
source_root_set_sha256: str
|
|
preserved_object_count: int
|
|
preserved_total_size_bytes: int
|
|
preserved_inventory_sha256: str
|
|
preserved_decode_valid_count: int
|
|
preserved_decode_invalid_count: int
|
|
required_decode_invalid_object_count: int
|
|
required_decode_invalid_reference_count: int
|
|
reference_count: int
|
|
unique_object_count: int
|
|
reference_set_sha256: str
|
|
|
|
|
|
def _sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def _json_bytes(value: object) -> bytes:
|
|
return (
|
|
json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
|
+ "\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def _has_reparse_attribute(path: Path) -> bool:
|
|
metadata = path.lstat()
|
|
attributes = getattr(metadata, "st_file_attributes", 0)
|
|
reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
return path.is_symlink() or bool(attributes & reparse_flag)
|
|
|
|
|
|
def _assert_no_reparse_ancestors(path: Path) -> None:
|
|
cursor = path
|
|
while True:
|
|
if cursor.exists() or cursor.is_symlink():
|
|
if _has_reparse_attribute(cursor):
|
|
raise InitializationError("path_reparse_point_rejected")
|
|
parent = cursor.parent
|
|
if parent == cursor:
|
|
return
|
|
cursor = parent
|
|
|
|
|
|
def _validated_absolute_directory(path: Path, *, must_exist: bool = True) -> Path:
|
|
if not path.is_absolute():
|
|
raise InitializationError("directory_must_be_absolute")
|
|
lexical = Path(os.path.abspath(path))
|
|
_assert_no_reparse_ancestors(lexical)
|
|
if must_exist and (not lexical.exists() or not lexical.is_dir()):
|
|
raise InitializationError("required_directory_unavailable")
|
|
resolved = lexical.resolve(strict=must_exist)
|
|
if resolved.parent == resolved:
|
|
raise InitializationError("filesystem_root_rejected")
|
|
_assert_no_reparse_ancestors(resolved)
|
|
return resolved
|
|
|
|
|
|
def _validated_regular_file(path: Path) -> Path:
|
|
if not path.is_absolute():
|
|
raise InitializationError("file_must_be_absolute")
|
|
lexical = Path(os.path.abspath(path))
|
|
_assert_no_reparse_ancestors(lexical)
|
|
try:
|
|
metadata = lexical.lstat()
|
|
except OSError as exc:
|
|
raise InitializationError("required_regular_file_unavailable") from exc
|
|
if _has_reparse_attribute(lexical) or not stat.S_ISREG(metadata.st_mode):
|
|
raise InitializationError("required_regular_file_unavailable")
|
|
return lexical.resolve(strict=True)
|
|
|
|
|
|
def _same_or_within(candidate: Path, parent: Path) -> bool:
|
|
candidate_key = os.path.normcase(str(candidate))
|
|
parent_key = os.path.normcase(str(parent))
|
|
try:
|
|
return os.path.commonpath((candidate_key, parent_key)) == parent_key
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _assert_disjoint(left: Path, right: Path) -> None:
|
|
if _same_or_within(left, right) or _same_or_within(right, left):
|
|
raise InitializationError("private_and_public_roots_must_be_disjoint")
|
|
|
|
|
|
def normalize_avatar_reference(value: object) -> str:
|
|
if not isinstance(value, str):
|
|
raise InitializationError("database_avatar_reference_invalid")
|
|
exact_prefix = "/uploads/profile-avatars/"
|
|
if not value.startswith(exact_prefix):
|
|
raise InitializationError("database_avatar_reference_invalid")
|
|
basename = value[len(exact_prefix) :]
|
|
if not basename or "/" in basename or "\\" in basename:
|
|
raise InitializationError("database_avatar_reference_invalid")
|
|
try:
|
|
relative = public_avatar_url_to_relative_path(value)
|
|
except ValueError as exc:
|
|
raise InitializationError("database_avatar_reference_invalid") from exc
|
|
if relative != f"profile-avatars/{basename}":
|
|
raise InitializationError("database_avatar_reference_invalid")
|
|
if (
|
|
"%" in basename
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in basename)
|
|
or _ALLOWED_AVATAR_NAME.fullmatch(basename) is None
|
|
):
|
|
raise InitializationError("database_avatar_reference_invalid")
|
|
return relative
|
|
|
|
|
|
def build_reference_inventory(
|
|
avatar_urls: Iterable[object], *, expected_reference_count: int
|
|
) -> ReferenceInventory:
|
|
if expected_reference_count < 0:
|
|
raise InitializationError("expected_reference_count_invalid")
|
|
counts: Counter[str] = Counter()
|
|
canonical_names: dict[str, str] = {}
|
|
for raw_value in avatar_urls:
|
|
normalized = normalize_avatar_reference(raw_value)
|
|
identity = normalized.casefold()
|
|
canonical_names.setdefault(identity, normalized)
|
|
counts[identity] += 1
|
|
reference_count = sum(counts.values())
|
|
if reference_count != expected_reference_count:
|
|
raise InitializationError("database_reference_count_mismatch")
|
|
objects = tuple(
|
|
sorted(
|
|
((canonical_names[key], count) for key, count in counts.items()),
|
|
key=lambda item: public_avatar_relative_path_sha256(item[0]),
|
|
)
|
|
)
|
|
hashed_counts = [
|
|
{
|
|
"path_sha256": public_avatar_relative_path_sha256(name),
|
|
"reference_count": count,
|
|
}
|
|
for name, count in objects
|
|
]
|
|
digest = _sha256_bytes(_json_bytes(hashed_counts).rstrip(b"\n"))
|
|
return ReferenceInventory(objects, reference_count, digest)
|
|
|
|
|
|
def _strict_lower_sha256(value: object, error_code: str) -> str:
|
|
if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None:
|
|
raise InitializationError(error_code)
|
|
return value
|
|
|
|
|
|
def _strict_process_identity(value: object) -> dict[str, object]:
|
|
if not isinstance(value, dict) or set(value) != {
|
|
"pid",
|
|
"started_at_utc",
|
|
"executable_sha256",
|
|
"command_line_sha256",
|
|
"cwd_sha256",
|
|
}:
|
|
raise InitializationError("offline_process_identity_invalid")
|
|
pid = value.get("pid")
|
|
started_at = value.get("started_at_utc")
|
|
if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
|
|
raise InitializationError("offline_process_identity_invalid")
|
|
if not isinstance(started_at, str) or _UTC_TIMESTAMP_RE.fullmatch(started_at) is None:
|
|
raise InitializationError("offline_process_identity_invalid")
|
|
return {
|
|
"pid": pid,
|
|
"started_at_utc": started_at,
|
|
"executable_sha256": _strict_lower_sha256(
|
|
value.get("executable_sha256"), "offline_process_identity_invalid"
|
|
),
|
|
"command_line_sha256": _strict_lower_sha256(
|
|
value.get("command_line_sha256"), "offline_process_identity_invalid"
|
|
),
|
|
"cwd_sha256": _strict_lower_sha256(
|
|
value.get("cwd_sha256"), "offline_process_identity_invalid"
|
|
),
|
|
}
|
|
|
|
|
|
def _source_root_bindings(
|
|
source_roots: Sequence[Path],
|
|
) -> tuple[tuple[str, ...], str]:
|
|
hashes: list[str] = []
|
|
seen_paths: set[str] = set()
|
|
for source_root in source_roots:
|
|
validated = _validated_absolute_directory(source_root)
|
|
key = os.path.normcase(str(validated))
|
|
if key in seen_paths:
|
|
raise InitializationError("source_root_duplicate")
|
|
seen_paths.add(key)
|
|
hashes.append(canonical_path_sha256(validated))
|
|
if not hashes:
|
|
raise InitializationError("source_root_required")
|
|
ordered_hashes = tuple(hashes)
|
|
set_sha256 = _sha256_bytes(
|
|
_json_bytes(sorted(ordered_hashes)).rstrip(b"\n")
|
|
)
|
|
return ordered_hashes, set_sha256
|
|
|
|
|
|
def decode_offline_quiescence_capture(
|
|
encoded_capture: str,
|
|
*,
|
|
legacy_source_roots: Sequence[Path],
|
|
expected_source_commit: str,
|
|
expected_source_tree: str,
|
|
) -> dict[str, object]:
|
|
if not isinstance(encoded_capture, str) or not encoded_capture:
|
|
raise InitializationError("offline_quiescence_capture_missing")
|
|
try:
|
|
raw = base64.b64decode(encoded_capture, validate=True)
|
|
if len(raw) > 16 * 1024:
|
|
raise InitializationError("offline_quiescence_capture_invalid")
|
|
payload = json.loads(raw.decode("utf-8"))
|
|
except InitializationError:
|
|
raise
|
|
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise InitializationError("offline_quiescence_capture_invalid") from exc
|
|
if not isinstance(payload, dict) or set(payload) != {
|
|
"schema_version",
|
|
"status",
|
|
"captured_at_utc",
|
|
"source_commit",
|
|
"source_tree",
|
|
"source_root_sha256s",
|
|
"source_root_set_sha256",
|
|
"api_identity",
|
|
"tunnel_identity",
|
|
"listener_absent",
|
|
"tunnel_absent",
|
|
"listener_endpoint_sha256",
|
|
"tunnel_config_sha256",
|
|
}:
|
|
raise InitializationError("offline_quiescence_capture_invalid")
|
|
if (
|
|
payload.get("schema_version") != OFFLINE_CAPTURE_SCHEMA_VERSION
|
|
or payload.get("status") != "quiesced"
|
|
or payload.get("listener_absent") is not True
|
|
or payload.get("tunnel_absent") is not True
|
|
):
|
|
raise InitializationError("offline_quiescence_capture_invalid")
|
|
captured_at = payload.get("captured_at_utc")
|
|
if not isinstance(captured_at, str) or _UTC_TIMESTAMP_RE.fullmatch(captured_at) is None:
|
|
raise InitializationError("offline_quiescence_capture_invalid")
|
|
source_commit = payload.get("source_commit")
|
|
source_tree = payload.get("source_tree")
|
|
if (
|
|
_GIT_OBJECT_RE.fullmatch(expected_source_commit or "") is None
|
|
or _GIT_OBJECT_RE.fullmatch(expected_source_tree or "") is None
|
|
or source_commit != expected_source_commit
|
|
or source_tree != expected_source_tree
|
|
or not isinstance(source_commit, str)
|
|
or _GIT_OBJECT_RE.fullmatch(source_commit) is None
|
|
or not isinstance(source_tree, str)
|
|
or _GIT_OBJECT_RE.fullmatch(source_tree) is None
|
|
):
|
|
raise InitializationError("offline_quiescence_capture_invalid")
|
|
source_root_sha256s, source_root_set_sha256 = _source_root_bindings(
|
|
legacy_source_roots
|
|
)
|
|
payload_source_hashes = payload.get("source_root_sha256s")
|
|
if (
|
|
not isinstance(payload_source_hashes, list)
|
|
or tuple(payload_source_hashes) != source_root_sha256s
|
|
or payload.get("source_root_set_sha256") != source_root_set_sha256
|
|
):
|
|
raise InitializationError("offline_legacy_source_root_drift")
|
|
return {
|
|
"schema_version": OFFLINE_CAPTURE_SCHEMA_VERSION,
|
|
"status": "quiesced",
|
|
"captured_at_utc": captured_at,
|
|
"source_commit": source_commit,
|
|
"source_tree": source_tree,
|
|
"source_root_sha256s": list(source_root_sha256s),
|
|
"source_root_set_sha256": source_root_set_sha256,
|
|
"api_identity": _strict_process_identity(payload.get("api_identity")),
|
|
"tunnel_identity": _strict_process_identity(
|
|
payload.get("tunnel_identity")
|
|
),
|
|
"listener_absent": True,
|
|
"tunnel_absent": True,
|
|
"listener_endpoint_sha256": _strict_lower_sha256(
|
|
payload.get("listener_endpoint_sha256"),
|
|
"offline_quiescence_capture_invalid",
|
|
),
|
|
"tunnel_config_sha256": _strict_lower_sha256(
|
|
payload.get("tunnel_config_sha256"),
|
|
"offline_quiescence_capture_invalid",
|
|
),
|
|
}
|
|
|
|
|
|
def build_offline_quiescence_receipt(
|
|
*,
|
|
capture: Mapping[str, object],
|
|
inventory: ReferenceInventory,
|
|
preserved: PreservedInventory,
|
|
database_target_digest: str,
|
|
) -> dict[str, object]:
|
|
database_digest = _strict_lower_sha256(
|
|
database_target_digest, "database_target_invalid"
|
|
)
|
|
invalid_object_count, invalid_reference_count = required_decode_invalid_counts(
|
|
references=inventory,
|
|
preserved=preserved,
|
|
)
|
|
return {
|
|
"schema_version": OFFLINE_RECEIPT_SCHEMA_VERSION,
|
|
"status": "quiesced",
|
|
"captured_at_utc": capture["captured_at_utc"],
|
|
"source_commit": capture["source_commit"],
|
|
"source_tree": capture["source_tree"],
|
|
"source_root_sha256s": capture["source_root_sha256s"],
|
|
"source_root_set_sha256": capture["source_root_set_sha256"],
|
|
"api_identity": capture["api_identity"],
|
|
"tunnel_identity": capture["tunnel_identity"],
|
|
"listener_absent": True,
|
|
"tunnel_absent": True,
|
|
"listener_endpoint_sha256": capture["listener_endpoint_sha256"],
|
|
"tunnel_config_sha256": capture["tunnel_config_sha256"],
|
|
"database_target_sha256": database_digest,
|
|
"inventory": {
|
|
"preserved_object_count": preserved.object_count,
|
|
"preserved_total_size_bytes": preserved.total_size_bytes,
|
|
"preserved_inventory_sha256": preserved.inventory_sha256,
|
|
"preserved_decode_valid_count": preserved.decode_valid_count,
|
|
"preserved_decode_invalid_count": preserved.decode_invalid_count,
|
|
"required_decode_invalid_object_count": invalid_object_count,
|
|
"required_decode_invalid_reference_count": invalid_reference_count,
|
|
"reference_count": inventory.reference_count,
|
|
"unique_reference_object_count": inventory.unique_object_count,
|
|
"reference_set_sha256": inventory.reference_set_sha256,
|
|
},
|
|
"privacy": {
|
|
"raw_paths_recorded": False,
|
|
"raw_urls_recorded": False,
|
|
"raw_user_ids_recorded": False,
|
|
"raw_filenames_recorded": False,
|
|
"raw_secrets_recorded": False,
|
|
},
|
|
}
|
|
|
|
|
|
def write_offline_quiescence_receipt_create_only(
|
|
*, manifest_state_dir: Path, payload: Mapping[str, object]
|
|
) -> tuple[Path, str]:
|
|
state_dir = _validated_absolute_directory(manifest_state_dir)
|
|
encoded = _json_bytes(dict(payload))
|
|
receipt_sha256 = _sha256_bytes(encoded)
|
|
destination = state_dir / f"public-upload-quiescence-{receipt_sha256}.json"
|
|
if destination.exists() or destination.is_symlink():
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != receipt_sha256 or regular.read_bytes() != encoded:
|
|
raise InitializationError("offline_quiescence_receipt_conflict")
|
|
return regular, receipt_sha256
|
|
try:
|
|
with destination.open("xb") as stream:
|
|
stream.write(encoded)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
except FileExistsError:
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != receipt_sha256 or regular.read_bytes() != encoded:
|
|
raise InitializationError("offline_quiescence_receipt_conflict")
|
|
return regular, receipt_sha256
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != receipt_sha256:
|
|
raise InitializationError("offline_quiescence_receipt_verification_failed")
|
|
return regular, receipt_sha256
|
|
|
|
|
|
def validate_offline_quiescence_receipt(
|
|
*,
|
|
receipt_path: Path,
|
|
expected_receipt_sha256: str,
|
|
stable_source_root: Path,
|
|
upload_root: Path,
|
|
expected_source_commit: str,
|
|
expected_source_tree: str,
|
|
expected_source_roots: Sequence[Path] | None,
|
|
expected_database_target_sha256: str,
|
|
expected_preserved_object_count: int,
|
|
expected_preserved_total_size_bytes: int,
|
|
expected_preserved_inventory_sha256: str,
|
|
expected_preserved_decode_valid_count: int,
|
|
expected_preserved_decode_invalid_count: int,
|
|
expected_required_decode_invalid_object_count: int,
|
|
expected_required_decode_invalid_reference_count: int,
|
|
expected_reference_count: int,
|
|
expected_unique_object_count: int,
|
|
expected_reference_set_sha256: str,
|
|
expected_source_root_sha256s: Sequence[str] | None = None,
|
|
expected_source_root_set_sha256: str = "",
|
|
) -> OfflineQuiescenceProof:
|
|
if _SHA256_RE.fullmatch(expected_receipt_sha256 or "") is None:
|
|
raise InitializationError("offline_quiescence_receipt_hash_invalid")
|
|
if _GIT_OBJECT_RE.fullmatch(expected_source_commit or "") is None or _GIT_OBJECT_RE.fullmatch(
|
|
expected_source_tree or ""
|
|
) is None:
|
|
raise InitializationError("offline_quiescence_source_pin_invalid")
|
|
stable_root = _validated_absolute_directory(stable_source_root)
|
|
public_root = _validated_absolute_directory(upload_root)
|
|
if expected_source_roots:
|
|
source_root_sha256s, source_root_set_sha256 = _source_root_bindings(
|
|
expected_source_roots
|
|
)
|
|
else:
|
|
if not expected_source_root_sha256s:
|
|
raise InitializationError("offline_quiescence_source_roots_missing")
|
|
source_root_sha256s = tuple(
|
|
_strict_lower_sha256(value, "offline_quiescence_source_roots_invalid")
|
|
for value in expected_source_root_sha256s
|
|
)
|
|
if len(set(source_root_sha256s)) != len(source_root_sha256s):
|
|
raise InitializationError("offline_quiescence_source_roots_invalid")
|
|
source_root_set_sha256 = _sha256_bytes(
|
|
_json_bytes(sorted(source_root_sha256s)).rstrip(b"\n")
|
|
)
|
|
if source_root_set_sha256 != expected_source_root_set_sha256:
|
|
raise InitializationError("offline_quiescence_source_roots_invalid")
|
|
regular = _validated_regular_file(receipt_path)
|
|
_assert_disjoint(regular, stable_root)
|
|
_assert_disjoint(regular, public_root)
|
|
for source_root in expected_source_roots or ():
|
|
_assert_disjoint(regular, _validated_absolute_directory(source_root))
|
|
if sha256_file(regular) != expected_receipt_sha256:
|
|
raise InitializationError("offline_quiescence_receipt_hash_drift")
|
|
try:
|
|
payload = json.loads(regular.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise InitializationError("offline_quiescence_receipt_invalid") from exc
|
|
if not isinstance(payload, dict) or set(payload) != {
|
|
"schema_version",
|
|
"status",
|
|
"captured_at_utc",
|
|
"source_commit",
|
|
"source_tree",
|
|
"source_root_sha256s",
|
|
"source_root_set_sha256",
|
|
"api_identity",
|
|
"tunnel_identity",
|
|
"listener_absent",
|
|
"tunnel_absent",
|
|
"listener_endpoint_sha256",
|
|
"tunnel_config_sha256",
|
|
"database_target_sha256",
|
|
"inventory",
|
|
"privacy",
|
|
}:
|
|
raise InitializationError("offline_quiescence_receipt_invalid")
|
|
if (
|
|
payload.get("schema_version") != OFFLINE_RECEIPT_SCHEMA_VERSION
|
|
or payload.get("status") != "quiesced"
|
|
or payload.get("source_commit") != expected_source_commit
|
|
or payload.get("source_tree") != expected_source_tree
|
|
or payload.get("listener_absent") is not True
|
|
or payload.get("tunnel_absent") is not True
|
|
or payload.get("database_target_sha256")
|
|
!= expected_database_target_sha256
|
|
or payload.get("source_root_sha256s") != list(source_root_sha256s)
|
|
or payload.get("source_root_set_sha256") != source_root_set_sha256
|
|
):
|
|
raise InitializationError("offline_quiescence_receipt_contract_drift")
|
|
_strict_lower_sha256(
|
|
payload.get("source_root_set_sha256"),
|
|
"offline_quiescence_receipt_invalid",
|
|
)
|
|
payload_source_hashes = payload.get("source_root_sha256s")
|
|
if not isinstance(payload_source_hashes, list) or any(
|
|
_SHA256_RE.fullmatch(value or "") is None
|
|
for value in payload_source_hashes
|
|
if isinstance(value, str)
|
|
) or any(not isinstance(value, str) for value in payload_source_hashes):
|
|
raise InitializationError("offline_quiescence_receipt_invalid")
|
|
_strict_lower_sha256(
|
|
payload.get("listener_endpoint_sha256"),
|
|
"offline_quiescence_receipt_invalid",
|
|
)
|
|
_strict_lower_sha256(
|
|
payload.get("tunnel_config_sha256"),
|
|
"offline_quiescence_receipt_invalid",
|
|
)
|
|
_strict_process_identity(payload.get("api_identity"))
|
|
_strict_process_identity(payload.get("tunnel_identity"))
|
|
inventory = payload.get("inventory")
|
|
if not isinstance(inventory, dict) or set(inventory) != {
|
|
"preserved_object_count",
|
|
"preserved_total_size_bytes",
|
|
"preserved_inventory_sha256",
|
|
"preserved_decode_valid_count",
|
|
"preserved_decode_invalid_count",
|
|
"required_decode_invalid_object_count",
|
|
"required_decode_invalid_reference_count",
|
|
"reference_count",
|
|
"unique_reference_object_count",
|
|
"reference_set_sha256",
|
|
}:
|
|
raise InitializationError("offline_quiescence_receipt_inventory_invalid")
|
|
if (
|
|
inventory.get("preserved_object_count")
|
|
!= expected_preserved_object_count
|
|
or inventory.get("preserved_total_size_bytes")
|
|
!= expected_preserved_total_size_bytes
|
|
or inventory.get("preserved_inventory_sha256")
|
|
!= expected_preserved_inventory_sha256
|
|
or inventory.get("preserved_decode_valid_count")
|
|
!= expected_preserved_decode_valid_count
|
|
or inventory.get("preserved_decode_invalid_count")
|
|
!= expected_preserved_decode_invalid_count
|
|
or inventory.get("required_decode_invalid_object_count")
|
|
!= expected_required_decode_invalid_object_count
|
|
or inventory.get("required_decode_invalid_reference_count")
|
|
!= expected_required_decode_invalid_reference_count
|
|
or inventory.get("reference_count") != expected_reference_count
|
|
or inventory.get("unique_reference_object_count")
|
|
!= expected_unique_object_count
|
|
or inventory.get("reference_set_sha256")
|
|
!= expected_reference_set_sha256
|
|
):
|
|
raise InitializationError("offline_quiescence_receipt_inventory_drift")
|
|
privacy = payload.get("privacy")
|
|
if not isinstance(privacy, dict) or set(privacy) != {
|
|
"raw_paths_recorded",
|
|
"raw_urls_recorded",
|
|
"raw_user_ids_recorded",
|
|
"raw_filenames_recorded",
|
|
"raw_secrets_recorded",
|
|
} or any(value is not False for value in privacy.values()):
|
|
raise InitializationError("offline_quiescence_receipt_privacy_invalid")
|
|
return OfflineQuiescenceProof(
|
|
receipt_path=regular,
|
|
receipt_sha256=expected_receipt_sha256,
|
|
database_target_sha256=expected_database_target_sha256,
|
|
source_root_sha256s=source_root_sha256s,
|
|
source_root_set_sha256=source_root_set_sha256,
|
|
preserved_object_count=expected_preserved_object_count,
|
|
preserved_total_size_bytes=expected_preserved_total_size_bytes,
|
|
preserved_inventory_sha256=expected_preserved_inventory_sha256,
|
|
preserved_decode_valid_count=expected_preserved_decode_valid_count,
|
|
preserved_decode_invalid_count=expected_preserved_decode_invalid_count,
|
|
required_decode_invalid_object_count=(
|
|
expected_required_decode_invalid_object_count
|
|
),
|
|
required_decode_invalid_reference_count=(
|
|
expected_required_decode_invalid_reference_count
|
|
),
|
|
reference_count=expected_reference_count,
|
|
unique_object_count=expected_unique_object_count,
|
|
reference_set_sha256=expected_reference_set_sha256,
|
|
)
|
|
|
|
|
|
def _hash_regular_file(path: Path) -> tuple[int, str]:
|
|
regular = _validated_regular_file(path)
|
|
digest = hashlib.sha256()
|
|
size_bytes = 0
|
|
try:
|
|
with regular.open("rb") as stream:
|
|
before = os.fstat(stream.fileno())
|
|
before_identity = _regular_file_identity(before)
|
|
if before_identity is None:
|
|
raise InitializationError("source_object_changed_during_hash")
|
|
while True:
|
|
chunk = stream.read(_COPY_CHUNK_SIZE)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
size_bytes += len(chunk)
|
|
after = os.fstat(stream.fileno())
|
|
path_after = regular.lstat()
|
|
except InitializationError:
|
|
raise
|
|
except OSError as exc:
|
|
raise InitializationError("source_object_changed_during_hash") from exc
|
|
if (
|
|
_has_reparse_attribute(regular)
|
|
or before_identity != _regular_file_identity(after)
|
|
or before_identity != _regular_file_identity(path_after)
|
|
or before.st_size != after.st_size
|
|
or before.st_mtime_ns != after.st_mtime_ns
|
|
or size_bytes != after.st_size
|
|
):
|
|
raise InitializationError("source_object_changed_during_hash")
|
|
return size_bytes, digest.hexdigest()
|
|
|
|
|
|
def _decode_regular_avatar(
|
|
path: Path,
|
|
*,
|
|
relative_path: str,
|
|
expected_size: int,
|
|
expected_sha256: str,
|
|
) -> bool:
|
|
regular = _validated_regular_file(path)
|
|
try:
|
|
content = regular.read_bytes()
|
|
except OSError as exc:
|
|
raise InitializationError("source_object_changed_during_decode") from exc
|
|
if (
|
|
len(content) != expected_size
|
|
or hashlib.sha256(content).hexdigest() != expected_sha256
|
|
):
|
|
raise InitializationError("source_object_changed_during_decode")
|
|
return inspect_public_avatar_image(relative_path, content).valid
|
|
|
|
|
|
def _preserved_inventory_records(
|
|
objects: Iterable[PreservedObject],
|
|
) -> list[dict[str, object]]:
|
|
return sorted(
|
|
(
|
|
{
|
|
"path_sha256": public_avatar_relative_path_sha256(
|
|
item.relative_path
|
|
),
|
|
"content_sha256": item.content_sha256,
|
|
"size_bytes": item.size_bytes,
|
|
"kind": "profile_avatar",
|
|
}
|
|
for item in objects
|
|
),
|
|
key=lambda item: str(item["path_sha256"]),
|
|
)
|
|
|
|
|
|
def scan_preserved_inventory(source_roots: Sequence[Path]) -> PreservedInventory:
|
|
validated_sources: list[Path] = []
|
|
source_keys: set[str] = set()
|
|
for source_root in source_roots:
|
|
validated = _validated_absolute_directory(source_root)
|
|
source_key = os.path.normcase(str(validated))
|
|
if source_key in source_keys:
|
|
raise InitializationError("source_root_duplicate")
|
|
source_keys.add(source_key)
|
|
validated_sources.append(validated)
|
|
if not validated_sources:
|
|
raise InitializationError("source_root_required")
|
|
|
|
candidates: dict[str, PreservedObject] = {}
|
|
for source_root in validated_sources:
|
|
avatar_root = source_root / "profile-avatars"
|
|
if not avatar_root.exists() and not avatar_root.is_symlink():
|
|
continue
|
|
avatar_root = _validated_absolute_directory(avatar_root)
|
|
try:
|
|
entries = tuple(avatar_root.iterdir())
|
|
except OSError as exc:
|
|
raise InitializationError("source_inventory_scan_failed") from exc
|
|
for candidate in entries:
|
|
basename = candidate.name
|
|
if (
|
|
"%" in basename
|
|
or any(
|
|
ord(character) < 32 or ord(character) == 127
|
|
for character in basename
|
|
)
|
|
or _ALLOWED_AVATAR_NAME.fullmatch(basename) is None
|
|
):
|
|
raise InitializationError("source_inventory_entry_invalid")
|
|
try:
|
|
metadata = candidate.lstat()
|
|
except OSError as exc:
|
|
raise InitializationError("source_inventory_changed_during_scan") from exc
|
|
if _has_reparse_attribute(candidate) or not stat.S_ISREG(metadata.st_mode):
|
|
raise InitializationError("source_inventory_entry_invalid")
|
|
relative_path = f"profile-avatars/{basename}"
|
|
size_bytes, content_sha256 = _hash_regular_file(candidate)
|
|
decode_valid = _decode_regular_avatar(
|
|
candidate,
|
|
relative_path=relative_path,
|
|
expected_size=size_bytes,
|
|
expected_sha256=content_sha256,
|
|
)
|
|
identity = relative_path.casefold()
|
|
existing = candidates.get(identity)
|
|
if existing is not None:
|
|
if (
|
|
existing.size_bytes != size_bytes
|
|
or existing.content_sha256 != content_sha256
|
|
):
|
|
raise InitializationError("source_object_conflict")
|
|
continue
|
|
candidates[identity] = PreservedObject(
|
|
relative_path=relative_path,
|
|
source_path=candidate,
|
|
size_bytes=size_bytes,
|
|
content_sha256=content_sha256,
|
|
decode_valid=decode_valid,
|
|
)
|
|
|
|
objects = tuple(
|
|
sorted(
|
|
candidates.values(),
|
|
key=lambda item: public_avatar_relative_path_sha256(
|
|
item.relative_path
|
|
),
|
|
)
|
|
)
|
|
records = _preserved_inventory_records(objects)
|
|
inventory_sha256 = _sha256_bytes(_json_bytes(records).rstrip(b"\n"))
|
|
return PreservedInventory(objects=objects, inventory_sha256=inventory_sha256)
|
|
|
|
|
|
def assert_database_references_preserved(
|
|
*, references: ReferenceInventory, preserved: PreservedInventory
|
|
) -> None:
|
|
available = {item.relative_path.casefold() for item in preserved.objects}
|
|
if any(name.casefold() not in available for name, _count in references.objects):
|
|
raise InitializationError("source_object_missing")
|
|
|
|
|
|
def required_decode_invalid_counts(
|
|
*,
|
|
references: ReferenceInventory,
|
|
preserved: PreservedInventory,
|
|
) -> tuple[int, int]:
|
|
decode_by_path = {
|
|
item.relative_path.casefold(): item.decode_valid
|
|
for item in preserved.objects
|
|
}
|
|
invalid_objects = 0
|
|
invalid_references = 0
|
|
for relative_path, reference_count in references.objects:
|
|
if decode_by_path.get(relative_path.casefold()) is False:
|
|
invalid_objects += 1
|
|
invalid_references += reference_count
|
|
return invalid_objects, invalid_references
|
|
|
|
|
|
def assert_expected_preserved_inventory(
|
|
*,
|
|
inventory: PreservedInventory,
|
|
expected_object_count: int,
|
|
expected_total_size_bytes: int,
|
|
expected_inventory_sha256: str,
|
|
) -> None:
|
|
if expected_object_count <= 0:
|
|
raise InitializationError("expected_preserved_object_count_invalid")
|
|
expected_digest = _strict_lower_sha256(
|
|
expected_inventory_sha256,
|
|
"expected_preserved_inventory_sha256_invalid",
|
|
)
|
|
if expected_total_size_bytes <= 0:
|
|
raise InitializationError("expected_preserved_total_size_bytes_invalid")
|
|
if (
|
|
inventory.object_count != expected_object_count
|
|
or inventory.total_size_bytes != expected_total_size_bytes
|
|
or inventory.inventory_sha256 != expected_digest
|
|
):
|
|
raise InitializationError("preserved_inventory_pin_mismatch")
|
|
|
|
|
|
def assert_preserved_inventory_stable(
|
|
*, expected: PreservedInventory, actual: PreservedInventory
|
|
) -> None:
|
|
if (
|
|
expected.object_count != actual.object_count
|
|
or expected.total_size_bytes != actual.total_size_bytes
|
|
or expected.decode_valid_count != actual.decode_valid_count
|
|
or expected.decode_invalid_count != actual.decode_invalid_count
|
|
or expected.inventory_sha256 != actual.inventory_sha256
|
|
or _preserved_inventory_records(expected.objects)
|
|
!= _preserved_inventory_records(actual.objects)
|
|
or tuple(
|
|
(item.relative_path.casefold(), item.decode_valid)
|
|
for item in expected.objects
|
|
)
|
|
!= tuple(
|
|
(item.relative_path.casefold(), item.decode_valid)
|
|
for item in actual.objects
|
|
)
|
|
):
|
|
raise InitializationError("source_inventory_changed_during_copy")
|
|
|
|
|
|
def _source_proof(
|
|
relative_name: str, source_roots: Sequence[Path]
|
|
) -> tuple[Path, int, str]:
|
|
candidates: list[tuple[Path, int, str]] = []
|
|
relative_parts = relative_name.split("/")
|
|
for source_root in source_roots:
|
|
candidate = source_root.joinpath(*relative_parts)
|
|
if candidate.exists() or candidate.is_symlink():
|
|
size_bytes, digest = _hash_regular_file(candidate)
|
|
candidates.append((candidate, size_bytes, digest))
|
|
if not candidates:
|
|
raise InitializationError("source_object_missing")
|
|
identities = {(size_bytes, digest) for _, size_bytes, digest in candidates}
|
|
if len(identities) != 1:
|
|
raise InitializationError("source_object_conflict")
|
|
return candidates[0]
|
|
|
|
|
|
def _target_matches(path: Path, *, expected_size: int, expected_sha256: str) -> bool:
|
|
if not path.exists() and not path.is_symlink():
|
|
return False
|
|
size_bytes, digest = _hash_regular_file(path)
|
|
return size_bytes == expected_size and digest == expected_sha256
|
|
|
|
|
|
def _regular_file_identity(metadata: os.stat_result) -> tuple[int, int] | None:
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
return None
|
|
device = int(getattr(metadata, "st_dev", 0))
|
|
inode = int(getattr(metadata, "st_ino", 0))
|
|
if inode <= 0:
|
|
return None
|
|
return device, inode
|
|
|
|
|
|
def _unlink_created_file_if_identity_matches(
|
|
path: Path, expected_identity: tuple[int, int] | None
|
|
) -> None:
|
|
# Once the create-only handle is closed, Windows offers no atomic
|
|
# "unlink-this-exact-file-id" primitive through pathlib. A lstat/identity
|
|
# check followed by unlink is still racy and could delete another writer's
|
|
# replacement. Leave any failed initializer artifact in the unpublished
|
|
# target so the freeze remains fail-closed and an operator can inspect it.
|
|
del path, expected_identity
|
|
|
|
|
|
def _copy_one_create_only(
|
|
*, source: Path, target: Path, expected_size: int, expected_sha256: str
|
|
) -> bool:
|
|
if target.exists() or target.is_symlink():
|
|
if not _target_matches(
|
|
target,
|
|
expected_size=expected_size,
|
|
expected_sha256=expected_sha256,
|
|
):
|
|
raise InitializationError("target_object_conflict")
|
|
return False
|
|
|
|
created_identity: tuple[int, int] | None = None
|
|
try:
|
|
source_file = _validated_regular_file(source)
|
|
_assert_no_reparse_ancestors(target.parent)
|
|
with source_file.open("rb") as input_stream, target.open("xb") as output_stream:
|
|
source_before = os.fstat(input_stream.fileno())
|
|
source_identity = _regular_file_identity(source_before)
|
|
source_path_before = source_file.lstat()
|
|
if (
|
|
source_identity is None
|
|
or source_identity != _regular_file_identity(source_path_before)
|
|
or _has_reparse_attribute(source_file)
|
|
):
|
|
raise InitializationError("source_object_changed_during_copy")
|
|
created_identity = _regular_file_identity(os.fstat(output_stream.fileno()))
|
|
copied_digest = hashlib.sha256()
|
|
copied_size = 0
|
|
while True:
|
|
chunk = input_stream.read(_COPY_CHUNK_SIZE)
|
|
if not chunk:
|
|
break
|
|
output_stream.write(chunk)
|
|
copied_digest.update(chunk)
|
|
copied_size += len(chunk)
|
|
output_stream.flush()
|
|
os.fsync(output_stream.fileno())
|
|
source_after = os.fstat(input_stream.fileno())
|
|
source_path_after = source_file.lstat()
|
|
if (
|
|
source_identity != _regular_file_identity(source_after)
|
|
or source_identity != _regular_file_identity(source_path_after)
|
|
or source_before.st_size != source_after.st_size
|
|
or source_before.st_mtime_ns != source_after.st_mtime_ns
|
|
or _has_reparse_attribute(source_file)
|
|
):
|
|
raise InitializationError("source_object_changed_during_copy")
|
|
if copied_size != expected_size or copied_digest.hexdigest() != expected_sha256:
|
|
raise InitializationError("source_object_changed_during_copy")
|
|
if not _target_matches(
|
|
target,
|
|
expected_size=expected_size,
|
|
expected_sha256=expected_sha256,
|
|
):
|
|
raise InitializationError("target_object_verification_failed")
|
|
return True
|
|
except FileExistsError:
|
|
if _target_matches(
|
|
target,
|
|
expected_size=expected_size,
|
|
expected_sha256=expected_sha256,
|
|
):
|
|
return False
|
|
raise InitializationError("target_object_conflict")
|
|
except BaseException:
|
|
_unlink_created_file_if_identity_matches(target, created_identity)
|
|
raise
|
|
|
|
|
|
def _assert_target_inventory_empty_or_exact(
|
|
*,
|
|
avatar_root: Path,
|
|
inventory: PreservedInventory,
|
|
allow_empty: bool,
|
|
) -> None:
|
|
expected = {
|
|
item.relative_path.split("/", 1)[1].casefold(): item
|
|
for item in inventory.objects
|
|
}
|
|
try:
|
|
entries = tuple(avatar_root.iterdir())
|
|
except OSError as exc:
|
|
raise InitializationError("target_inventory_scan_failed") from exc
|
|
if not entries and allow_empty:
|
|
return
|
|
if len(entries) != len(expected):
|
|
raise InitializationError("target_inventory_not_empty_or_exact")
|
|
seen: set[str] = set()
|
|
for candidate in entries:
|
|
key = candidate.name.casefold()
|
|
preserved = expected.get(key)
|
|
if key in seen or preserved is None:
|
|
raise InitializationError("target_inventory_not_empty_or_exact")
|
|
seen.add(key)
|
|
if not _target_matches(
|
|
candidate,
|
|
expected_size=preserved.size_bytes,
|
|
expected_sha256=preserved.content_sha256,
|
|
):
|
|
raise InitializationError("target_inventory_not_empty_or_exact")
|
|
if seen != set(expected):
|
|
raise InitializationError("target_inventory_not_empty_or_exact")
|
|
|
|
|
|
def copy_required_objects(
|
|
*,
|
|
inventory: PreservedInventory,
|
|
source_roots: Sequence[Path],
|
|
upload_root: Path,
|
|
) -> CopyProof:
|
|
validated_upload_root = _validated_absolute_directory(upload_root)
|
|
validated_sources = tuple(
|
|
_validated_absolute_directory(source_root) for source_root in source_roots
|
|
)
|
|
if not validated_sources:
|
|
raise InitializationError("source_root_required")
|
|
for source_root in validated_sources:
|
|
_assert_disjoint(source_root, validated_upload_root)
|
|
|
|
avatar_root = validated_upload_root / "profile-avatars"
|
|
if avatar_root.exists() or avatar_root.is_symlink():
|
|
avatar_root = _validated_absolute_directory(avatar_root)
|
|
else:
|
|
_assert_no_reparse_ancestors(avatar_root.parent)
|
|
avatar_root.mkdir()
|
|
avatar_root = _validated_absolute_directory(avatar_root)
|
|
|
|
_assert_target_inventory_empty_or_exact(
|
|
avatar_root=avatar_root,
|
|
inventory=inventory,
|
|
allow_empty=True,
|
|
)
|
|
copied = 0
|
|
reused = 0
|
|
for preserved_object in inventory.objects:
|
|
source = preserved_object.source_path
|
|
size_bytes, digest = _hash_regular_file(source)
|
|
if (
|
|
size_bytes != preserved_object.size_bytes
|
|
or digest != preserved_object.content_sha256
|
|
):
|
|
raise InitializationError("source_inventory_changed_during_copy")
|
|
target = avatar_root / preserved_object.relative_path.split("/", 1)[1]
|
|
if _copy_one_create_only(
|
|
source=source,
|
|
target=target,
|
|
expected_size=size_bytes,
|
|
expected_sha256=digest,
|
|
):
|
|
copied += 1
|
|
else:
|
|
reused += 1
|
|
_assert_target_inventory_empty_or_exact(
|
|
avatar_root=avatar_root,
|
|
inventory=inventory,
|
|
allow_empty=False,
|
|
)
|
|
return CopyProof(inventory.object_count, copied, reused)
|
|
|
|
|
|
def _read_freeze_token(freeze_path: Path) -> tuple[str, str]:
|
|
regular = _validated_regular_file(freeze_path)
|
|
try:
|
|
payload = json.loads(regular.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise InitializationError("write_freeze_sentinel_invalid") from exc
|
|
token = payload.get("token") if isinstance(payload, dict) else None
|
|
if (
|
|
not isinstance(payload, dict)
|
|
or payload.get("schema_version") != WRITE_FREEZE_SCHEMA_VERSION
|
|
or not isinstance(token, str)
|
|
or len(token) < 32
|
|
):
|
|
raise InitializationError("write_freeze_sentinel_invalid")
|
|
return token, _sha256_bytes(token.encode("utf-8"))
|
|
|
|
|
|
def wait_for_drained_runtime_freeze(
|
|
*, health_url: str, token_sha256: str, timeout_seconds: float
|
|
) -> None:
|
|
parsed_health_url = urllib.parse.urlsplit(health_url)
|
|
try:
|
|
health_port = parsed_health_url.port
|
|
except ValueError as exc:
|
|
raise InitializationError("runtime_health_url_invalid") from exc
|
|
if (
|
|
parsed_health_url.scheme != "http"
|
|
or parsed_health_url.hostname != "127.0.0.1"
|
|
or parsed_health_url.username is not None
|
|
or parsed_health_url.password is not None
|
|
or health_port is None
|
|
or parsed_health_url.path != "/health"
|
|
or parsed_health_url.query
|
|
or parsed_health_url.fragment
|
|
):
|
|
raise InitializationError("runtime_health_url_invalid")
|
|
deadline = time.monotonic() + timeout_seconds
|
|
opener = urllib.request.build_opener(_RejectRedirects())
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
request = urllib.request.Request(
|
|
health_url, headers={"Accept": "application/json"}, method="GET"
|
|
)
|
|
with opener.open(request, timeout=2.0) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
freeze = (
|
|
payload.get("upload_write_freeze")
|
|
if isinstance(payload, dict)
|
|
else None
|
|
)
|
|
if (
|
|
isinstance(freeze, dict)
|
|
and freeze.get("capable") is True
|
|
and freeze.get("active") is True
|
|
and freeze.get("valid") is True
|
|
and freeze.get("in_flight") == 0
|
|
and freeze.get("token_sha256") == token_sha256
|
|
):
|
|
return
|
|
except (
|
|
OSError,
|
|
UnicodeError,
|
|
json.JSONDecodeError,
|
|
urllib.error.URLError,
|
|
):
|
|
pass
|
|
time.sleep(0.25)
|
|
raise InitializationError("runtime_write_freeze_not_drained")
|
|
|
|
|
|
async def fetch_database_inventory() -> tuple[list[object], int, str]:
|
|
try:
|
|
import asyncpg
|
|
from app.config import settings
|
|
except ImportError as exc:
|
|
raise InitializationError("database_client_unavailable") from exc
|
|
|
|
connection = None
|
|
try:
|
|
connection = await asyncpg.connect(
|
|
settings.database_url, command_timeout=settings.db_command_timeout
|
|
)
|
|
async with connection.transaction(isolation="repeatable_read", readonly=True):
|
|
database_target_digest = await connected_database_target_sha256(connection)
|
|
await connection.execute(
|
|
"SELECT set_config('app.ai_context', '', true), "
|
|
"set_config('app.current_role', 'admin', true)"
|
|
)
|
|
rows = await connection.fetch(
|
|
"""
|
|
SELECT avatar_url
|
|
FROM app.app_user
|
|
WHERE avatar_url IS NOT NULL
|
|
AND left(avatar_url, 9) = '/uploads/'
|
|
ORDER BY avatar_url
|
|
"""
|
|
)
|
|
active_private_audio_count = int(
|
|
await connection.fetchval(
|
|
"""
|
|
SELECT count(*)
|
|
FROM app.multimodal_audio_asset AS audio
|
|
WHERE audio.retained_until > now()
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM audit.multimodal_deletion_tombstone AS tombstone
|
|
WHERE tombstone.session_id = audio.session_id
|
|
AND tombstone.scope = 'audio'
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
return (
|
|
[row["avatar_url"] for row in rows],
|
|
active_private_audio_count,
|
|
database_target_digest,
|
|
)
|
|
except InitializationError:
|
|
raise
|
|
except Exception as exc:
|
|
raise InitializationError("database_inventory_query_failed") from exc
|
|
finally:
|
|
if connection is not None:
|
|
await connection.close()
|
|
|
|
|
|
def assert_no_active_private_audio(active_private_audio_count: int) -> None:
|
|
if active_private_audio_count != 0:
|
|
raise InitializationError("active_private_audio_requires_separate_migration")
|
|
|
|
|
|
def privacy_safe_manifest(
|
|
*,
|
|
upload_root: Path,
|
|
inventory: ReferenceInventory,
|
|
preserved: PreservedInventory,
|
|
database_target_sha256: str,
|
|
freeze_token_sha256: str,
|
|
freeze_path: Path,
|
|
offline_quiescence: OfflineQuiescenceProof | None = None,
|
|
) -> dict[str, object]:
|
|
if _SHA256_RE.fullmatch(freeze_token_sha256) is None:
|
|
raise InitializationError("write_freeze_token_hash_invalid")
|
|
manifest = build_privacy_safe_manifest(
|
|
upload_root=upload_root,
|
|
references=[
|
|
(name, "profile_avatar")
|
|
for name, reference_count in inventory.objects
|
|
for _ in range(reference_count)
|
|
],
|
|
preserved_paths=[item.relative_path for item in preserved.objects],
|
|
database_target_sha256=database_target_sha256,
|
|
write_freeze_token_sha256=freeze_token_sha256,
|
|
write_freeze_path=freeze_path,
|
|
)
|
|
(
|
|
required_decode_invalid_object_count,
|
|
required_decode_invalid_reference_count,
|
|
) = required_decode_invalid_counts(
|
|
references=inventory,
|
|
preserved=preserved,
|
|
)
|
|
if (
|
|
manifest.get("database_target_sha256") != database_target_sha256
|
|
or manifest.get("preserved_object_count") != preserved.object_count
|
|
or manifest.get("preserved_total_size_bytes")
|
|
!= preserved.total_size_bytes
|
|
or manifest.get("preserved_decode_valid_count")
|
|
!= preserved.decode_valid_count
|
|
or manifest.get("preserved_decode_invalid_count")
|
|
!= preserved.decode_invalid_count
|
|
or manifest.get("required_decode_invalid_object_count")
|
|
!= required_decode_invalid_object_count
|
|
or manifest.get("required_decode_invalid_reference_count")
|
|
!= required_decode_invalid_reference_count
|
|
or manifest.get("preserved_object_set_sha256")
|
|
!= preserved.inventory_sha256
|
|
or manifest.get("required_object_count") != inventory.unique_object_count
|
|
or manifest.get("required_reference_count") != inventory.reference_count
|
|
or manifest.get("reference_set_sha256") != inventory.reference_set_sha256
|
|
):
|
|
raise InitializationError("manifest_inventory_contract_drift")
|
|
if offline_quiescence is not None:
|
|
manifest["offline_quiescence"] = {
|
|
"receipt_sha256": offline_quiescence.receipt_sha256,
|
|
"database_target_sha256": offline_quiescence.database_target_sha256,
|
|
"source_root_sha256s": list(
|
|
offline_quiescence.source_root_sha256s
|
|
),
|
|
"source_root_set_sha256": offline_quiescence.source_root_set_sha256,
|
|
"preserved_object_count": offline_quiescence.preserved_object_count,
|
|
"preserved_total_size_bytes": (
|
|
offline_quiescence.preserved_total_size_bytes
|
|
),
|
|
"preserved_inventory_sha256": (
|
|
offline_quiescence.preserved_inventory_sha256
|
|
),
|
|
"preserved_decode_valid_count": (
|
|
offline_quiescence.preserved_decode_valid_count
|
|
),
|
|
"preserved_decode_invalid_count": (
|
|
offline_quiescence.preserved_decode_invalid_count
|
|
),
|
|
"required_decode_invalid_object_count": (
|
|
offline_quiescence.required_decode_invalid_object_count
|
|
),
|
|
"required_decode_invalid_reference_count": (
|
|
offline_quiescence.required_decode_invalid_reference_count
|
|
),
|
|
"reference_count": offline_quiescence.reference_count,
|
|
"unique_object_count": offline_quiescence.unique_object_count,
|
|
"reference_set_sha256": offline_quiescence.reference_set_sha256,
|
|
}
|
|
return manifest
|
|
|
|
|
|
def write_manifest_create_only(
|
|
*, manifest_state_dir: Path, payload: Mapping[str, object]
|
|
) -> tuple[Path, str]:
|
|
manifest_state_dir = _validated_absolute_directory(manifest_state_dir)
|
|
encoded = _json_bytes(dict(payload))
|
|
manifest_sha256 = _sha256_bytes(encoded)
|
|
destination = manifest_state_dir / f"public-avatar-upload-{manifest_sha256}.json"
|
|
if destination.exists() or destination.is_symlink():
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != manifest_sha256 or regular.read_bytes() != encoded:
|
|
raise InitializationError("manifest_create_conflict")
|
|
return regular, manifest_sha256
|
|
try:
|
|
with destination.open("xb") as stream:
|
|
stream.write(encoded)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
except FileExistsError:
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != manifest_sha256 or regular.read_bytes() != encoded:
|
|
raise InitializationError("manifest_create_conflict")
|
|
return regular, manifest_sha256
|
|
regular = _validated_regular_file(destination)
|
|
if sha256_file(regular) != manifest_sha256:
|
|
raise InitializationError("manifest_verification_failed")
|
|
return regular, manifest_sha256
|
|
|
|
|
|
def privacy_safe_result(
|
|
*,
|
|
manifest_path: Path,
|
|
manifest_sha256: str,
|
|
upload_root: Path,
|
|
inventory: ReferenceInventory,
|
|
preserved: PreservedInventory,
|
|
copy_proof: CopyProof,
|
|
freeze_token_sha256: str,
|
|
offline_quiescence: OfflineQuiescenceProof | None = None,
|
|
) -> dict[str, object]:
|
|
(
|
|
required_decode_invalid_object_count,
|
|
required_decode_invalid_reference_count,
|
|
) = required_decode_invalid_counts(
|
|
references=inventory,
|
|
preserved=preserved,
|
|
)
|
|
result = {
|
|
"status": "initialized",
|
|
"manifest_sha256": manifest_sha256,
|
|
"manifest_path_sha256": canonical_path_sha256(manifest_path),
|
|
"root_path_sha256": canonical_path_sha256(upload_root),
|
|
"preserved_object_count": preserved.object_count,
|
|
"preserved_total_size_bytes": preserved.total_size_bytes,
|
|
"preserved_inventory_sha256": preserved.inventory_sha256,
|
|
"preserved_decode_valid_count": preserved.decode_valid_count,
|
|
"preserved_decode_invalid_count": preserved.decode_invalid_count,
|
|
"required_decode_invalid_object_count": (
|
|
required_decode_invalid_object_count
|
|
),
|
|
"required_decode_invalid_reference_count": (
|
|
required_decode_invalid_reference_count
|
|
),
|
|
"required_object_count": inventory.unique_object_count,
|
|
"database_reference_count": inventory.reference_count,
|
|
"database_reference_set_sha256": inventory.reference_set_sha256,
|
|
"copied_object_count": copy_proof.copied_count,
|
|
"reused_exact_object_count": copy_proof.reused_exact_count,
|
|
"write_freeze_token_sha256": freeze_token_sha256,
|
|
}
|
|
if offline_quiescence is not None:
|
|
result["offline_quiescence_receipt_sha256"] = (
|
|
offline_quiescence.receipt_sha256
|
|
)
|
|
result["offline_quiescence_receipt_path_sha256"] = canonical_path_sha256(
|
|
offline_quiescence.receipt_path
|
|
)
|
|
return result
|
|
|
|
|
|
async def initialize(args: argparse.Namespace) -> dict[str, object]:
|
|
upload_root = _validated_absolute_directory(Path(args.upload_root))
|
|
state_dir = _validated_absolute_directory(Path(args.manifest_state_dir))
|
|
freeze_path = _validated_regular_file(Path(args.write_freeze_path))
|
|
if freeze_path.parent != state_dir:
|
|
raise InitializationError("write_freeze_must_be_direct_private_state_child")
|
|
_assert_disjoint(upload_root, state_dir)
|
|
source_roots = tuple(
|
|
_validated_absolute_directory(Path(value)) for value in args.source_root
|
|
)
|
|
if not source_roots:
|
|
raise InitializationError("source_root_required")
|
|
for source_root in source_roots:
|
|
_assert_disjoint(source_root, upload_root)
|
|
_assert_disjoint(source_root, state_dir)
|
|
|
|
_, freeze_token_sha256 = _read_freeze_token(freeze_path)
|
|
offline_capture: dict[str, object] | None = None
|
|
offline_pin_pair_present = bool(
|
|
args.expected_offline_source_commit and args.expected_offline_source_tree
|
|
)
|
|
if bool(args.offline_quiescence_capture_base64) != offline_pin_pair_present:
|
|
raise InitializationError("offline_quiescence_source_pins_required_together")
|
|
if args.offline_quiescence_capture_base64:
|
|
offline_capture = decode_offline_quiescence_capture(
|
|
args.offline_quiescence_capture_base64,
|
|
legacy_source_roots=source_roots,
|
|
expected_source_commit=args.expected_offline_source_commit,
|
|
expected_source_tree=args.expected_offline_source_tree,
|
|
)
|
|
else:
|
|
wait_for_drained_runtime_freeze(
|
|
health_url=args.health_url,
|
|
token_sha256=freeze_token_sha256,
|
|
timeout_seconds=args.freeze_timeout_seconds,
|
|
)
|
|
(
|
|
avatar_urls,
|
|
active_private_audio_count,
|
|
database_digest,
|
|
) = await fetch_database_inventory()
|
|
assert_no_active_private_audio(active_private_audio_count)
|
|
inventory = build_reference_inventory(
|
|
avatar_urls, expected_reference_count=args.expected_reference_count
|
|
)
|
|
preserved = scan_preserved_inventory(source_roots)
|
|
assert_expected_preserved_inventory(
|
|
inventory=preserved,
|
|
expected_object_count=args.expected_preserved_object_count,
|
|
expected_total_size_bytes=args.expected_preserved_total_size_bytes,
|
|
expected_inventory_sha256=args.expected_preserved_inventory_sha256,
|
|
)
|
|
assert_database_references_preserved(
|
|
references=inventory,
|
|
preserved=preserved,
|
|
)
|
|
(
|
|
required_decode_invalid_object_count,
|
|
required_decode_invalid_reference_count,
|
|
) = required_decode_invalid_counts(
|
|
references=inventory,
|
|
preserved=preserved,
|
|
)
|
|
offline_quiescence: OfflineQuiescenceProof | None = None
|
|
if offline_capture is not None:
|
|
receipt_payload = build_offline_quiescence_receipt(
|
|
capture=offline_capture,
|
|
inventory=inventory,
|
|
preserved=preserved,
|
|
database_target_digest=database_digest,
|
|
)
|
|
receipt_path, receipt_sha256 = (
|
|
write_offline_quiescence_receipt_create_only(
|
|
manifest_state_dir=state_dir,
|
|
payload=receipt_payload,
|
|
)
|
|
)
|
|
offline_quiescence = validate_offline_quiescence_receipt(
|
|
receipt_path=receipt_path,
|
|
expected_receipt_sha256=receipt_sha256,
|
|
stable_source_root=REPO_ROOT,
|
|
upload_root=upload_root,
|
|
expected_source_commit=str(offline_capture["source_commit"]),
|
|
expected_source_tree=str(offline_capture["source_tree"]),
|
|
expected_source_roots=source_roots,
|
|
expected_database_target_sha256=database_digest,
|
|
expected_preserved_object_count=preserved.object_count,
|
|
expected_preserved_total_size_bytes=preserved.total_size_bytes,
|
|
expected_preserved_inventory_sha256=preserved.inventory_sha256,
|
|
expected_preserved_decode_valid_count=preserved.decode_valid_count,
|
|
expected_preserved_decode_invalid_count=preserved.decode_invalid_count,
|
|
expected_required_decode_invalid_object_count=(
|
|
required_decode_invalid_object_count
|
|
),
|
|
expected_required_decode_invalid_reference_count=(
|
|
required_decode_invalid_reference_count
|
|
),
|
|
expected_reference_count=inventory.reference_count,
|
|
expected_unique_object_count=inventory.unique_object_count,
|
|
expected_reference_set_sha256=inventory.reference_set_sha256,
|
|
)
|
|
copy_proof = copy_required_objects(
|
|
inventory=preserved,
|
|
source_roots=source_roots,
|
|
upload_root=upload_root,
|
|
)
|
|
preserved_after_copy = scan_preserved_inventory(source_roots)
|
|
assert_expected_preserved_inventory(
|
|
inventory=preserved_after_copy,
|
|
expected_object_count=args.expected_preserved_object_count,
|
|
expected_total_size_bytes=args.expected_preserved_total_size_bytes,
|
|
expected_inventory_sha256=args.expected_preserved_inventory_sha256,
|
|
)
|
|
assert_preserved_inventory_stable(
|
|
expected=preserved,
|
|
actual=preserved_after_copy,
|
|
)
|
|
payload = privacy_safe_manifest(
|
|
upload_root=upload_root,
|
|
inventory=inventory,
|
|
preserved=preserved,
|
|
database_target_sha256=database_digest,
|
|
freeze_token_sha256=freeze_token_sha256,
|
|
freeze_path=freeze_path,
|
|
offline_quiescence=offline_quiescence,
|
|
)
|
|
manifest_path, manifest_sha256 = write_manifest_create_only(
|
|
manifest_state_dir=state_dir,
|
|
payload=payload,
|
|
)
|
|
if manifest_path.parent != state_dir:
|
|
raise InitializationError("manifest_private_state_boundary_failed")
|
|
return privacy_safe_result(
|
|
manifest_path=manifest_path,
|
|
manifest_sha256=manifest_sha256,
|
|
upload_root=upload_root,
|
|
inventory=inventory,
|
|
preserved=preserved,
|
|
copy_proof=copy_proof,
|
|
freeze_token_sha256=freeze_token_sha256,
|
|
offline_quiescence=offline_quiescence,
|
|
)
|
|
|
|
|
|
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Initialize the public avatar upload root without overwrites."
|
|
)
|
|
parser.add_argument("--upload-root", required=True)
|
|
parser.add_argument("--manifest-state-dir", required=True)
|
|
parser.add_argument("--write-freeze-path", required=True)
|
|
parser.add_argument("--expected-reference-count", type=int, required=True)
|
|
parser.add_argument("--expected-preserved-object-count", type=int, required=True)
|
|
parser.add_argument(
|
|
"--expected-preserved-total-size-bytes", type=int, required=True
|
|
)
|
|
parser.add_argument("--expected-preserved-inventory-sha256", required=True)
|
|
parser.add_argument("--source-root", action="append", required=True)
|
|
parser.add_argument("--health-url", default="http://127.0.0.1:8001/health")
|
|
parser.add_argument("--freeze-timeout-seconds", type=float, default=30.0)
|
|
parser.add_argument("--offline-quiescence-capture-base64", default="")
|
|
parser.add_argument("--expected-offline-source-commit", default="")
|
|
parser.add_argument("--expected-offline-source-tree", default="")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def _verify_preserved_inventory_cli(argv: Sequence[str]) -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Verify a caller-pinned, privacy-safe preserved avatar inventory."
|
|
)
|
|
parser.add_argument("--source-root", action="append", required=True)
|
|
parser.add_argument("--expected-preserved-object-count", type=int, required=True)
|
|
parser.add_argument(
|
|
"--expected-preserved-total-size-bytes", type=int, required=True
|
|
)
|
|
parser.add_argument("--expected-preserved-inventory-sha256", required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
inventory = scan_preserved_inventory(
|
|
tuple(Path(value) for value in args.source_root)
|
|
)
|
|
assert_expected_preserved_inventory(
|
|
inventory=inventory,
|
|
expected_object_count=args.expected_preserved_object_count,
|
|
expected_total_size_bytes=args.expected_preserved_total_size_bytes,
|
|
expected_inventory_sha256=args.expected_preserved_inventory_sha256,
|
|
)
|
|
except InitializationError as exc:
|
|
print(
|
|
json.dumps(
|
|
{"status": "failed", "reason": str(exc)},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 1
|
|
except BaseException:
|
|
print(
|
|
json.dumps(
|
|
{"status": "failed", "reason": "preserved_inventory_probe_failed"},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 1
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "verified",
|
|
"preserved_object_count": inventory.object_count,
|
|
"preserved_total_size_bytes": inventory.total_size_bytes,
|
|
"preserved_inventory_sha256": inventory.inventory_sha256,
|
|
"preserved_decode_valid_count": inventory.decode_valid_count,
|
|
"preserved_decode_invalid_count": inventory.decode_invalid_count,
|
|
},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
effective_argv = list(sys.argv[1:] if argv is None else argv)
|
|
if effective_argv and effective_argv[0] == "probe-preserved-inventory":
|
|
return _verify_preserved_inventory_cli(effective_argv[1:])
|
|
try:
|
|
result = asyncio.run(initialize(parse_args(effective_argv)))
|
|
except InitializationError as exc:
|
|
print(
|
|
json.dumps(
|
|
{"status": "failed", "reason": str(exc)},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 1
|
|
except BaseException:
|
|
print(
|
|
json.dumps(
|
|
{"status": "failed", "reason": "unexpected_initializer_failure"},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 1
|
|
print(
|
|
json.dumps(
|
|
result,
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|