"""Public avatar storage manifest and upload-write freeze contracts. The public surface owns only ``profile-avatars/``. Private multimodal audio is deliberately excluded from both manifests and static serving. """ from __future__ import annotations import hashlib import io import json import os import re import stat import threading import warnings from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any, Iterable, Iterator, Sequence from PIL import Image, UnidentifiedImageError, __version__ as PILLOW_VERSION MANIFEST_SCHEMA_VERSION = "vignette.public-avatar-upload-manifest.v3" WRITE_FREEZE_SCHEMA_VERSION = "vignette.public-upload-write-freeze.v1" PUBLIC_AVATAR_DIRECTORY = "profile-avatars" PUBLIC_AVATAR_MAX_BYTES = 3 * 1024 * 1024 MAX_PRESERVED_AVATAR_CACHE_BYTES = 64 * 1024 * 1024 IMAGE_DECODE_CONTRACT_VERSION = "vignette.public-avatar-image-decode.v1" _SHA256_LENGTH = 64 _PUBLIC_AVATAR_BASENAME = re.compile( r"^[A-Za-z0-9][A-Za-z0-9._-]{0,239}\.(?:png|jpe?g|webp)$", re.IGNORECASE, ) _RUNTIME_GENERATED_AVATAR_BASENAME = re.compile( r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" r"-[A-Za-z0-9_-]{8,32}\.(?:png|jpg|jpeg|webp)$" ) _IMAGE_FORMAT_BY_SUFFIX = { ".png": "PNG", ".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP", } DATABASE_IDENTITY_QUERY = """ SELECT current_database()::text AS database_name, current_user::text AS database_role, COALESCE(inet_server_addr()::text, 'local') AS server_address, inet_server_port() AS server_port """ def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def _sha256_text(value: str) -> str: return _sha256_bytes(value.encode("utf-8")) def database_target_sha256( *, database_name: str, database_role: str, server_address: str, server_port: int, ) -> str: """Hash the same credential-free connected-DB identity as release probes.""" if ( not isinstance(database_name, str) or not database_name.strip() or not isinstance(database_role, str) or not database_role.strip() or not isinstance(server_address, str) or not server_address.strip() ): raise ValueError("database target identity is incomplete") if ( not isinstance(server_port, int) or isinstance(server_port, bool) or server_port < 1 or server_port > 65535 ): raise ValueError("database target port is invalid") canonical = json.dumps( { "database": database_name, "database_role": database_role, "server_address": server_address, "server_port": server_port, }, ensure_ascii=True, separators=(",", ":"), sort_keys=True, ) return _sha256_text(canonical) async def connected_database_target_sha256(connection: Any) -> str: target = await connection.fetchrow(DATABASE_IDENTITY_QUERY) if target is None: raise ValueError("database target identity is unavailable") return database_target_sha256( database_name=str(target["database_name"]), database_role=str(target["database_role"]), server_address=str(target["server_address"]), server_port=int(target["server_port"]), ) def sha256_file(path: Path) -> str: assert_regular_file_without_reparse(path, "public avatar object") digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() class PublicAvatarImageInvalid(ValueError): """The authorized file exists, but cannot be served as a public avatar.""" def _read_regular_file_bytes( path: Path, label: str, *, max_bytes: int | None = None, ) -> bytes: """Read one already-authorized inode once for size/hash/cache binding.""" assert_regular_file_without_reparse(path, label) try: with path.open("rb") as stream: metadata = os.fstat(stream.fileno()) if not stat.S_ISREG(metadata.st_mode): raise ValueError(f"{label} must be a regular non-reparse file") if max_bytes is not None and metadata.st_size > max_bytes: raise PublicAvatarImageInvalid(f"{label} exceeds the size limit") content = stream.read(max_bytes + 1 if max_bytes is not None else -1) if max_bytes is not None and len(content) > max_bytes: raise PublicAvatarImageInvalid(f"{label} exceeds the size limit") return content except OSError as exc: raise ValueError(f"{label} could not be read") from exc def _public_avatar_media_type(relative_path: str) -> str: suffix = PurePosixPath(relative_path).suffix.casefold() return { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", }[suffix] def canonical_path_identity(path: Path) -> str: return str(path.resolve(strict=False)).replace("\\", "/").casefold() def canonical_path_sha256(path: Path) -> str: return _sha256_text(canonical_path_identity(path)) def _assert_outside_root(path: Path, root: Path, label: str) -> None: try: path.resolve(strict=False).relative_to(root.resolve(strict=True)) except ValueError: return raise ValueError(f"{label} must be outside the public upload root") def _is_reparse_or_symlink(path: Path) -> bool: try: metadata = path.lstat() except FileNotFoundError: return False if stat.S_ISLNK(metadata.st_mode): return True reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) return bool(getattr(metadata, "st_file_attributes", 0) & reparse_attribute) def _path_entry_exists_without_following(path: Path) -> bool: try: path.lstat() except FileNotFoundError: return False except OSError: # An unreadable directory entry is not equivalent to a proven absence. return True return True def assert_path_without_reparse( path: Path, label: str, *, require_absolute: bool = True, ) -> None: if require_absolute and not path.is_absolute(): raise ValueError(f"{label} must be absolute") absolute = Path(os.path.abspath(path)) for candidate in (absolute, *absolute.parents): try: candidate.lstat() except FileNotFoundError: continue except OSError as exc: raise ValueError(f"{label} path identity is unreadable") from exc if _is_reparse_or_symlink(candidate): raise ValueError(f"{label} cannot traverse a symlink or reparse point") def assert_regular_file_without_reparse(path: Path, label: str) -> None: assert_path_without_reparse(path, label) try: metadata = path.lstat() except FileNotFoundError as exc: raise ValueError(f"{label} is missing") from exc if _is_reparse_or_symlink(path) or not stat.S_ISREG(metadata.st_mode): raise ValueError(f"{label} must be a regular non-reparse file") def _is_lower_hex_sha256(value: object) -> bool: return ( isinstance(value, str) and len(value) == _SHA256_LENGTH and all(character in "0123456789abcdef" for character in value) ) def normalize_public_avatar_relative_path(value: str) -> str: if not isinstance(value, str): raise ValueError("public avatar reference must be text") if "%" in value or any( ord(character) < 32 or ord(character) == 127 for character in value ): raise ValueError("public avatar reference contains forbidden characters") normalized = value.replace("\\", "/").strip("/") path = PurePosixPath(normalized) if ( path.is_absolute() or len(path.parts) != 2 or path.parts[0] != PUBLIC_AVATAR_DIRECTORY or path.parts[1] in {"", ".", ".."} or any(part in {"", ".", ".."} for part in path.parts) or not _PUBLIC_AVATAR_BASENAME.fullmatch(path.parts[1]) ): raise ValueError( "public avatar reference must be a flat profile-avatars image path" ) return path.as_posix() def public_avatar_relative_path_sha256(value: str) -> str: return _sha256_text(normalize_public_avatar_relative_path(value).casefold()) def public_avatar_url_to_relative_path(value: str) -> str: prefix = "/uploads/" if not value.startswith(prefix) or "?" in value or "#" in value: raise ValueError("public avatar URL must be an exact /uploads/ path") return normalize_public_avatar_relative_path(value[len(prefix) :]) @dataclass(frozen=True, slots=True) class AvatarImageDecodeProof: valid: bool image_format: str | None def inspect_public_avatar_image( relative_path: str, content: bytes, ) -> AvatarImageDecodeProof: """Fully decode one public avatar and bind its format to the suffix. Legacy bytes stay preservable even when invalid. Callers use this proof to keep those bytes forensic-only instead of returning them as public images. """ normalized = normalize_public_avatar_relative_path(relative_path) expected_format = _IMAGE_FORMAT_BY_SUFFIX[PurePosixPath(normalized).suffix.lower()] if not isinstance(content, bytes) or not content: return AvatarImageDecodeProof(valid=False, image_format=None) try: with warnings.catch_warnings(): warnings.simplefilter("error", Image.DecompressionBombWarning) with Image.open(io.BytesIO(content)) as candidate: verified_format = candidate.format candidate.verify() with Image.open(io.BytesIO(content)) as candidate: loaded_format = candidate.format candidate.load() image_format = str(loaded_format or verified_format or "").upper() or None return AvatarImageDecodeProof( valid=( image_format == expected_format and str(verified_format or "").upper() == expected_format ), image_format=image_format, ) except ( OSError, SyntaxError, UnidentifiedImageError, ValueError, Warning, Image.DecompressionBombError, ): return AvatarImageDecodeProof(valid=False, image_format=None) @dataclass(frozen=True, slots=True) class PublicAvatarObjectProof: content: bytes content_sha256: str size_bytes: int media_type: str def read_decodable_public_avatar_object( *, relative_path: str, path: Path, label: str, ) -> PublicAvatarObjectProof: """Bind one regular file read to its full image decode and response metadata.""" normalized = normalize_public_avatar_relative_path(relative_path) content = _read_regular_file_bytes( path, label, max_bytes=PUBLIC_AVATAR_MAX_BYTES, ) if not inspect_public_avatar_image(normalized, content).valid: raise PublicAvatarImageInvalid(f"{label} is not a decodable image") return PublicAvatarObjectProof( content=content, content_sha256=_sha256_bytes(content), size_bytes=len(content), media_type=_public_avatar_media_type(normalized), ) def is_runtime_generated_public_avatar_relative_path(value: str) -> bool: """Return true only for the server's UUID + random-token filename shape.""" try: normalized = normalize_public_avatar_relative_path(value) except ValueError: return False return _RUNTIME_GENERATED_AVATAR_BASENAME.fullmatch( PurePosixPath(normalized).name ) is not None def _object_record( upload_root: Path, relative_path: str, kind: str, reference_count: int, ) -> dict[str, object]: normalized = normalize_public_avatar_relative_path(relative_path) target = upload_root / Path(*PurePosixPath(normalized).parts) assert_regular_file_without_reparse(target, "required public avatar object") if kind != "profile_avatar": raise ValueError("manifest can contain only public profile avatars") content = _read_regular_file_bytes(target, "required public avatar object") decode_proof = inspect_public_avatar_image(normalized, content) return { "path_sha256": public_avatar_relative_path_sha256(normalized), "content_sha256": _sha256_bytes(content), "size_bytes": len(content), "kind": kind, "reference_count": reference_count, "decode_valid": decode_proof.valid, } def _aggregate_references( references: Sequence[tuple[str, str]], ) -> list[tuple[str, str, int]]: counts: dict[str, int] = {} canonical: dict[str, str] = {} for relative_path, kind in references: if kind != "profile_avatar": raise ValueError("manifest can contain only public profile avatars") normalized = normalize_public_avatar_relative_path(relative_path) key = normalized.casefold() canonical.setdefault(key, normalized) counts[key] = counts.get(key, 0) + 1 return sorted( ((canonical[key], "profile_avatar", counts[key]) for key in counts), key=lambda item: public_avatar_relative_path_sha256(item[0]), ) def _reference_records_from_objects( records: Sequence[dict[str, object]], ) -> list[dict[str, object]]: return sorted( ( { "path_sha256": str(record["path_sha256"]), "reference_count": int(record["reference_count"]), } for record in records ), key=lambda item: str(item["path_sha256"]), ) def _reference_set_sha256(records: Sequence[dict[str, object]]) -> str: serialized = json.dumps(list(records), sort_keys=True, separators=(",", ":")) return _sha256_text(serialized) def _preserved_object_set_sha256(records: Sequence[dict[str, object]]) -> str: identity_records = [ { "path_sha256": str(record["path_sha256"]), "content_sha256": str(record["content_sha256"]), "size_bytes": int(record["size_bytes"]), "kind": str(record["kind"]), } for record in records ] serialized = json.dumps( identity_records, sort_keys=True, separators=(",", ":"), ) return _sha256_text(serialized) def build_privacy_safe_manifest( *, upload_root: Path, references: Sequence[tuple[str, str]], preserved_paths: Sequence[str], database_target_sha256: str, write_freeze_token_sha256: str, write_freeze_path: Path | None = None, ) -> dict[str, object]: assert_path_without_reparse(upload_root, "public upload root") root = upload_root.resolve(strict=True) aggregated = _aggregate_references(references) reference_counts = { relative_path.casefold(): reference_count for relative_path, _kind, reference_count in aggregated } canonical_preserved: dict[str, str] = {} for raw_path in preserved_paths: normalized = normalize_public_avatar_relative_path(raw_path) key = normalized.casefold() if key in canonical_preserved: raise ValueError("preserved public avatar path is duplicated") canonical_preserved[key] = normalized if not canonical_preserved: raise ValueError("preserved public avatar inventory cannot be empty") if not set(reference_counts).issubset(canonical_preserved): raise ValueError("database avatar reference is absent from preserved inventory") records = sorted( ( _object_record( root, relative_path, "profile_avatar", reference_counts.get(key, 0), ) for key, relative_path in canonical_preserved.items() ), key=lambda record: str(record["path_sha256"]), ) database_records = _reference_records_from_objects( [record for record in records if int(record["reference_count"]) > 0] ) preserved_decode_valid_count = sum( 1 for record in records if bool(record["decode_valid"]) ) required_decode_invalid_records = [ record for record in records if int(record["reference_count"]) > 0 and not bool(record["decode_valid"]) ] if not _is_lower_hex_sha256(write_freeze_token_sha256): raise ValueError("write freeze token SHA256 is invalid") if not _is_lower_hex_sha256(database_target_sha256): raise ValueError("database target SHA256 is invalid") freeze: dict[str, object] = {"token_sha256": write_freeze_token_sha256} if write_freeze_path is not None: if not write_freeze_path.is_absolute(): raise ValueError("upload write-freeze path must be absolute") assert_path_without_reparse(write_freeze_path, "upload write-freeze path") _assert_outside_root(write_freeze_path, root, "upload write-freeze path") freeze["path_sha256"] = canonical_path_sha256(write_freeze_path) return { "schema_version": MANIFEST_SCHEMA_VERSION, "status": "initialized_complete", "root_path_sha256": canonical_path_sha256(root), "database_target_sha256": database_target_sha256, "preserved_object_count": len(records), "preserved_total_size_bytes": sum( int(record["size_bytes"]) for record in records ), "image_decode_contract": { "schema_version": IMAGE_DECODE_CONTRACT_VERSION, "decoder": "Pillow", "decoder_version": PILLOW_VERSION, "mode": "verify_load_extension_match", }, "preserved_decode_valid_count": preserved_decode_valid_count, "preserved_decode_invalid_count": len(records) - preserved_decode_valid_count, "required_decode_invalid_object_count": len( required_decode_invalid_records ), "required_decode_invalid_reference_count": sum( int(record["reference_count"]) for record in required_decode_invalid_records ), "preserved_object_set_sha256": _preserved_object_set_sha256(records), "preserved_objects": records, "required_object_count": len(database_records), "required_reference_count": sum( int(record["reference_count"]) for record in records ), "reference_set_sha256": _reference_set_sha256(database_records), "database_reference_contract": { "reference_count": sum( int(record["reference_count"]) for record in records ), "unique_object_count": len(database_records), "reference_set_sha256": _reference_set_sha256(database_records), "objects": database_records, }, "write_freeze": freeze, "privacy": { "raw_relative_paths_recorded": False, "raw_user_ids_recorded": False, "raw_filenames_recorded": False, "emails_recorded": False, }, } @dataclass(frozen=True, slots=True) class UploadManifestProof: manifest_sha256: str root_path_sha256: str database_target_sha256: str preserved_object_count: int preserved_total_size_bytes: int preserved_decode_valid_count: int preserved_decode_invalid_count: int required_decode_invalid_object_count: int required_decode_invalid_reference_count: int preserved_object_set_sha256: str required_object_count: int required_reference_count: int reference_set_sha256: str write_freeze_token_sha256: str preserved_records: tuple[tuple[str, str, int, str, bytes], ...] preserved_path_decode_records: tuple[tuple[str, bool], ...] reference_records: tuple[tuple[str, int], ...] def _iter_keys(value: object) -> Iterable[str]: if isinstance(value, dict): for key, child in value.items(): yield str(key) yield from _iter_keys(child) elif isinstance(value, list): for child in value: yield from _iter_keys(child) def validate_upload_manifest( *, upload_root: Path, manifest_path: Path, expected_manifest_sha256: str, expected_write_freeze_path: Path, expected_database_target_sha256: str, require_freeze_path_binding: bool = True, verify_preserved_objects: bool = True, reject_unbound_extras: bool = True, ) -> UploadManifestProof: if not upload_root.is_absolute(): raise ValueError("public upload root must be absolute") if not manifest_path.is_absolute(): raise ValueError("upload manifest path must be absolute") if not expected_write_freeze_path.is_absolute(): raise ValueError("upload write-freeze path must be absolute") assert_path_without_reparse(upload_root, "public upload root") assert_path_without_reparse(manifest_path, "upload manifest path") assert_path_without_reparse( expected_write_freeze_path, "upload write-freeze path", ) root = upload_root.resolve(strict=True) manifest = manifest_path.resolve(strict=True) assert_regular_file_without_reparse(manifest, "upload manifest") _assert_outside_root(manifest, root, "upload manifest") _assert_outside_root( expected_write_freeze_path, root, "upload write-freeze path", ) try: manifest_bytes = manifest.read_bytes() except OSError as exc: raise ValueError("upload manifest could not be read") from exc actual_manifest_sha256 = _sha256_bytes(manifest_bytes) if not _is_lower_hex_sha256(expected_manifest_sha256): raise ValueError("expected upload manifest SHA256 is invalid") if actual_manifest_sha256 != expected_manifest_sha256: raise ValueError("upload manifest SHA256 drift") try: payload = json.loads(manifest_bytes.decode("utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise ValueError("upload manifest is not valid UTF-8 JSON") from exc if not isinstance(payload, dict): raise ValueError("upload manifest must be a JSON object") forbidden_keys = { "relative_path", "source_path", "avatar_url", "user_id", "email", "filename", } if forbidden_keys.intersection(key.casefold() for key in _iter_keys(payload)): raise ValueError("upload manifest contains a privacy-sensitive key") if payload.get("schema_version") != MANIFEST_SCHEMA_VERSION: raise ValueError("upload manifest schema version mismatch") if payload.get("status") != "initialized_complete": raise ValueError("upload manifest is not initialized and complete") root_sha256 = canonical_path_sha256(root) if payload.get("root_path_sha256") != root_sha256: raise ValueError("upload manifest root binding mismatch") if ( not _is_lower_hex_sha256(expected_database_target_sha256) or payload.get("database_target_sha256") != expected_database_target_sha256 ): raise ValueError("upload manifest database target binding mismatch") if "objects" in payload: raise ValueError("legacy upload manifest object contract is rejected") preserved_objects = payload.get("preserved_objects") if not isinstance(preserved_objects, list): raise ValueError("upload manifest preserved objects must be an array") preserved_count = payload.get("preserved_object_count") if ( not isinstance(preserved_count, int) or preserved_count <= 0 or preserved_count != len(preserved_objects) ): raise ValueError("upload manifest preserved object count mismatch") preserved_total_size_bytes = payload.get("preserved_total_size_bytes") if ( not isinstance(preserved_total_size_bytes, int) or isinstance(preserved_total_size_bytes, bool) or preserved_total_size_bytes < 0 ): raise ValueError("upload manifest preserved total size is invalid") expected_decode_contract = { "schema_version": IMAGE_DECODE_CONTRACT_VERSION, "decoder": "Pillow", "decoder_version": PILLOW_VERSION, "mode": "verify_load_extension_match", } if payload.get("image_decode_contract") != expected_decode_contract: raise ValueError("upload manifest image decode contract mismatch") decode_count_names = ( "preserved_decode_valid_count", "preserved_decode_invalid_count", "required_decode_invalid_object_count", "required_decode_invalid_reference_count", ) decode_counts: dict[str, int] = {} for name in decode_count_names: value = payload.get(name) if not isinstance(value, int) or isinstance(value, bool) or value < 0: raise ValueError("upload manifest image decode count is invalid") decode_counts[name] = value required_count = payload.get("required_object_count") if ( not isinstance(required_count, int) or required_count < 0 or required_count > preserved_count ): raise ValueError("upload manifest database object count mismatch") required_reference_count = payload.get("required_reference_count") if ( not isinstance(required_reference_count, int) or required_reference_count < 0 or required_reference_count < required_count ): raise ValueError("upload manifest reference count is invalid") freeze = payload.get("write_freeze") if not isinstance(freeze, dict): raise ValueError("upload manifest write-freeze binding is missing") token_sha256 = freeze.get("token_sha256") if not _is_lower_hex_sha256(token_sha256): raise ValueError("upload manifest write-freeze token binding is invalid") if require_freeze_path_binding: if freeze.get("path_sha256") != canonical_path_sha256( expected_write_freeze_path ): raise ValueError("upload manifest write-freeze path binding mismatch") available: dict[str, tuple[str, Path]] = {} avatar_root = root / PUBLIC_AVATAR_DIRECTORY if verify_preserved_objects or reject_unbound_extras: if not avatar_root.is_dir() or _is_reparse_or_symlink(avatar_root): raise ValueError("public avatar directory must be a non-reparse directory") for candidate in avatar_root.iterdir(): if _is_reparse_or_symlink(candidate) or not candidate.is_file(): raise ValueError("public avatar directory contains an unsafe extra") try: relative = normalize_public_avatar_relative_path( f"{PUBLIC_AVATAR_DIRECTORY}/{candidate.name}" ) except ValueError as exc: raise ValueError( "public avatar directory contains an invalid extra" ) from exc path_digest = public_avatar_relative_path_sha256(relative) if path_digest in available: raise ValueError("public avatar directory contains a duplicate path") available[path_digest] = (relative, candidate) normalized_records: list[dict[str, object]] = [] preserved_cache: list[tuple[str, str, int, str, bytes]] = [] preserved_cache_bytes = 0 seen: set[str] = set() for item in preserved_objects: if not isinstance(item, dict): raise ValueError("upload manifest object entry must be an object") path_sha256 = item.get("path_sha256") content_sha256 = item.get("content_sha256") size_bytes = item.get("size_bytes") reference_count = item.get("reference_count") decode_valid = item.get("decode_valid") if not _is_lower_hex_sha256(path_sha256) or path_sha256 in seen: raise ValueError("upload manifest path hash is invalid or duplicated") if not _is_lower_hex_sha256(content_sha256): raise ValueError("upload manifest content hash is invalid") if not isinstance(size_bytes, int) or size_bytes < 0: raise ValueError("upload manifest object size is invalid") if not isinstance(reference_count, int) or reference_count < 0: raise ValueError("upload manifest object reference count is invalid") if not isinstance(decode_valid, bool): raise ValueError("upload manifest object decode status is invalid") if item.get("kind") != "profile_avatar": raise ValueError("upload manifest contains a non-public object kind") if verify_preserved_objects: available_record = available.get(path_sha256) if available_record is None: raise ValueError("preserved public avatar object is missing") _relative_path, target = available_record assert_regular_file_without_reparse( target, "preserved public avatar object", ) content = _read_regular_file_bytes( target, "preserved public avatar object", ) if len(content) != size_bytes or _sha256_bytes(content) != content_sha256: raise ValueError( "preserved public avatar object content hash or size drift" ) actual_decode_valid = inspect_public_avatar_image( _relative_path, content, ).valid if actual_decode_valid is not decode_valid: raise ValueError("preserved public avatar decode status drift") if decode_valid: preserved_cache_bytes += len(content) if preserved_cache_bytes > MAX_PRESERVED_AVATAR_CACHE_BYTES: raise ValueError( "preserved public avatar cache size limit exceeded" ) preserved_cache.append( ( str(path_sha256), str(content_sha256), int(size_bytes), _public_avatar_media_type(_relative_path), content, ) ) seen.add(path_sha256) normalized_records.append( { "path_sha256": path_sha256, "content_sha256": content_sha256, "size_bytes": size_bytes, "kind": "profile_avatar", "reference_count": reference_count, "decode_valid": decode_valid, } ) normalized_records.sort(key=lambda item: str(item["path_sha256"])) if sum(int(record["size_bytes"]) for record in normalized_records) != ( preserved_total_size_bytes ): raise ValueError("upload manifest preserved total size mismatch") preserved_decode_valid_count = sum( 1 for record in normalized_records if bool(record["decode_valid"]) ) preserved_decode_invalid_count = len(normalized_records) - ( preserved_decode_valid_count ) required_decode_invalid_records = [ record for record in normalized_records if int(record["reference_count"]) > 0 and not bool(record["decode_valid"]) ] if ( decode_counts["preserved_decode_valid_count"] != preserved_decode_valid_count or decode_counts["preserved_decode_invalid_count"] != preserved_decode_invalid_count or decode_counts["required_decode_invalid_object_count"] != len(required_decode_invalid_records) or decode_counts["required_decode_invalid_reference_count"] != sum( int(record["reference_count"]) for record in required_decode_invalid_records ) ): raise ValueError("upload manifest image decode count mismatch") unbound = set(available).difference(seen) if reject_unbound_extras and unbound: raise ValueError("public avatar directory contains an unbound extra") if any( not is_runtime_generated_public_avatar_relative_path(available[digest][0]) for digest in unbound ): raise ValueError("public avatar directory contains an unauthorized extra") preserved_set_sha256 = _preserved_object_set_sha256(normalized_records) if payload.get("preserved_object_set_sha256") != preserved_set_sha256: raise ValueError("upload manifest preserved-object set hash mismatch") referenced_records = [ record for record in normalized_records if int(record["reference_count"]) > 0 ] reference_records = _reference_records_from_objects(referenced_records) reference_set_sha256 = _reference_set_sha256(reference_records) if payload.get("reference_set_sha256") != reference_set_sha256: raise ValueError("upload manifest reference-set hash mismatch") if ( len(reference_records) != required_count or sum(int(record["reference_count"]) for record in referenced_records) != required_reference_count ): raise ValueError("upload manifest database reference count mismatch") database_contract = payload.get("database_reference_contract") if not isinstance(database_contract, dict): raise ValueError("upload manifest database reference contract is invalid") if ( database_contract.get("reference_count") != required_reference_count or database_contract.get("unique_object_count") != required_count or database_contract.get("reference_set_sha256") != reference_set_sha256 or database_contract.get("objects") != reference_records ): raise ValueError("upload manifest database reference contract drift") return UploadManifestProof( manifest_sha256=actual_manifest_sha256, root_path_sha256=root_sha256, database_target_sha256=expected_database_target_sha256, preserved_object_count=preserved_count, preserved_total_size_bytes=preserved_total_size_bytes, preserved_decode_valid_count=preserved_decode_valid_count, preserved_decode_invalid_count=preserved_decode_invalid_count, required_decode_invalid_object_count=len( required_decode_invalid_records ), required_decode_invalid_reference_count=sum( int(record["reference_count"]) for record in required_decode_invalid_records ), preserved_object_set_sha256=preserved_set_sha256, required_object_count=required_count, required_reference_count=required_reference_count, reference_set_sha256=reference_set_sha256, write_freeze_token_sha256=token_sha256, preserved_records=tuple(preserved_cache), preserved_path_decode_records=tuple( (str(record["path_sha256"]), bool(record["decode_valid"])) for record in normalized_records ), reference_records=tuple( (str(record["path_sha256"]), int(record["reference_count"])) for record in reference_records ), ) @dataclass(frozen=True, slots=True) class CurrentAvatarReferenceProof: object_count: int reference_count: int reference_set_sha256: str decode_invalid_object_count: int decode_invalid_reference_count: int def validate_current_avatar_references( *, upload_root: Path, avatar_urls: Sequence[str], initial_manifest: UploadManifestProof, expected_database_target_sha256: str, ) -> CurrentAvatarReferenceProof: if ( not _is_lower_hex_sha256(expected_database_target_sha256) or initial_manifest.database_target_sha256 != expected_database_target_sha256 ): raise ValueError("current avatar inventory database target is unverified") references = [ (public_avatar_url_to_relative_path(value), "profile_avatar") for value in avatar_urls if isinstance(value, str) and value.strip() ] aggregated = _aggregate_references(references) if initial_manifest.required_reference_count > 0 and not aggregated: raise ValueError( "current public avatar inventory is empty without an explicit reset receipt" ) assert_path_without_reparse(upload_root, "public upload root") root = upload_root.resolve(strict=True) records: list[dict[str, object]] = [] preserved_decode_status = dict(initial_manifest.preserved_path_decode_records) decode_invalid_object_count = 0 decode_invalid_reference_count = 0 for relative_path, _kind, reference_count in aggregated: target = root / Path(*PurePosixPath(relative_path).parts) path_sha256 = public_avatar_relative_path_sha256(relative_path) preserved_decode_valid = preserved_decode_status.get(path_sha256) if preserved_decode_valid is None: if not is_runtime_generated_public_avatar_relative_path(relative_path): raise ValueError( "current DB-referenced avatar object is not authorized" ) try: read_decodable_public_avatar_object( relative_path=relative_path, path=target, label="current DB-referenced avatar object", ) except PublicAvatarImageInvalid: decode_invalid_object_count += 1 decode_invalid_reference_count += reference_count else: sha256_file(target) if preserved_decode_valid is False: decode_invalid_object_count += 1 decode_invalid_reference_count += reference_count records.append( {"path_sha256": path_sha256, "reference_count": reference_count} ) records.sort(key=lambda item: str(item["path_sha256"])) return CurrentAvatarReferenceProof( object_count=len(records), reference_count=sum(int(item["reference_count"]) for item in records), reference_set_sha256=_reference_set_sha256(records), decode_invalid_object_count=decode_invalid_object_count, decode_invalid_reference_count=decode_invalid_reference_count, ) class UploadWriteFrozen(RuntimeError): pass @dataclass(frozen=True, slots=True) class UploadWriteFreezeStatus: capable: bool active: bool valid: bool in_flight: int path_sha256: str | None token_sha256: str | None class UploadWriteFreezeGate: def __init__(self, freeze_path: Path | None): self._freeze_path = freeze_path self._lock = threading.Lock() self._in_flight = 0 def status(self) -> UploadWriteFreezeStatus: path = self._freeze_path with self._lock: in_flight = self._in_flight if path is None: return UploadWriteFreezeStatus( False, False, False, in_flight, None, None ) path_sha256 = canonical_path_sha256(path) try: assert_path_without_reparse(path, "upload write-freeze path") except ValueError: return UploadWriteFreezeStatus( True, True, False, in_flight, path_sha256, None ) if not _path_entry_exists_without_following(path): return UploadWriteFreezeStatus( True, False, True, in_flight, path_sha256, None ) try: payload = json.loads(path.read_text(encoding="utf-8")) token = payload.get("token") if isinstance(payload, dict) else None valid = ( isinstance(payload, dict) and payload.get("schema_version") == WRITE_FREEZE_SCHEMA_VERSION and isinstance(token, str) and len(token) >= 32 ) token_sha256 = _sha256_text(token) if valid else None except (OSError, UnicodeError, json.JSONDecodeError): valid = False token_sha256 = None return UploadWriteFreezeStatus( True, True, valid, in_flight, path_sha256, token_sha256, ) @contextmanager def write_lease(self) -> Iterator[None]: with self._lock: if self._freeze_path is not None and _path_entry_exists_without_following( self._freeze_path ): raise UploadWriteFrozen("public upload writes are temporarily frozen") self._in_flight += 1 try: yield finally: with self._lock: self._in_flight -= 1