1059 lines
40 KiB
Python
1059 lines
40 KiB
Python
"""Public avatar storage manifest and upload-write freeze contracts.
|
|
|
|
The public surface owns only ``profile-avatars/<file>``. Private multimodal
|
|
audio is deliberately excluded from both manifests and static serving.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
import threading
|
|
import warnings
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, Iterable, Iterator, Sequence
|
|
|
|
from PIL import Image, UnidentifiedImageError, __version__ as PILLOW_VERSION
|
|
|
|
MANIFEST_SCHEMA_VERSION = "vignette.public-avatar-upload-manifest.v3"
|
|
WRITE_FREEZE_SCHEMA_VERSION = "vignette.public-upload-write-freeze.v1"
|
|
PUBLIC_AVATAR_DIRECTORY = "profile-avatars"
|
|
PUBLIC_AVATAR_MAX_BYTES = 3 * 1024 * 1024
|
|
MAX_PRESERVED_AVATAR_CACHE_BYTES = 64 * 1024 * 1024
|
|
IMAGE_DECODE_CONTRACT_VERSION = "vignette.public-avatar-image-decode.v1"
|
|
_SHA256_LENGTH = 64
|
|
_PUBLIC_AVATAR_BASENAME = re.compile(
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,239}\.(?:png|jpe?g|webp)$",
|
|
re.IGNORECASE,
|
|
)
|
|
_RUNTIME_GENERATED_AVATAR_BASENAME = re.compile(
|
|
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
|
|
r"-[A-Za-z0-9_-]{8,32}\.(?:png|jpg|jpeg|webp)$"
|
|
)
|
|
_IMAGE_FORMAT_BY_SUFFIX = {
|
|
".png": "PNG",
|
|
".jpg": "JPEG",
|
|
".jpeg": "JPEG",
|
|
".webp": "WEBP",
|
|
}
|
|
DATABASE_IDENTITY_QUERY = """
|
|
SELECT
|
|
current_database()::text AS database_name,
|
|
current_user::text AS database_role,
|
|
COALESCE(inet_server_addr()::text, 'local') AS server_address,
|
|
inet_server_port() AS server_port
|
|
"""
|
|
|
|
|
|
def _sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def _sha256_text(value: str) -> str:
|
|
return _sha256_bytes(value.encode("utf-8"))
|
|
|
|
|
|
def database_target_sha256(
|
|
*,
|
|
database_name: str,
|
|
database_role: str,
|
|
server_address: str,
|
|
server_port: int,
|
|
) -> str:
|
|
"""Hash the same credential-free connected-DB identity as release probes."""
|
|
|
|
if (
|
|
not isinstance(database_name, str)
|
|
or not database_name.strip()
|
|
or not isinstance(database_role, str)
|
|
or not database_role.strip()
|
|
or not isinstance(server_address, str)
|
|
or not server_address.strip()
|
|
):
|
|
raise ValueError("database target identity is incomplete")
|
|
if (
|
|
not isinstance(server_port, int)
|
|
or isinstance(server_port, bool)
|
|
or server_port < 1
|
|
or server_port > 65535
|
|
):
|
|
raise ValueError("database target port is invalid")
|
|
canonical = json.dumps(
|
|
{
|
|
"database": database_name,
|
|
"database_role": database_role,
|
|
"server_address": server_address,
|
|
"server_port": server_port,
|
|
},
|
|
ensure_ascii=True,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
)
|
|
return _sha256_text(canonical)
|
|
|
|
|
|
async def connected_database_target_sha256(connection: Any) -> str:
|
|
target = await connection.fetchrow(DATABASE_IDENTITY_QUERY)
|
|
if target is None:
|
|
raise ValueError("database target identity is unavailable")
|
|
return database_target_sha256(
|
|
database_name=str(target["database_name"]),
|
|
database_role=str(target["database_role"]),
|
|
server_address=str(target["server_address"]),
|
|
server_port=int(target["server_port"]),
|
|
)
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
assert_regular_file_without_reparse(path, "public avatar object")
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
class PublicAvatarImageInvalid(ValueError):
|
|
"""The authorized file exists, but cannot be served as a public avatar."""
|
|
|
|
|
|
def _read_regular_file_bytes(
|
|
path: Path,
|
|
label: str,
|
|
*,
|
|
max_bytes: int | None = None,
|
|
) -> bytes:
|
|
"""Read one already-authorized inode once for size/hash/cache binding."""
|
|
|
|
assert_regular_file_without_reparse(path, label)
|
|
try:
|
|
with path.open("rb") as stream:
|
|
metadata = os.fstat(stream.fileno())
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise ValueError(f"{label} must be a regular non-reparse file")
|
|
if max_bytes is not None and metadata.st_size > max_bytes:
|
|
raise PublicAvatarImageInvalid(f"{label} exceeds the size limit")
|
|
content = stream.read(max_bytes + 1 if max_bytes is not None else -1)
|
|
if max_bytes is not None and len(content) > max_bytes:
|
|
raise PublicAvatarImageInvalid(f"{label} exceeds the size limit")
|
|
return content
|
|
except OSError as exc:
|
|
raise ValueError(f"{label} could not be read") from exc
|
|
|
|
|
|
def _public_avatar_media_type(relative_path: str) -> str:
|
|
suffix = PurePosixPath(relative_path).suffix.casefold()
|
|
return {
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".webp": "image/webp",
|
|
}[suffix]
|
|
|
|
|
|
def canonical_path_identity(path: Path) -> str:
|
|
return str(path.resolve(strict=False)).replace("\\", "/").casefold()
|
|
|
|
|
|
def canonical_path_sha256(path: Path) -> str:
|
|
return _sha256_text(canonical_path_identity(path))
|
|
|
|
|
|
def _assert_outside_root(path: Path, root: Path, label: str) -> None:
|
|
try:
|
|
path.resolve(strict=False).relative_to(root.resolve(strict=True))
|
|
except ValueError:
|
|
return
|
|
raise ValueError(f"{label} must be outside the public upload root")
|
|
|
|
|
|
def _is_reparse_or_symlink(path: Path) -> bool:
|
|
try:
|
|
metadata = path.lstat()
|
|
except FileNotFoundError:
|
|
return False
|
|
if stat.S_ISLNK(metadata.st_mode):
|
|
return True
|
|
reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
return bool(getattr(metadata, "st_file_attributes", 0) & reparse_attribute)
|
|
|
|
|
|
def _path_entry_exists_without_following(path: Path) -> bool:
|
|
try:
|
|
path.lstat()
|
|
except FileNotFoundError:
|
|
return False
|
|
except OSError:
|
|
# An unreadable directory entry is not equivalent to a proven absence.
|
|
return True
|
|
return True
|
|
|
|
|
|
def assert_path_without_reparse(
|
|
path: Path,
|
|
label: str,
|
|
*,
|
|
require_absolute: bool = True,
|
|
) -> None:
|
|
if require_absolute and not path.is_absolute():
|
|
raise ValueError(f"{label} must be absolute")
|
|
absolute = Path(os.path.abspath(path))
|
|
for candidate in (absolute, *absolute.parents):
|
|
try:
|
|
candidate.lstat()
|
|
except FileNotFoundError:
|
|
continue
|
|
except OSError as exc:
|
|
raise ValueError(f"{label} path identity is unreadable") from exc
|
|
if _is_reparse_or_symlink(candidate):
|
|
raise ValueError(f"{label} cannot traverse a symlink or reparse point")
|
|
|
|
|
|
def assert_regular_file_without_reparse(path: Path, label: str) -> None:
|
|
assert_path_without_reparse(path, label)
|
|
try:
|
|
metadata = path.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise ValueError(f"{label} is missing") from exc
|
|
if _is_reparse_or_symlink(path) or not stat.S_ISREG(metadata.st_mode):
|
|
raise ValueError(f"{label} must be a regular non-reparse file")
|
|
|
|
|
|
def _is_lower_hex_sha256(value: object) -> bool:
|
|
return (
|
|
isinstance(value, str)
|
|
and len(value) == _SHA256_LENGTH
|
|
and all(character in "0123456789abcdef" for character in value)
|
|
)
|
|
|
|
|
|
def normalize_public_avatar_relative_path(value: str) -> str:
|
|
if not isinstance(value, str):
|
|
raise ValueError("public avatar reference must be text")
|
|
if "%" in value or any(
|
|
ord(character) < 32 or ord(character) == 127 for character in value
|
|
):
|
|
raise ValueError("public avatar reference contains forbidden characters")
|
|
normalized = value.replace("\\", "/").strip("/")
|
|
path = PurePosixPath(normalized)
|
|
if (
|
|
path.is_absolute()
|
|
or len(path.parts) != 2
|
|
or path.parts[0] != PUBLIC_AVATAR_DIRECTORY
|
|
or path.parts[1] in {"", ".", ".."}
|
|
or any(part in {"", ".", ".."} for part in path.parts)
|
|
or not _PUBLIC_AVATAR_BASENAME.fullmatch(path.parts[1])
|
|
):
|
|
raise ValueError(
|
|
"public avatar reference must be a flat profile-avatars image path"
|
|
)
|
|
return path.as_posix()
|
|
|
|
|
|
def public_avatar_relative_path_sha256(value: str) -> str:
|
|
return _sha256_text(normalize_public_avatar_relative_path(value).casefold())
|
|
|
|
|
|
def public_avatar_url_to_relative_path(value: str) -> str:
|
|
prefix = "/uploads/"
|
|
if not value.startswith(prefix) or "?" in value or "#" in value:
|
|
raise ValueError("public avatar URL must be an exact /uploads/ path")
|
|
return normalize_public_avatar_relative_path(value[len(prefix) :])
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class AvatarImageDecodeProof:
|
|
valid: bool
|
|
image_format: str | None
|
|
|
|
|
|
def inspect_public_avatar_image(
|
|
relative_path: str,
|
|
content: bytes,
|
|
) -> AvatarImageDecodeProof:
|
|
"""Fully decode one public avatar and bind its format to the suffix.
|
|
|
|
Legacy bytes stay preservable even when invalid. Callers use this proof to
|
|
keep those bytes forensic-only instead of returning them as public images.
|
|
"""
|
|
|
|
normalized = normalize_public_avatar_relative_path(relative_path)
|
|
expected_format = _IMAGE_FORMAT_BY_SUFFIX[PurePosixPath(normalized).suffix.lower()]
|
|
if not isinstance(content, bytes) or not content:
|
|
return AvatarImageDecodeProof(valid=False, image_format=None)
|
|
try:
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
|
with Image.open(io.BytesIO(content)) as candidate:
|
|
verified_format = candidate.format
|
|
candidate.verify()
|
|
with Image.open(io.BytesIO(content)) as candidate:
|
|
loaded_format = candidate.format
|
|
candidate.load()
|
|
image_format = str(loaded_format or verified_format or "").upper() or None
|
|
return AvatarImageDecodeProof(
|
|
valid=(
|
|
image_format == expected_format
|
|
and str(verified_format or "").upper() == expected_format
|
|
),
|
|
image_format=image_format,
|
|
)
|
|
except (
|
|
OSError,
|
|
SyntaxError,
|
|
UnidentifiedImageError,
|
|
ValueError,
|
|
Warning,
|
|
Image.DecompressionBombError,
|
|
):
|
|
return AvatarImageDecodeProof(valid=False, image_format=None)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublicAvatarObjectProof:
|
|
content: bytes
|
|
content_sha256: str
|
|
size_bytes: int
|
|
media_type: str
|
|
|
|
|
|
def read_decodable_public_avatar_object(
|
|
*,
|
|
relative_path: str,
|
|
path: Path,
|
|
label: str,
|
|
) -> PublicAvatarObjectProof:
|
|
"""Bind one regular file read to its full image decode and response metadata."""
|
|
|
|
normalized = normalize_public_avatar_relative_path(relative_path)
|
|
content = _read_regular_file_bytes(
|
|
path,
|
|
label,
|
|
max_bytes=PUBLIC_AVATAR_MAX_BYTES,
|
|
)
|
|
if not inspect_public_avatar_image(normalized, content).valid:
|
|
raise PublicAvatarImageInvalid(f"{label} is not a decodable image")
|
|
return PublicAvatarObjectProof(
|
|
content=content,
|
|
content_sha256=_sha256_bytes(content),
|
|
size_bytes=len(content),
|
|
media_type=_public_avatar_media_type(normalized),
|
|
)
|
|
|
|
|
|
def is_runtime_generated_public_avatar_relative_path(value: str) -> bool:
|
|
"""Return true only for the server's UUID + random-token filename shape."""
|
|
|
|
try:
|
|
normalized = normalize_public_avatar_relative_path(value)
|
|
except ValueError:
|
|
return False
|
|
return _RUNTIME_GENERATED_AVATAR_BASENAME.fullmatch(
|
|
PurePosixPath(normalized).name
|
|
) is not None
|
|
|
|
|
|
def _object_record(
|
|
upload_root: Path,
|
|
relative_path: str,
|
|
kind: str,
|
|
reference_count: int,
|
|
) -> dict[str, object]:
|
|
normalized = normalize_public_avatar_relative_path(relative_path)
|
|
target = upload_root / Path(*PurePosixPath(normalized).parts)
|
|
assert_regular_file_without_reparse(target, "required public avatar object")
|
|
if kind != "profile_avatar":
|
|
raise ValueError("manifest can contain only public profile avatars")
|
|
content = _read_regular_file_bytes(target, "required public avatar object")
|
|
decode_proof = inspect_public_avatar_image(normalized, content)
|
|
return {
|
|
"path_sha256": public_avatar_relative_path_sha256(normalized),
|
|
"content_sha256": _sha256_bytes(content),
|
|
"size_bytes": len(content),
|
|
"kind": kind,
|
|
"reference_count": reference_count,
|
|
"decode_valid": decode_proof.valid,
|
|
}
|
|
|
|
|
|
def _aggregate_references(
|
|
references: Sequence[tuple[str, str]],
|
|
) -> list[tuple[str, str, int]]:
|
|
counts: dict[str, int] = {}
|
|
canonical: dict[str, str] = {}
|
|
for relative_path, kind in references:
|
|
if kind != "profile_avatar":
|
|
raise ValueError("manifest can contain only public profile avatars")
|
|
normalized = normalize_public_avatar_relative_path(relative_path)
|
|
key = normalized.casefold()
|
|
canonical.setdefault(key, normalized)
|
|
counts[key] = counts.get(key, 0) + 1
|
|
return sorted(
|
|
((canonical[key], "profile_avatar", counts[key]) for key in counts),
|
|
key=lambda item: public_avatar_relative_path_sha256(item[0]),
|
|
)
|
|
|
|
|
|
def _reference_records_from_objects(
|
|
records: Sequence[dict[str, object]],
|
|
) -> list[dict[str, object]]:
|
|
return sorted(
|
|
(
|
|
{
|
|
"path_sha256": str(record["path_sha256"]),
|
|
"reference_count": int(record["reference_count"]),
|
|
}
|
|
for record in records
|
|
),
|
|
key=lambda item: str(item["path_sha256"]),
|
|
)
|
|
|
|
|
|
def _reference_set_sha256(records: Sequence[dict[str, object]]) -> str:
|
|
serialized = json.dumps(list(records), sort_keys=True, separators=(",", ":"))
|
|
return _sha256_text(serialized)
|
|
|
|
|
|
def _preserved_object_set_sha256(records: Sequence[dict[str, object]]) -> str:
|
|
identity_records = [
|
|
{
|
|
"path_sha256": str(record["path_sha256"]),
|
|
"content_sha256": str(record["content_sha256"]),
|
|
"size_bytes": int(record["size_bytes"]),
|
|
"kind": str(record["kind"]),
|
|
}
|
|
for record in records
|
|
]
|
|
serialized = json.dumps(
|
|
identity_records,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
return _sha256_text(serialized)
|
|
|
|
|
|
def build_privacy_safe_manifest(
|
|
*,
|
|
upload_root: Path,
|
|
references: Sequence[tuple[str, str]],
|
|
preserved_paths: Sequence[str],
|
|
database_target_sha256: str,
|
|
write_freeze_token_sha256: str,
|
|
write_freeze_path: Path | None = None,
|
|
) -> dict[str, object]:
|
|
assert_path_without_reparse(upload_root, "public upload root")
|
|
root = upload_root.resolve(strict=True)
|
|
aggregated = _aggregate_references(references)
|
|
reference_counts = {
|
|
relative_path.casefold(): reference_count
|
|
for relative_path, _kind, reference_count in aggregated
|
|
}
|
|
canonical_preserved: dict[str, str] = {}
|
|
for raw_path in preserved_paths:
|
|
normalized = normalize_public_avatar_relative_path(raw_path)
|
|
key = normalized.casefold()
|
|
if key in canonical_preserved:
|
|
raise ValueError("preserved public avatar path is duplicated")
|
|
canonical_preserved[key] = normalized
|
|
if not canonical_preserved:
|
|
raise ValueError("preserved public avatar inventory cannot be empty")
|
|
if not set(reference_counts).issubset(canonical_preserved):
|
|
raise ValueError("database avatar reference is absent from preserved inventory")
|
|
records = sorted(
|
|
(
|
|
_object_record(
|
|
root,
|
|
relative_path,
|
|
"profile_avatar",
|
|
reference_counts.get(key, 0),
|
|
)
|
|
for key, relative_path in canonical_preserved.items()
|
|
),
|
|
key=lambda record: str(record["path_sha256"]),
|
|
)
|
|
database_records = _reference_records_from_objects(
|
|
[record for record in records if int(record["reference_count"]) > 0]
|
|
)
|
|
preserved_decode_valid_count = sum(
|
|
1 for record in records if bool(record["decode_valid"])
|
|
)
|
|
required_decode_invalid_records = [
|
|
record
|
|
for record in records
|
|
if int(record["reference_count"]) > 0
|
|
and not bool(record["decode_valid"])
|
|
]
|
|
if not _is_lower_hex_sha256(write_freeze_token_sha256):
|
|
raise ValueError("write freeze token SHA256 is invalid")
|
|
if not _is_lower_hex_sha256(database_target_sha256):
|
|
raise ValueError("database target SHA256 is invalid")
|
|
freeze: dict[str, object] = {"token_sha256": write_freeze_token_sha256}
|
|
if write_freeze_path is not None:
|
|
if not write_freeze_path.is_absolute():
|
|
raise ValueError("upload write-freeze path must be absolute")
|
|
assert_path_without_reparse(write_freeze_path, "upload write-freeze path")
|
|
_assert_outside_root(write_freeze_path, root, "upload write-freeze path")
|
|
freeze["path_sha256"] = canonical_path_sha256(write_freeze_path)
|
|
return {
|
|
"schema_version": MANIFEST_SCHEMA_VERSION,
|
|
"status": "initialized_complete",
|
|
"root_path_sha256": canonical_path_sha256(root),
|
|
"database_target_sha256": database_target_sha256,
|
|
"preserved_object_count": len(records),
|
|
"preserved_total_size_bytes": sum(
|
|
int(record["size_bytes"]) for record in records
|
|
),
|
|
"image_decode_contract": {
|
|
"schema_version": IMAGE_DECODE_CONTRACT_VERSION,
|
|
"decoder": "Pillow",
|
|
"decoder_version": PILLOW_VERSION,
|
|
"mode": "verify_load_extension_match",
|
|
},
|
|
"preserved_decode_valid_count": preserved_decode_valid_count,
|
|
"preserved_decode_invalid_count": len(records)
|
|
- preserved_decode_valid_count,
|
|
"required_decode_invalid_object_count": len(
|
|
required_decode_invalid_records
|
|
),
|
|
"required_decode_invalid_reference_count": sum(
|
|
int(record["reference_count"])
|
|
for record in required_decode_invalid_records
|
|
),
|
|
"preserved_object_set_sha256": _preserved_object_set_sha256(records),
|
|
"preserved_objects": records,
|
|
"required_object_count": len(database_records),
|
|
"required_reference_count": sum(
|
|
int(record["reference_count"]) for record in records
|
|
),
|
|
"reference_set_sha256": _reference_set_sha256(database_records),
|
|
"database_reference_contract": {
|
|
"reference_count": sum(
|
|
int(record["reference_count"]) for record in records
|
|
),
|
|
"unique_object_count": len(database_records),
|
|
"reference_set_sha256": _reference_set_sha256(database_records),
|
|
"objects": database_records,
|
|
},
|
|
"write_freeze": freeze,
|
|
"privacy": {
|
|
"raw_relative_paths_recorded": False,
|
|
"raw_user_ids_recorded": False,
|
|
"raw_filenames_recorded": False,
|
|
"emails_recorded": False,
|
|
},
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class UploadManifestProof:
|
|
manifest_sha256: str
|
|
root_path_sha256: str
|
|
database_target_sha256: str
|
|
preserved_object_count: int
|
|
preserved_total_size_bytes: int
|
|
preserved_decode_valid_count: int
|
|
preserved_decode_invalid_count: int
|
|
required_decode_invalid_object_count: int
|
|
required_decode_invalid_reference_count: int
|
|
preserved_object_set_sha256: str
|
|
required_object_count: int
|
|
required_reference_count: int
|
|
reference_set_sha256: str
|
|
write_freeze_token_sha256: str
|
|
preserved_records: tuple[tuple[str, str, int, str, bytes], ...]
|
|
preserved_path_decode_records: tuple[tuple[str, bool], ...]
|
|
reference_records: tuple[tuple[str, int], ...]
|
|
|
|
|
|
def _iter_keys(value: object) -> Iterable[str]:
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
yield str(key)
|
|
yield from _iter_keys(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
yield from _iter_keys(child)
|
|
|
|
|
|
def validate_upload_manifest(
|
|
*,
|
|
upload_root: Path,
|
|
manifest_path: Path,
|
|
expected_manifest_sha256: str,
|
|
expected_write_freeze_path: Path,
|
|
expected_database_target_sha256: str,
|
|
require_freeze_path_binding: bool = True,
|
|
verify_preserved_objects: bool = True,
|
|
reject_unbound_extras: bool = True,
|
|
) -> UploadManifestProof:
|
|
if not upload_root.is_absolute():
|
|
raise ValueError("public upload root must be absolute")
|
|
if not manifest_path.is_absolute():
|
|
raise ValueError("upload manifest path must be absolute")
|
|
if not expected_write_freeze_path.is_absolute():
|
|
raise ValueError("upload write-freeze path must be absolute")
|
|
assert_path_without_reparse(upload_root, "public upload root")
|
|
assert_path_without_reparse(manifest_path, "upload manifest path")
|
|
assert_path_without_reparse(
|
|
expected_write_freeze_path,
|
|
"upload write-freeze path",
|
|
)
|
|
root = upload_root.resolve(strict=True)
|
|
manifest = manifest_path.resolve(strict=True)
|
|
assert_regular_file_without_reparse(manifest, "upload manifest")
|
|
_assert_outside_root(manifest, root, "upload manifest")
|
|
_assert_outside_root(
|
|
expected_write_freeze_path,
|
|
root,
|
|
"upload write-freeze path",
|
|
)
|
|
try:
|
|
manifest_bytes = manifest.read_bytes()
|
|
except OSError as exc:
|
|
raise ValueError("upload manifest could not be read") from exc
|
|
actual_manifest_sha256 = _sha256_bytes(manifest_bytes)
|
|
if not _is_lower_hex_sha256(expected_manifest_sha256):
|
|
raise ValueError("expected upload manifest SHA256 is invalid")
|
|
if actual_manifest_sha256 != expected_manifest_sha256:
|
|
raise ValueError("upload manifest SHA256 drift")
|
|
try:
|
|
payload = json.loads(manifest_bytes.decode("utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise ValueError("upload manifest is not valid UTF-8 JSON") from exc
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("upload manifest must be a JSON object")
|
|
forbidden_keys = {
|
|
"relative_path",
|
|
"source_path",
|
|
"avatar_url",
|
|
"user_id",
|
|
"email",
|
|
"filename",
|
|
}
|
|
if forbidden_keys.intersection(key.casefold() for key in _iter_keys(payload)):
|
|
raise ValueError("upload manifest contains a privacy-sensitive key")
|
|
if payload.get("schema_version") != MANIFEST_SCHEMA_VERSION:
|
|
raise ValueError("upload manifest schema version mismatch")
|
|
if payload.get("status") != "initialized_complete":
|
|
raise ValueError("upload manifest is not initialized and complete")
|
|
root_sha256 = canonical_path_sha256(root)
|
|
if payload.get("root_path_sha256") != root_sha256:
|
|
raise ValueError("upload manifest root binding mismatch")
|
|
if (
|
|
not _is_lower_hex_sha256(expected_database_target_sha256)
|
|
or payload.get("database_target_sha256")
|
|
!= expected_database_target_sha256
|
|
):
|
|
raise ValueError("upload manifest database target binding mismatch")
|
|
if "objects" in payload:
|
|
raise ValueError("legacy upload manifest object contract is rejected")
|
|
preserved_objects = payload.get("preserved_objects")
|
|
if not isinstance(preserved_objects, list):
|
|
raise ValueError("upload manifest preserved objects must be an array")
|
|
preserved_count = payload.get("preserved_object_count")
|
|
if (
|
|
not isinstance(preserved_count, int)
|
|
or preserved_count <= 0
|
|
or preserved_count != len(preserved_objects)
|
|
):
|
|
raise ValueError("upload manifest preserved object count mismatch")
|
|
preserved_total_size_bytes = payload.get("preserved_total_size_bytes")
|
|
if (
|
|
not isinstance(preserved_total_size_bytes, int)
|
|
or isinstance(preserved_total_size_bytes, bool)
|
|
or preserved_total_size_bytes < 0
|
|
):
|
|
raise ValueError("upload manifest preserved total size is invalid")
|
|
expected_decode_contract = {
|
|
"schema_version": IMAGE_DECODE_CONTRACT_VERSION,
|
|
"decoder": "Pillow",
|
|
"decoder_version": PILLOW_VERSION,
|
|
"mode": "verify_load_extension_match",
|
|
}
|
|
if payload.get("image_decode_contract") != expected_decode_contract:
|
|
raise ValueError("upload manifest image decode contract mismatch")
|
|
decode_count_names = (
|
|
"preserved_decode_valid_count",
|
|
"preserved_decode_invalid_count",
|
|
"required_decode_invalid_object_count",
|
|
"required_decode_invalid_reference_count",
|
|
)
|
|
decode_counts: dict[str, int] = {}
|
|
for name in decode_count_names:
|
|
value = payload.get(name)
|
|
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
|
raise ValueError("upload manifest image decode count is invalid")
|
|
decode_counts[name] = value
|
|
required_count = payload.get("required_object_count")
|
|
if (
|
|
not isinstance(required_count, int)
|
|
or required_count < 0
|
|
or required_count > preserved_count
|
|
):
|
|
raise ValueError("upload manifest database object count mismatch")
|
|
required_reference_count = payload.get("required_reference_count")
|
|
if (
|
|
not isinstance(required_reference_count, int)
|
|
or required_reference_count < 0
|
|
or required_reference_count < required_count
|
|
):
|
|
raise ValueError("upload manifest reference count is invalid")
|
|
freeze = payload.get("write_freeze")
|
|
if not isinstance(freeze, dict):
|
|
raise ValueError("upload manifest write-freeze binding is missing")
|
|
token_sha256 = freeze.get("token_sha256")
|
|
if not _is_lower_hex_sha256(token_sha256):
|
|
raise ValueError("upload manifest write-freeze token binding is invalid")
|
|
if require_freeze_path_binding:
|
|
if freeze.get("path_sha256") != canonical_path_sha256(
|
|
expected_write_freeze_path
|
|
):
|
|
raise ValueError("upload manifest write-freeze path binding mismatch")
|
|
|
|
available: dict[str, tuple[str, Path]] = {}
|
|
avatar_root = root / PUBLIC_AVATAR_DIRECTORY
|
|
if verify_preserved_objects or reject_unbound_extras:
|
|
if not avatar_root.is_dir() or _is_reparse_or_symlink(avatar_root):
|
|
raise ValueError("public avatar directory must be a non-reparse directory")
|
|
for candidate in avatar_root.iterdir():
|
|
if _is_reparse_or_symlink(candidate) or not candidate.is_file():
|
|
raise ValueError("public avatar directory contains an unsafe extra")
|
|
try:
|
|
relative = normalize_public_avatar_relative_path(
|
|
f"{PUBLIC_AVATAR_DIRECTORY}/{candidate.name}"
|
|
)
|
|
except ValueError as exc:
|
|
raise ValueError(
|
|
"public avatar directory contains an invalid extra"
|
|
) from exc
|
|
path_digest = public_avatar_relative_path_sha256(relative)
|
|
if path_digest in available:
|
|
raise ValueError("public avatar directory contains a duplicate path")
|
|
available[path_digest] = (relative, candidate)
|
|
|
|
normalized_records: list[dict[str, object]] = []
|
|
preserved_cache: list[tuple[str, str, int, str, bytes]] = []
|
|
preserved_cache_bytes = 0
|
|
seen: set[str] = set()
|
|
for item in preserved_objects:
|
|
if not isinstance(item, dict):
|
|
raise ValueError("upload manifest object entry must be an object")
|
|
path_sha256 = item.get("path_sha256")
|
|
content_sha256 = item.get("content_sha256")
|
|
size_bytes = item.get("size_bytes")
|
|
reference_count = item.get("reference_count")
|
|
decode_valid = item.get("decode_valid")
|
|
if not _is_lower_hex_sha256(path_sha256) or path_sha256 in seen:
|
|
raise ValueError("upload manifest path hash is invalid or duplicated")
|
|
if not _is_lower_hex_sha256(content_sha256):
|
|
raise ValueError("upload manifest content hash is invalid")
|
|
if not isinstance(size_bytes, int) or size_bytes < 0:
|
|
raise ValueError("upload manifest object size is invalid")
|
|
if not isinstance(reference_count, int) or reference_count < 0:
|
|
raise ValueError("upload manifest object reference count is invalid")
|
|
if not isinstance(decode_valid, bool):
|
|
raise ValueError("upload manifest object decode status is invalid")
|
|
if item.get("kind") != "profile_avatar":
|
|
raise ValueError("upload manifest contains a non-public object kind")
|
|
if verify_preserved_objects:
|
|
available_record = available.get(path_sha256)
|
|
if available_record is None:
|
|
raise ValueError("preserved public avatar object is missing")
|
|
_relative_path, target = available_record
|
|
assert_regular_file_without_reparse(
|
|
target,
|
|
"preserved public avatar object",
|
|
)
|
|
content = _read_regular_file_bytes(
|
|
target,
|
|
"preserved public avatar object",
|
|
)
|
|
if len(content) != size_bytes or _sha256_bytes(content) != content_sha256:
|
|
raise ValueError(
|
|
"preserved public avatar object content hash or size drift"
|
|
)
|
|
actual_decode_valid = inspect_public_avatar_image(
|
|
_relative_path,
|
|
content,
|
|
).valid
|
|
if actual_decode_valid is not decode_valid:
|
|
raise ValueError("preserved public avatar decode status drift")
|
|
if decode_valid:
|
|
preserved_cache_bytes += len(content)
|
|
if preserved_cache_bytes > MAX_PRESERVED_AVATAR_CACHE_BYTES:
|
|
raise ValueError(
|
|
"preserved public avatar cache size limit exceeded"
|
|
)
|
|
preserved_cache.append(
|
|
(
|
|
str(path_sha256),
|
|
str(content_sha256),
|
|
int(size_bytes),
|
|
_public_avatar_media_type(_relative_path),
|
|
content,
|
|
)
|
|
)
|
|
seen.add(path_sha256)
|
|
normalized_records.append(
|
|
{
|
|
"path_sha256": path_sha256,
|
|
"content_sha256": content_sha256,
|
|
"size_bytes": size_bytes,
|
|
"kind": "profile_avatar",
|
|
"reference_count": reference_count,
|
|
"decode_valid": decode_valid,
|
|
}
|
|
)
|
|
normalized_records.sort(key=lambda item: str(item["path_sha256"]))
|
|
if sum(int(record["size_bytes"]) for record in normalized_records) != (
|
|
preserved_total_size_bytes
|
|
):
|
|
raise ValueError("upload manifest preserved total size mismatch")
|
|
preserved_decode_valid_count = sum(
|
|
1 for record in normalized_records if bool(record["decode_valid"])
|
|
)
|
|
preserved_decode_invalid_count = len(normalized_records) - (
|
|
preserved_decode_valid_count
|
|
)
|
|
required_decode_invalid_records = [
|
|
record
|
|
for record in normalized_records
|
|
if int(record["reference_count"]) > 0
|
|
and not bool(record["decode_valid"])
|
|
]
|
|
if (
|
|
decode_counts["preserved_decode_valid_count"]
|
|
!= preserved_decode_valid_count
|
|
or decode_counts["preserved_decode_invalid_count"]
|
|
!= preserved_decode_invalid_count
|
|
or decode_counts["required_decode_invalid_object_count"]
|
|
!= len(required_decode_invalid_records)
|
|
or decode_counts["required_decode_invalid_reference_count"]
|
|
!= sum(
|
|
int(record["reference_count"])
|
|
for record in required_decode_invalid_records
|
|
)
|
|
):
|
|
raise ValueError("upload manifest image decode count mismatch")
|
|
unbound = set(available).difference(seen)
|
|
if reject_unbound_extras and unbound:
|
|
raise ValueError("public avatar directory contains an unbound extra")
|
|
if any(
|
|
not is_runtime_generated_public_avatar_relative_path(available[digest][0])
|
|
for digest in unbound
|
|
):
|
|
raise ValueError("public avatar directory contains an unauthorized extra")
|
|
preserved_set_sha256 = _preserved_object_set_sha256(normalized_records)
|
|
if payload.get("preserved_object_set_sha256") != preserved_set_sha256:
|
|
raise ValueError("upload manifest preserved-object set hash mismatch")
|
|
referenced_records = [
|
|
record
|
|
for record in normalized_records
|
|
if int(record["reference_count"]) > 0
|
|
]
|
|
reference_records = _reference_records_from_objects(referenced_records)
|
|
reference_set_sha256 = _reference_set_sha256(reference_records)
|
|
if payload.get("reference_set_sha256") != reference_set_sha256:
|
|
raise ValueError("upload manifest reference-set hash mismatch")
|
|
if (
|
|
len(reference_records) != required_count
|
|
or sum(int(record["reference_count"]) for record in referenced_records)
|
|
!= required_reference_count
|
|
):
|
|
raise ValueError("upload manifest database reference count mismatch")
|
|
database_contract = payload.get("database_reference_contract")
|
|
if not isinstance(database_contract, dict):
|
|
raise ValueError("upload manifest database reference contract is invalid")
|
|
if (
|
|
database_contract.get("reference_count") != required_reference_count
|
|
or database_contract.get("unique_object_count") != required_count
|
|
or database_contract.get("reference_set_sha256") != reference_set_sha256
|
|
or database_contract.get("objects") != reference_records
|
|
):
|
|
raise ValueError("upload manifest database reference contract drift")
|
|
return UploadManifestProof(
|
|
manifest_sha256=actual_manifest_sha256,
|
|
root_path_sha256=root_sha256,
|
|
database_target_sha256=expected_database_target_sha256,
|
|
preserved_object_count=preserved_count,
|
|
preserved_total_size_bytes=preserved_total_size_bytes,
|
|
preserved_decode_valid_count=preserved_decode_valid_count,
|
|
preserved_decode_invalid_count=preserved_decode_invalid_count,
|
|
required_decode_invalid_object_count=len(
|
|
required_decode_invalid_records
|
|
),
|
|
required_decode_invalid_reference_count=sum(
|
|
int(record["reference_count"])
|
|
for record in required_decode_invalid_records
|
|
),
|
|
preserved_object_set_sha256=preserved_set_sha256,
|
|
required_object_count=required_count,
|
|
required_reference_count=required_reference_count,
|
|
reference_set_sha256=reference_set_sha256,
|
|
write_freeze_token_sha256=token_sha256,
|
|
preserved_records=tuple(preserved_cache),
|
|
preserved_path_decode_records=tuple(
|
|
(str(record["path_sha256"]), bool(record["decode_valid"]))
|
|
for record in normalized_records
|
|
),
|
|
reference_records=tuple(
|
|
(str(record["path_sha256"]), int(record["reference_count"]))
|
|
for record in reference_records
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CurrentAvatarReferenceProof:
|
|
object_count: int
|
|
reference_count: int
|
|
reference_set_sha256: str
|
|
decode_invalid_object_count: int
|
|
decode_invalid_reference_count: int
|
|
|
|
|
|
def validate_current_avatar_references(
|
|
*,
|
|
upload_root: Path,
|
|
avatar_urls: Sequence[str],
|
|
initial_manifest: UploadManifestProof,
|
|
expected_database_target_sha256: str,
|
|
) -> CurrentAvatarReferenceProof:
|
|
if (
|
|
not _is_lower_hex_sha256(expected_database_target_sha256)
|
|
or initial_manifest.database_target_sha256
|
|
!= expected_database_target_sha256
|
|
):
|
|
raise ValueError("current avatar inventory database target is unverified")
|
|
references = [
|
|
(public_avatar_url_to_relative_path(value), "profile_avatar")
|
|
for value in avatar_urls
|
|
if isinstance(value, str) and value.strip()
|
|
]
|
|
aggregated = _aggregate_references(references)
|
|
if initial_manifest.required_reference_count > 0 and not aggregated:
|
|
raise ValueError(
|
|
"current public avatar inventory is empty without an explicit reset receipt"
|
|
)
|
|
assert_path_without_reparse(upload_root, "public upload root")
|
|
root = upload_root.resolve(strict=True)
|
|
records: list[dict[str, object]] = []
|
|
preserved_decode_status = dict(initial_manifest.preserved_path_decode_records)
|
|
decode_invalid_object_count = 0
|
|
decode_invalid_reference_count = 0
|
|
for relative_path, _kind, reference_count in aggregated:
|
|
target = root / Path(*PurePosixPath(relative_path).parts)
|
|
path_sha256 = public_avatar_relative_path_sha256(relative_path)
|
|
preserved_decode_valid = preserved_decode_status.get(path_sha256)
|
|
if preserved_decode_valid is None:
|
|
if not is_runtime_generated_public_avatar_relative_path(relative_path):
|
|
raise ValueError(
|
|
"current DB-referenced avatar object is not authorized"
|
|
)
|
|
try:
|
|
read_decodable_public_avatar_object(
|
|
relative_path=relative_path,
|
|
path=target,
|
|
label="current DB-referenced avatar object",
|
|
)
|
|
except PublicAvatarImageInvalid:
|
|
decode_invalid_object_count += 1
|
|
decode_invalid_reference_count += reference_count
|
|
else:
|
|
sha256_file(target)
|
|
if preserved_decode_valid is False:
|
|
decode_invalid_object_count += 1
|
|
decode_invalid_reference_count += reference_count
|
|
records.append(
|
|
{"path_sha256": path_sha256, "reference_count": reference_count}
|
|
)
|
|
records.sort(key=lambda item: str(item["path_sha256"]))
|
|
return CurrentAvatarReferenceProof(
|
|
object_count=len(records),
|
|
reference_count=sum(int(item["reference_count"]) for item in records),
|
|
reference_set_sha256=_reference_set_sha256(records),
|
|
decode_invalid_object_count=decode_invalid_object_count,
|
|
decode_invalid_reference_count=decode_invalid_reference_count,
|
|
)
|
|
|
|
|
|
class UploadWriteFrozen(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class UploadWriteFreezeStatus:
|
|
capable: bool
|
|
active: bool
|
|
valid: bool
|
|
in_flight: int
|
|
path_sha256: str | None
|
|
token_sha256: str | None
|
|
|
|
|
|
class UploadWriteFreezeGate:
|
|
def __init__(self, freeze_path: Path | None):
|
|
self._freeze_path = freeze_path
|
|
self._lock = threading.Lock()
|
|
self._in_flight = 0
|
|
|
|
def status(self) -> UploadWriteFreezeStatus:
|
|
path = self._freeze_path
|
|
with self._lock:
|
|
in_flight = self._in_flight
|
|
if path is None:
|
|
return UploadWriteFreezeStatus(
|
|
False, False, False, in_flight, None, None
|
|
)
|
|
path_sha256 = canonical_path_sha256(path)
|
|
try:
|
|
assert_path_without_reparse(path, "upload write-freeze path")
|
|
except ValueError:
|
|
return UploadWriteFreezeStatus(
|
|
True, True, False, in_flight, path_sha256, None
|
|
)
|
|
if not _path_entry_exists_without_following(path):
|
|
return UploadWriteFreezeStatus(
|
|
True, False, True, in_flight, path_sha256, None
|
|
)
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
token = payload.get("token") if isinstance(payload, dict) else None
|
|
valid = (
|
|
isinstance(payload, dict)
|
|
and payload.get("schema_version") == WRITE_FREEZE_SCHEMA_VERSION
|
|
and isinstance(token, str)
|
|
and len(token) >= 32
|
|
)
|
|
token_sha256 = _sha256_text(token) if valid else None
|
|
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
valid = False
|
|
token_sha256 = None
|
|
return UploadWriteFreezeStatus(
|
|
True,
|
|
True,
|
|
valid,
|
|
in_flight,
|
|
path_sha256,
|
|
token_sha256,
|
|
)
|
|
|
|
@contextmanager
|
|
def write_lease(self) -> Iterator[None]:
|
|
with self._lock:
|
|
if self._freeze_path is not None and _path_entry_exists_without_following(
|
|
self._freeze_path
|
|
):
|
|
raise UploadWriteFrozen("public upload writes are temporarily frozen")
|
|
self._in_flight += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
with self._lock:
|
|
self._in_flight -= 1
|