16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
935 lines
32 KiB
Python
935 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""Capture bounded, metadata-only G7 Linux/NAS topology evidence.
|
|
|
|
The sampler pins two running Docker Compose containers (API and Caddy) to
|
|
their exact project/service labels, container IDs, image IDs, init PIDs,
|
|
start timestamps, and restart counts. Every sample validates those pins both
|
|
before and after collection so a restart or image replacement cannot be
|
|
reported as one continuous run.
|
|
|
|
Only infrastructure counters are retained. Socket endpoints, request data,
|
|
audio, transcripts, command output, and Cloudflare-internal queue data are
|
|
never written to evidence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from dataclasses import asdict, dataclass
|
|
from datetime import UTC, datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Protocol, Sequence
|
|
|
|
|
|
SCHEMA_VERSION = "vignette.g7-topology-evidence.v1"
|
|
DEFAULT_INTERVAL_SECONDS = 1.0
|
|
DEFAULT_SAMPLES = 10
|
|
MAX_SAMPLES = 7_200
|
|
MAX_INTERVAL_SECONDS = 60.0
|
|
MAX_CAPTURE_WINDOW_SECONDS = 7_200.0
|
|
_SHA256_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
_BYTE_VALUE_PATTERN = re.compile(r"^([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)$")
|
|
_RETRANSMIT_PATTERN = re.compile(r"(?:^|\s)retrans:(\d+)(?:/(\d+))?(?:\s|$)")
|
|
_BYTE_MULTIPLIERS = {
|
|
"B": Decimal(1),
|
|
"KB": Decimal(1_000),
|
|
"MB": Decimal(1_000_000),
|
|
"GB": Decimal(1_000_000_000),
|
|
"TB": Decimal(1_000_000_000_000),
|
|
"KIB": Decimal(1_024),
|
|
"MIB": Decimal(1_048_576),
|
|
"GIB": Decimal(1_073_741_824),
|
|
"TIB": Decimal(1_099_511_627_776),
|
|
}
|
|
|
|
|
|
class EvidenceFailure(RuntimeError):
|
|
"""A required evidence source or pinned runtime contract failed."""
|
|
|
|
|
|
class CommandRunner(Protocol):
|
|
def available(self, command: str) -> bool: ...
|
|
|
|
def run(self, args: Sequence[str]) -> str: ...
|
|
|
|
|
|
class FileSource(Protocol):
|
|
def read_text(self, path: str) -> str: ...
|
|
|
|
def list_names(self, path: str) -> list[str]: ...
|
|
|
|
|
|
class SubprocessCommandRunner:
|
|
def available(self, command: str) -> bool:
|
|
return shutil.which(command) is not None
|
|
|
|
def run(self, args: Sequence[str]) -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
list(args),
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=30,
|
|
shell=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise EvidenceFailure(f"command_unavailable:{args[0]}") from exc
|
|
if completed.returncode != 0:
|
|
raise EvidenceFailure(f"command_failed:{args[0]}")
|
|
return completed.stdout
|
|
|
|
|
|
class LocalFileSource:
|
|
def read_text(self, path: str) -> str:
|
|
try:
|
|
return Path(path).read_text(encoding="utf-8")
|
|
except (OSError, UnicodeError) as exc:
|
|
raise EvidenceFailure("required_linux_source_unreadable") from exc
|
|
|
|
def list_names(self, path: str) -> list[str]:
|
|
try:
|
|
return os.listdir(path)
|
|
except OSError as exc:
|
|
raise EvidenceFailure("required_linux_source_unreadable") from exc
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CaptureConfig:
|
|
compose_project: str
|
|
public_host: str
|
|
api_container: str
|
|
api_service: str
|
|
api_image_digest: str
|
|
caddy_container: str
|
|
caddy_service: str
|
|
caddy_image_digest: str
|
|
samples: int = DEFAULT_SAMPLES
|
|
interval_seconds: float = DEFAULT_INTERVAL_SECONDS
|
|
proc_root: str = "/proc"
|
|
cgroup_root: str = "/sys/fs/cgroup"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ContainerIdentity:
|
|
role: str
|
|
compose_project: str
|
|
compose_service: str
|
|
container_id: str
|
|
container_name: str
|
|
image_digest: str
|
|
init_pid: int
|
|
started_at: str
|
|
restart_count: int
|
|
cgroup_path: str
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _clean_identifier(value: object, *, code: str) -> str:
|
|
if not isinstance(value, str):
|
|
raise EvidenceFailure(code)
|
|
cleaned = value.strip()
|
|
if not cleaned or len(cleaned) > 256 or any(ord(char) < 32 for char in cleaned):
|
|
raise EvidenceFailure(code)
|
|
return cleaned
|
|
|
|
|
|
def _parse_nonnegative_int(value: object, *, code: str) -> int:
|
|
if isinstance(value, bool):
|
|
raise EvidenceFailure(code)
|
|
try:
|
|
parsed = int(value)
|
|
except (TypeError, ValueError) as exc:
|
|
raise EvidenceFailure(code) from exc
|
|
if parsed < 0:
|
|
raise EvidenceFailure(code)
|
|
return parsed
|
|
|
|
|
|
def validate_config(config: CaptureConfig) -> None:
|
|
for value, code in (
|
|
(config.compose_project, "compose_project_invalid"),
|
|
(config.public_host, "public_host_invalid"),
|
|
(config.api_container, "api_container_invalid"),
|
|
(config.api_service, "api_service_invalid"),
|
|
(config.caddy_container, "caddy_container_invalid"),
|
|
(config.caddy_service, "caddy_service_invalid"),
|
|
):
|
|
_clean_identifier(value, code=code)
|
|
if config.api_service == config.caddy_service:
|
|
raise EvidenceFailure("compose_services_not_distinct")
|
|
for digest, code in (
|
|
(config.api_image_digest, "api_image_digest_invalid"),
|
|
(config.caddy_image_digest, "caddy_image_digest_invalid"),
|
|
):
|
|
if not _SHA256_PATTERN.fullmatch(digest.lower()):
|
|
raise EvidenceFailure(code)
|
|
if not 1 <= config.samples <= MAX_SAMPLES:
|
|
raise EvidenceFailure("sample_count_out_of_bounds")
|
|
if not math.isfinite(config.interval_seconds) or not (
|
|
0.05 <= config.interval_seconds <= MAX_INTERVAL_SECONDS
|
|
):
|
|
raise EvidenceFailure("sample_interval_out_of_bounds")
|
|
if (config.samples - 1) * config.interval_seconds > MAX_CAPTURE_WINDOW_SECONDS:
|
|
raise EvidenceFailure("capture_window_out_of_bounds")
|
|
for root, code in (
|
|
(config.proc_root, "proc_root_invalid"),
|
|
(config.cgroup_root, "cgroup_root_invalid"),
|
|
):
|
|
pure = PurePosixPath(root)
|
|
if not pure.is_absolute() or ".." in pure.parts:
|
|
raise EvidenceFailure(code)
|
|
|
|
|
|
def parse_container_inspect(stdout: str) -> list[dict[str, object]]:
|
|
try:
|
|
payload = json.loads(stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise EvidenceFailure("docker_inspect_invalid_json") from exc
|
|
if not isinstance(payload, list) or not payload:
|
|
raise EvidenceFailure("docker_inspect_empty")
|
|
if not all(isinstance(item, dict) for item in payload):
|
|
raise EvidenceFailure("docker_inspect_invalid_shape")
|
|
return payload
|
|
|
|
|
|
def _identity_from_inspect(
|
|
item: dict[str, object],
|
|
*,
|
|
role: str,
|
|
expected_project: str,
|
|
expected_service: str,
|
|
expected_digest: str,
|
|
cgroup_path: str,
|
|
) -> ContainerIdentity:
|
|
config = item.get("Config")
|
|
state = item.get("State")
|
|
if not isinstance(config, dict) or not isinstance(state, dict):
|
|
raise EvidenceFailure(f"docker_inspect_fields_missing:{role}")
|
|
labels = config.get("Labels")
|
|
if not isinstance(labels, dict):
|
|
raise EvidenceFailure(f"compose_labels_missing:{role}")
|
|
project = labels.get("com.docker.compose.project")
|
|
service = labels.get("com.docker.compose.service")
|
|
if project != expected_project:
|
|
raise EvidenceFailure(f"compose_project_mismatch:{role}")
|
|
if service != expected_service:
|
|
raise EvidenceFailure(f"compose_service_mismatch:{role}")
|
|
container_id = _clean_identifier(
|
|
item.get("Id"), code=f"container_id_missing:{role}"
|
|
).lower()
|
|
if not re.fullmatch(r"[0-9a-f]{64}", container_id):
|
|
raise EvidenceFailure(f"container_id_invalid:{role}")
|
|
image_digest = _clean_identifier(
|
|
item.get("Image"), code=f"image_digest_missing:{role}"
|
|
).lower()
|
|
if image_digest != expected_digest.lower():
|
|
raise EvidenceFailure(f"image_digest_mismatch:{role}")
|
|
if state.get("Running") is not True:
|
|
raise EvidenceFailure(f"container_not_running:{role}")
|
|
init_pid = _parse_nonnegative_int(state.get("Pid"), code=f"init_pid_missing:{role}")
|
|
if init_pid <= 0:
|
|
raise EvidenceFailure(f"init_pid_invalid:{role}")
|
|
started_at = _clean_identifier(
|
|
state.get("StartedAt"), code=f"started_at_missing:{role}"
|
|
)
|
|
restart_count = _parse_nonnegative_int(
|
|
item.get("RestartCount"), code=f"restart_count_missing:{role}"
|
|
)
|
|
container_name = _clean_identifier(
|
|
item.get("Name"), code=f"container_name_missing:{role}"
|
|
).lstrip("/")
|
|
if not container_name:
|
|
raise EvidenceFailure(f"container_name_missing:{role}")
|
|
return ContainerIdentity(
|
|
role=role,
|
|
compose_project=expected_project,
|
|
compose_service=expected_service,
|
|
container_id=container_id,
|
|
container_name=container_name,
|
|
image_digest=image_digest,
|
|
init_pid=init_pid,
|
|
started_at=started_at,
|
|
restart_count=restart_count,
|
|
cgroup_path=cgroup_path,
|
|
)
|
|
|
|
|
|
def _posix_join(root: str, *parts: str) -> str:
|
|
result = PurePosixPath(root)
|
|
for part in parts:
|
|
result /= part
|
|
return str(result)
|
|
|
|
|
|
def read_cgroup_path(source: FileSource, *, proc_root: str, pid: int, role: str) -> str:
|
|
path = _posix_join(proc_root, str(pid), "cgroup")
|
|
try:
|
|
content = source.read_text(path)
|
|
except Exception as exc:
|
|
if isinstance(exc, EvidenceFailure):
|
|
raise
|
|
raise EvidenceFailure(f"proc_cgroup_missing:{role}") from exc
|
|
unified_paths = []
|
|
for line in content.splitlines():
|
|
parts = line.split(":", 2)
|
|
if len(parts) == 3 and parts[0] == "0" and parts[1] == "":
|
|
unified_paths.append(parts[2].strip())
|
|
if len(unified_paths) != 1:
|
|
raise EvidenceFailure(f"cgroup_v2_path_missing:{role}")
|
|
cgroup_path = PurePosixPath(unified_paths[0])
|
|
if not cgroup_path.is_absolute() or ".." in cgroup_path.parts:
|
|
raise EvidenceFailure(f"cgroup_v2_path_invalid:{role}")
|
|
return str(cgroup_path)
|
|
|
|
|
|
def _inspect(
|
|
runner: CommandRunner,
|
|
selectors: Sequence[str],
|
|
) -> list[dict[str, object]]:
|
|
stdout = runner.run(["docker", "inspect", "--type", "container", *selectors])
|
|
items = parse_container_inspect(stdout)
|
|
if len(items) != len(selectors):
|
|
raise EvidenceFailure("docker_inspect_target_count_mismatch")
|
|
return items
|
|
|
|
|
|
def pin_targets(
|
|
config: CaptureConfig,
|
|
runner: CommandRunner,
|
|
source: FileSource,
|
|
) -> dict[str, ContainerIdentity]:
|
|
items = _inspect(runner, [config.api_container, config.caddy_container])
|
|
by_service: dict[str, dict[str, object]] = {}
|
|
for item in items:
|
|
item_config = item.get("Config")
|
|
labels = item_config.get("Labels") if isinstance(item_config, dict) else None
|
|
service = (
|
|
labels.get("com.docker.compose.service")
|
|
if isinstance(labels, dict)
|
|
else None
|
|
)
|
|
if isinstance(service, str):
|
|
if service in by_service:
|
|
raise EvidenceFailure("docker_inspect_duplicate_service")
|
|
by_service[service] = item
|
|
|
|
targets: dict[str, ContainerIdentity] = {}
|
|
for role, service, digest in (
|
|
("api", config.api_service, config.api_image_digest),
|
|
("caddy", config.caddy_service, config.caddy_image_digest),
|
|
):
|
|
item = by_service.get(service)
|
|
if item is None:
|
|
raise EvidenceFailure(f"compose_service_missing:{role}")
|
|
state = item.get("State")
|
|
if not isinstance(state, dict):
|
|
raise EvidenceFailure(f"docker_inspect_fields_missing:{role}")
|
|
pid = _parse_nonnegative_int(state.get("Pid"), code=f"init_pid_missing:{role}")
|
|
cgroup_path = read_cgroup_path(
|
|
source,
|
|
proc_root=config.proc_root,
|
|
pid=pid,
|
|
role=role,
|
|
)
|
|
targets[role] = _identity_from_inspect(
|
|
item,
|
|
role=role,
|
|
expected_project=config.compose_project,
|
|
expected_service=service,
|
|
expected_digest=digest,
|
|
cgroup_path=cgroup_path,
|
|
)
|
|
if targets["api"].container_id == targets["caddy"].container_id:
|
|
raise EvidenceFailure("container_targets_not_distinct")
|
|
return targets
|
|
|
|
|
|
def validate_pins(
|
|
runner: CommandRunner,
|
|
source: FileSource,
|
|
config: CaptureConfig,
|
|
targets: dict[str, ContainerIdentity],
|
|
) -> None:
|
|
ordered = [targets["api"], targets["caddy"]]
|
|
items = _inspect(runner, [target.container_id for target in ordered])
|
|
by_id = {
|
|
str(item.get("Id") or "").lower(): item
|
|
for item in items
|
|
if isinstance(item, dict)
|
|
}
|
|
for baseline in ordered:
|
|
item = by_id.get(baseline.container_id)
|
|
if item is None:
|
|
raise EvidenceFailure(f"container_id_drift:{baseline.role}")
|
|
current_image_digest = str(item.get("Image") or "").strip().lower()
|
|
if current_image_digest != baseline.image_digest:
|
|
raise EvidenceFailure(f"image_digest_drift:{baseline.role}")
|
|
current = _identity_from_inspect(
|
|
item,
|
|
role=baseline.role,
|
|
expected_project=baseline.compose_project,
|
|
expected_service=baseline.compose_service,
|
|
expected_digest=baseline.image_digest,
|
|
cgroup_path=baseline.cgroup_path,
|
|
)
|
|
if current.container_name != baseline.container_name:
|
|
raise EvidenceFailure(f"container_name_drift:{baseline.role}")
|
|
if current.init_pid != baseline.init_pid:
|
|
raise EvidenceFailure(f"init_pid_drift:{baseline.role}")
|
|
if (
|
|
current.started_at != baseline.started_at
|
|
or current.restart_count != baseline.restart_count
|
|
):
|
|
raise EvidenceFailure(f"container_restart_drift:{baseline.role}")
|
|
current_cgroup_path = read_cgroup_path(
|
|
source,
|
|
proc_root=config.proc_root,
|
|
pid=current.init_pid,
|
|
role=current.role,
|
|
)
|
|
if current_cgroup_path != baseline.cgroup_path:
|
|
raise EvidenceFailure(f"cgroup_path_drift:{baseline.role}")
|
|
|
|
|
|
def _read_source(source: FileSource, path: str, *, code: str) -> str:
|
|
try:
|
|
value = source.read_text(path)
|
|
except Exception as exc:
|
|
if isinstance(exc, EvidenceFailure):
|
|
raise
|
|
raise EvidenceFailure(code) from exc
|
|
if not value.strip():
|
|
raise EvidenceFailure(code)
|
|
return value
|
|
|
|
|
|
def parse_cpu_stat(content: str, *, role: str) -> dict[str, int]:
|
|
parsed: dict[str, int] = {}
|
|
for line in content.splitlines():
|
|
parts = line.split()
|
|
if len(parts) != 2:
|
|
raise EvidenceFailure(f"cgroup_cpu_stat_invalid:{role}")
|
|
key, raw_value = parts
|
|
if key in parsed:
|
|
raise EvidenceFailure(f"cgroup_cpu_stat_duplicate:{role}")
|
|
parsed[key] = _parse_nonnegative_int(
|
|
raw_value, code=f"cgroup_cpu_stat_invalid:{role}"
|
|
)
|
|
required = {"usage_usec", "user_usec", "system_usec"}
|
|
if not required.issubset(parsed):
|
|
raise EvidenceFailure(f"cgroup_cpu_stat_fields_missing:{role}")
|
|
return dict(sorted(parsed.items()))
|
|
|
|
|
|
def read_cgroup_metrics(
|
|
source: FileSource,
|
|
*,
|
|
cgroup_root: str,
|
|
target: ContainerIdentity,
|
|
) -> dict[str, object]:
|
|
base = _posix_join(cgroup_root, target.cgroup_path.lstrip("/"))
|
|
|
|
def read_integer(filename: str) -> int:
|
|
content = _read_source(
|
|
source,
|
|
_posix_join(base, filename),
|
|
code=f"cgroup_source_missing:{target.role}:{filename}",
|
|
)
|
|
return _parse_nonnegative_int(
|
|
content.strip(), code=f"cgroup_source_invalid:{target.role}:{filename}"
|
|
)
|
|
|
|
cpu_stat = parse_cpu_stat(
|
|
_read_source(
|
|
source,
|
|
_posix_join(base, "cpu.stat"),
|
|
code=f"cgroup_source_missing:{target.role}:cpu.stat",
|
|
),
|
|
role=target.role,
|
|
)
|
|
return {
|
|
"memory_current_bytes": read_integer("memory.current"),
|
|
"memory_peak_bytes": read_integer("memory.peak"),
|
|
"cpu_stat": cpu_stat,
|
|
"pids_current": read_integer("pids.current"),
|
|
}
|
|
|
|
|
|
def parse_proc_status(content: str, *, role: str) -> dict[str, int]:
|
|
values: dict[str, int] = {}
|
|
for line in content.splitlines():
|
|
if ":" not in line:
|
|
continue
|
|
key, raw = line.split(":", 1)
|
|
if key in {"VmRSS", "VmHWM"}:
|
|
parts = raw.split()
|
|
if len(parts) != 2 or parts[1] != "kB":
|
|
raise EvidenceFailure(f"proc_status_field_invalid:{role}:{key}")
|
|
values[f"{key}_bytes"] = (
|
|
_parse_nonnegative_int(
|
|
parts[0], code=f"proc_status_field_invalid:{role}:{key}"
|
|
)
|
|
* 1_024
|
|
)
|
|
elif key == "Threads":
|
|
values["Threads"] = _parse_nonnegative_int(
|
|
raw.strip(), code=f"proc_status_field_invalid:{role}:Threads"
|
|
)
|
|
required = {"VmRSS_bytes", "VmHWM_bytes", "Threads"}
|
|
if not required.issubset(values):
|
|
raise EvidenceFailure(f"proc_status_fields_missing:{role}")
|
|
return {
|
|
"vm_rss_bytes": values["VmRSS_bytes"],
|
|
"vm_hwm_bytes": values["VmHWM_bytes"],
|
|
"threads": values["Threads"],
|
|
}
|
|
|
|
|
|
def read_proc_metrics(
|
|
source: FileSource,
|
|
*,
|
|
proc_root: str,
|
|
target: ContainerIdentity,
|
|
) -> dict[str, int]:
|
|
status_path = _posix_join(proc_root, str(target.init_pid), "status")
|
|
metrics = parse_proc_status(
|
|
_read_source(
|
|
source,
|
|
status_path,
|
|
code=f"proc_status_missing:{target.role}",
|
|
),
|
|
role=target.role,
|
|
)
|
|
try:
|
|
fd_names = source.list_names(_posix_join(proc_root, str(target.init_pid), "fd"))
|
|
except Exception as exc:
|
|
if isinstance(exc, EvidenceFailure):
|
|
raise
|
|
raise EvidenceFailure(f"proc_fd_missing:{target.role}") from exc
|
|
metrics["fd_count"] = len(fd_names)
|
|
return metrics
|
|
|
|
|
|
def parse_human_bytes(value: str, *, code: str) -> int:
|
|
match = _BYTE_VALUE_PATTERN.fullmatch(value.strip())
|
|
if match is None:
|
|
raise EvidenceFailure(code)
|
|
try:
|
|
number = Decimal(match.group(1))
|
|
except InvalidOperation as exc:
|
|
raise EvidenceFailure(code) from exc
|
|
multiplier = _BYTE_MULTIPLIERS.get(match.group(2).upper())
|
|
if multiplier is None:
|
|
raise EvidenceFailure(code)
|
|
return int(number * multiplier)
|
|
|
|
|
|
def _parse_percent(value: object, *, code: str) -> float:
|
|
if not isinstance(value, str) or not value.strip().endswith("%"):
|
|
raise EvidenceFailure(code)
|
|
try:
|
|
result = float(value.strip()[:-1])
|
|
except ValueError as exc:
|
|
raise EvidenceFailure(code) from exc
|
|
if not math.isfinite(result) or result < 0:
|
|
raise EvidenceFailure(code)
|
|
return result
|
|
|
|
|
|
def _parse_pair(value: object, *, code: str) -> tuple[int, int]:
|
|
if not isinstance(value, str):
|
|
raise EvidenceFailure(code)
|
|
parts = value.split("/")
|
|
if len(parts) != 2:
|
|
raise EvidenceFailure(code)
|
|
return (
|
|
parse_human_bytes(parts[0], code=code),
|
|
parse_human_bytes(parts[1], code=code),
|
|
)
|
|
|
|
|
|
def parse_docker_stats(
|
|
stdout: str,
|
|
targets: dict[str, ContainerIdentity],
|
|
) -> dict[str, dict[str, object]]:
|
|
records: list[dict[str, object]] = []
|
|
for line in stdout.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise EvidenceFailure("docker_stats_invalid_json") from exc
|
|
if not isinstance(record, dict):
|
|
raise EvidenceFailure("docker_stats_invalid_shape")
|
|
records.append(record)
|
|
if len(records) != len(targets):
|
|
raise EvidenceFailure("docker_stats_target_count_mismatch")
|
|
|
|
result: dict[str, dict[str, object]] = {}
|
|
for role, target in targets.items():
|
|
matches = []
|
|
for record in records:
|
|
stats_id = str(record.get("ID") or record.get("Container") or "").lower()
|
|
if stats_id and target.container_id.startswith(stats_id):
|
|
matches.append(record)
|
|
if len(matches) != 1:
|
|
raise EvidenceFailure(f"docker_stats_target_missing:{role}")
|
|
record = matches[0]
|
|
memory_usage, memory_limit = _parse_pair(
|
|
record.get("MemUsage"), code=f"docker_stats_mem_usage_invalid:{role}"
|
|
)
|
|
network_rx, network_tx = _parse_pair(
|
|
record.get("NetIO"), code=f"docker_stats_net_io_invalid:{role}"
|
|
)
|
|
block_read, block_write = _parse_pair(
|
|
record.get("BlockIO"), code=f"docker_stats_block_io_invalid:{role}"
|
|
)
|
|
result[role] = {
|
|
"cpu_percent": _parse_percent(
|
|
record.get("CPUPerc"), code=f"docker_stats_cpu_invalid:{role}"
|
|
),
|
|
"memory_usage_bytes": memory_usage,
|
|
"memory_limit_bytes": memory_limit,
|
|
"memory_percent": _parse_percent(
|
|
record.get("MemPerc"), code=f"docker_stats_mem_percent_invalid:{role}"
|
|
),
|
|
"network_rx_bytes": network_rx,
|
|
"network_tx_bytes": network_tx,
|
|
"block_read_bytes": block_read,
|
|
"block_write_bytes": block_write,
|
|
"pids": _parse_nonnegative_int(
|
|
record.get("PIDs"), code=f"docker_stats_pids_invalid:{role}"
|
|
),
|
|
}
|
|
return result
|
|
|
|
|
|
def parse_ss_tcp(stdout: str) -> dict[str, int]:
|
|
lines = stdout.splitlines()
|
|
if not lines:
|
|
raise EvidenceFailure("ss_output_empty")
|
|
header_index = -1
|
|
recv_index = -1
|
|
send_index = -1
|
|
for index, line in enumerate(lines):
|
|
tokens = line.split()
|
|
if "Recv-Q" in tokens and "Send-Q" in tokens:
|
|
header_index = index
|
|
recv_index = tokens.index("Recv-Q")
|
|
send_index = tokens.index("Send-Q")
|
|
break
|
|
if header_index < 0:
|
|
raise EvidenceFailure("ss_queue_fields_missing")
|
|
|
|
connections = 0
|
|
recv_q_total = 0
|
|
send_q_total = 0
|
|
retransmit_current_total = 0
|
|
retransmit_cumulative_total = 0
|
|
retransmit_fields_observed = 0
|
|
for line in lines[header_index + 1 :]:
|
|
if not line.strip():
|
|
continue
|
|
if not line[0].isspace():
|
|
tokens = line.split()
|
|
if len(tokens) <= max(recv_index, send_index):
|
|
raise EvidenceFailure("ss_connection_fields_missing")
|
|
recv_q_total += _parse_nonnegative_int(
|
|
tokens[recv_index], code="ss_recv_q_invalid"
|
|
)
|
|
send_q_total += _parse_nonnegative_int(
|
|
tokens[send_index], code="ss_send_q_invalid"
|
|
)
|
|
connections += 1
|
|
for match in _RETRANSMIT_PATTERN.finditer(line):
|
|
current = int(match.group(1))
|
|
cumulative = int(match.group(2) or match.group(1))
|
|
retransmit_current_total += current
|
|
retransmit_cumulative_total += cumulative
|
|
retransmit_fields_observed += 1
|
|
return {
|
|
"connections": connections,
|
|
"recv_q_bytes_total": recv_q_total,
|
|
"send_q_bytes_total": send_q_total,
|
|
"retransmit_current_total": retransmit_current_total,
|
|
"retransmit_cumulative_total": retransmit_cumulative_total,
|
|
"retransmit_fields_observed": retransmit_fields_observed,
|
|
}
|
|
|
|
|
|
def collect_sample(
|
|
runner: CommandRunner,
|
|
source: FileSource,
|
|
config: CaptureConfig,
|
|
targets: dict[str, ContainerIdentity],
|
|
*,
|
|
sequence: int,
|
|
) -> dict[str, object]:
|
|
validate_pins(runner, source, config, targets)
|
|
container_metrics: dict[str, dict[str, object]] = {}
|
|
for role, target in targets.items():
|
|
container_metrics[role] = {
|
|
"cgroup_v2": read_cgroup_metrics(
|
|
source,
|
|
cgroup_root=config.cgroup_root,
|
|
target=target,
|
|
),
|
|
"init_process": read_proc_metrics(
|
|
source,
|
|
proc_root=config.proc_root,
|
|
target=target,
|
|
),
|
|
}
|
|
stats_stdout = runner.run(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{json .}}",
|
|
targets["api"].container_id,
|
|
targets["caddy"].container_id,
|
|
]
|
|
)
|
|
stats = parse_docker_stats(stats_stdout, targets)
|
|
for role in targets:
|
|
container_metrics[role]["docker_stats"] = stats[role]
|
|
host_tcp = parse_ss_tcp(runner.run(["ss", "-tinm"]))
|
|
validate_pins(runner, source, config, targets)
|
|
return {
|
|
"sequence": sequence,
|
|
"observed_at_utc": utc_now(),
|
|
"containers": container_metrics,
|
|
"host_tcp": host_tcp,
|
|
}
|
|
|
|
|
|
def _summary(samples: list[dict[str, object]]) -> dict[str, object]:
|
|
containers: dict[str, dict[str, int | float]] = {}
|
|
for role in ("api", "caddy"):
|
|
role_samples = [sample["containers"][role] for sample in samples] # type: ignore[index]
|
|
containers[role] = {
|
|
"cgroup_memory_current_bytes_max": max(
|
|
item["cgroup_v2"]["memory_current_bytes"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"cgroup_memory_peak_bytes_max": max(
|
|
item["cgroup_v2"]["memory_peak_bytes"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"cgroup_cpu_usage_usec_max": max(
|
|
item["cgroup_v2"]["cpu_stat"]["usage_usec"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"cgroup_pids_current_max": max(
|
|
item["cgroup_v2"]["pids_current"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"proc_vm_rss_bytes_max": max(
|
|
item["init_process"]["vm_rss_bytes"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"proc_vm_hwm_bytes_max": max(
|
|
item["init_process"]["vm_hwm_bytes"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"proc_threads_max": max(
|
|
item["init_process"]["threads"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"proc_fd_count_max": max(
|
|
item["init_process"]["fd_count"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"docker_cpu_percent_max": max(
|
|
item["docker_stats"]["cpu_percent"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"docker_memory_usage_bytes_max": max(
|
|
item["docker_stats"]["memory_usage_bytes"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
"docker_pids_max": max(
|
|
item["docker_stats"]["pids"]
|
|
for item in role_samples # type: ignore[index]
|
|
),
|
|
}
|
|
tcp_samples = [sample["host_tcp"] for sample in samples]
|
|
host_tcp = {
|
|
f"{field}_max": max(item[field] for item in tcp_samples) # type: ignore[index]
|
|
for field in (
|
|
"connections",
|
|
"recv_q_bytes_total",
|
|
"send_q_bytes_total",
|
|
"retransmit_current_total",
|
|
"retransmit_cumulative_total",
|
|
)
|
|
}
|
|
return {"containers": containers, "host_tcp": host_tcp}
|
|
|
|
|
|
def _scope() -> dict[str, object]:
|
|
return {
|
|
"metadata_only": True,
|
|
"raw_command_output_retained": False,
|
|
"socket_endpoints_retained": False,
|
|
"request_payloads_retained": False,
|
|
"audio_retained": False,
|
|
"transcripts_retained": False,
|
|
"host_tcp_scope": "all_tcp_sockets_visible_to_host_sampler",
|
|
"cloudflare_edge": {
|
|
"measured": False,
|
|
"internal_queue_measured": False,
|
|
"evidence_boundary": "separate_external_artifact_required",
|
|
},
|
|
}
|
|
|
|
|
|
def base_evidence(config: CaptureConfig) -> dict[str, object]:
|
|
return {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"status": "running",
|
|
"started_at_utc": utc_now(),
|
|
"ended_at_utc": None,
|
|
"scope": _scope(),
|
|
"requested": {
|
|
"compose_project": config.compose_project,
|
|
"public_host": config.public_host,
|
|
"samples": config.samples,
|
|
"interval_seconds": config.interval_seconds,
|
|
"roles": {
|
|
"api": {
|
|
"compose_service": config.api_service,
|
|
"expected_image_digest": config.api_image_digest.lower(),
|
|
},
|
|
"caddy": {
|
|
"compose_service": config.caddy_service,
|
|
"expected_image_digest": config.caddy_image_digest.lower(),
|
|
},
|
|
},
|
|
},
|
|
"targets": {},
|
|
"samples_completed": 0,
|
|
"samples": [],
|
|
"summary": {},
|
|
"failure_type": None,
|
|
}
|
|
|
|
|
|
def capture_evidence(
|
|
config: CaptureConfig,
|
|
runner: CommandRunner,
|
|
source: FileSource,
|
|
*,
|
|
sleep=time.sleep,
|
|
) -> dict[str, object]:
|
|
validate_config(config)
|
|
for command in ("docker", "ss"):
|
|
if not runner.available(command):
|
|
raise EvidenceFailure(f"command_unavailable:{command}")
|
|
targets = pin_targets(config, runner, source)
|
|
evidence = base_evidence(config)
|
|
evidence["targets"] = {role: asdict(target) for role, target in targets.items()}
|
|
samples: list[dict[str, object]] = []
|
|
for index in range(config.samples):
|
|
if index:
|
|
sleep(config.interval_seconds)
|
|
samples.append(
|
|
collect_sample(
|
|
runner,
|
|
source,
|
|
config,
|
|
targets,
|
|
sequence=index + 1,
|
|
)
|
|
)
|
|
evidence["samples"] = samples
|
|
evidence["samples_completed"] = len(samples)
|
|
evidence["summary"] = _summary(samples)
|
|
evidence["status"] = "passed"
|
|
evidence["ended_at_utc"] = utc_now()
|
|
return evidence
|
|
|
|
|
|
def parser() -> argparse.ArgumentParser:
|
|
result = argparse.ArgumentParser(
|
|
description="Bounded metadata-only G7 Docker/cgroup/proc/socket sampler"
|
|
)
|
|
result.add_argument("--compose-project", required=True)
|
|
result.add_argument(
|
|
"--public-host",
|
|
required=True,
|
|
help="public WSS/API host bound to this exact Compose deployment",
|
|
)
|
|
result.add_argument("--api-container", required=True)
|
|
result.add_argument("--api-service", default="api")
|
|
result.add_argument("--api-image-digest", required=True)
|
|
result.add_argument("--caddy-container", required=True)
|
|
result.add_argument("--caddy-service", default="caddy")
|
|
result.add_argument("--caddy-image-digest", required=True)
|
|
result.add_argument("--samples", type=int, default=DEFAULT_SAMPLES)
|
|
result.add_argument(
|
|
"--interval-seconds", type=float, default=DEFAULT_INTERVAL_SECONDS
|
|
)
|
|
result.add_argument("--proc-root", default="/proc")
|
|
result.add_argument("--cgroup-root", default="/sys/fs/cgroup")
|
|
result.add_argument("--evidence-output", type=Path)
|
|
return result
|
|
|
|
|
|
def _write_evidence(evidence: dict[str, object], output: Path | None) -> None:
|
|
payload = json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
|
if output is not None:
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(payload, encoding="utf-8")
|
|
print(payload, end="")
|
|
|
|
|
|
def main() -> int:
|
|
args = parser().parse_args()
|
|
config = CaptureConfig(
|
|
compose_project=args.compose_project,
|
|
public_host=args.public_host,
|
|
api_container=args.api_container,
|
|
api_service=args.api_service,
|
|
api_image_digest=args.api_image_digest,
|
|
caddy_container=args.caddy_container,
|
|
caddy_service=args.caddy_service,
|
|
caddy_image_digest=args.caddy_image_digest,
|
|
samples=args.samples,
|
|
interval_seconds=args.interval_seconds,
|
|
proc_root=args.proc_root,
|
|
cgroup_root=args.cgroup_root,
|
|
)
|
|
try:
|
|
evidence = capture_evidence(
|
|
config, SubprocessCommandRunner(), LocalFileSource()
|
|
)
|
|
_write_evidence(evidence, args.evidence_output)
|
|
return 0
|
|
except EvidenceFailure as exc:
|
|
evidence = base_evidence(config)
|
|
evidence["status"] = "failed"
|
|
evidence["failure_type"] = str(exc)
|
|
evidence["ended_at_utc"] = utc_now()
|
|
_write_evidence(evidence, args.evidence_output)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|