vignette/apps/api/app/services/admin_health_maintenance.py
Yun Chan b6de32c1dd 관리자 상태 기록 보존 정책을 API 안에서 하루 한 번 실행한다
소유자가 2026-10-27 실제 적용과 정기 실행을 승인했다. 원본 90일·집계 365일·
집계 대상 7일 창은 2026-10-20 운영 dry-run과 같은 값으로 고정하고, 데이터를
지우는 작업이라 기본값은 꺼짐이다. DB 풀이 정상일 때만 예약하며 실패는 기록만
하고 다음 회차를 계속한다. NAS compose는 환경변수를 명시 전달하므로 설정 3개를
api 환경에 추가한다.
2026-10-27 11:20:44 +09:00

332 lines
12 KiB
Python

"""Maintenance helpers for persisted admin health samples."""
from __future__ import annotations
import asyncio
from dataclasses import dataclass
import logging
import re
from ..config import settings
from ..db import acquire
# 2026-10-20 운영 dry-run과 같은 값으로 소유자가 승인한 고정 보존 창이다.
# 설정으로 노출하지 않는다.
_RETENTION_ROLLUP_DAYS = 7
_RETENTION_RETENTION_DAYS = 90
_RETENTION_ROLLUP_RETENTION_DAYS = 365
_RETENTION_TASK: asyncio.Task[None] | None = None
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class AdminHealthMaintenanceResult:
applied: bool
rollup_days: int
retention_days: int
rollup_retention_days: int
rollup_event_count: int
rollup_bucket_count: int
upserted_rollups: int
prunable_event_count: int
deleted_events: int
# apply 전에 이미 존재한 만료 일별 rollup 수다. 이번 집계에서 새로 생긴 행은 포함하지 않는다.
prunable_rollup_count: int
deleted_rollups: int
def _validate_windows(
*,
rollup_days: int,
retention_days: int,
rollup_retention_days: int,
) -> None:
if rollup_days < 1:
raise ValueError("rollup_days must be 1 or greater")
if retention_days < rollup_days:
raise ValueError("retention_days must be greater than or equal to rollup_days")
if rollup_retention_days < retention_days:
raise ValueError(
"rollup_retention_days must be greater than or equal to retention_days"
)
def _deleted_count(command_tag: str) -> int:
parts = (command_tag or "").split()
if len(parts) >= 2 and parts[0].upper() == "DELETE":
try:
return int(parts[-1])
except ValueError:
return 0
return 0
def _row_int(row, key: str) -> int:
if row is None:
return 0
try:
return int(row[key] or 0)
except (KeyError, TypeError, ValueError):
return 0
_EXPECTED_ROLLUP_DELETE_QUAL = "(app.current_role_name()='admin')"
def _normalize_policy_expression(expression: object) -> str:
if not isinstance(expression, str):
return ""
return re.sub(r"\s+", "", expression).replace("::text", "")
def _is_exact_rollup_delete_policy(policy) -> bool:
try:
return (
policy["policy_name"] == "p_admin_health_daily_rollup_delete"
and policy["command"] == "d"
and policy["permissive"] is True
and policy["rls_enabled"] is True
and _normalize_policy_expression(policy["qual"])
== _EXPECTED_ROLLUP_DELETE_QUAL
)
except (KeyError, TypeError):
return False
async def _require_rollup_delete_policy(conn) -> None:
policies = await conn.fetch(
"""
SELECT
policy.polname AS policy_name,
policy.polcmd::text AS command,
policy.polpermissive AS permissive,
relation.relrowsecurity AS rls_enabled,
pg_get_expr(policy.polqual, policy.polrelid) AS qual
FROM pg_policy AS policy
JOIN pg_class AS relation ON relation.oid = policy.polrelid
JOIN pg_namespace AS namespace ON namespace.oid = relation.relnamespace
WHERE namespace.nspname = 'app'
AND relation.relname = 'admin_health_daily_rollup'
AND policy.polpermissive IS TRUE
AND policy.polcmd IN ('d', '*')
ORDER BY policy.polname
"""
)
if len(policies) != 1 or not _is_exact_rollup_delete_policy(policies[0]):
raise RuntimeError(
"admin health daily rollup DELETE policy is missing, invalid, or permissive; "
"owner must apply migration 25_admin_health_rollup_retention.sql"
)
async def maintain_admin_health_events(
*,
rollup_days: int,
retention_days: int = 90,
rollup_retention_days: int = 365,
apply: bool = False,
) -> AdminHealthMaintenanceResult:
"""오래된 관리자 헬스 표본을 일별로 집계하고 보존 기간이 지난 행을 정리한다.
dry-run은 데이터를 바꾸지 않고 영향받는 표본과 버킷 수만 반환한다.
prunable_rollup_count는 apply 전에 존재한 만료 일별 rollup 수다. 이번 집계에서
새로 생긴 만료 rollup은 이 수에 포함하지 않지만, apply에서는 일별 rollup upsert 후
raw 표본과 만료된 일별 rollup을 같은 트랜잭션에서 순서대로 삭제한다.
"""
_validate_windows(
rollup_days=rollup_days,
retention_days=retention_days,
rollup_retention_days=rollup_retention_days,
)
async with acquire(role="admin") as conn:
rollup_stats = await conn.fetchrow(
"""
SELECT
COUNT(*)::int AS event_count,
COUNT(DISTINCT (observed_at::date, environment, engine_mode, service_key))::int
AS bucket_count
FROM app.admin_health_event
WHERE observed_at::date < current_date - $1::int
""",
rollup_days,
)
prune_stats = await conn.fetchrow(
"""
SELECT COUNT(*)::int AS event_count
FROM app.admin_health_event
WHERE observed_at::date < current_date - $1::int
""",
retention_days,
)
rollup_prune_stats = await conn.fetchrow(
"""
SELECT COUNT(*)::int AS rollup_count
FROM app.admin_health_daily_rollup
WHERE rollup_date < current_date - $1::int
""",
rollup_retention_days,
)
upserted_rollups = 0
deleted_events = 0
deleted_rollups = 0
if apply:
await _require_rollup_delete_policy(conn)
rows = await conn.fetch(
"""
WITH rolled AS (
SELECT
observed_at::date AS rollup_date,
environment,
engine_mode,
service_key,
(array_agg(service_name ORDER BY observed_at DESC, id DESC))[1] AS service_name,
COUNT(*)::int AS sample_count,
COUNT(*) FILTER (WHERE service_status = 'ok')::int AS ok_samples,
COUNT(*) FILTER (WHERE service_status = 'degraded')::int AS degraded_samples,
COUNT(*) FILTER (WHERE service_status = 'down')::int AS down_samples,
MIN(observed_at) AS first_observed_at,
MAX(observed_at) AS last_observed_at,
(array_agg(service_status ORDER BY observed_at DESC, id DESC))[1] AS latest_status,
MAX(observed_at) FILTER (WHERE service_status = 'down') AS last_down_at,
MAX(load)::real AS max_load
FROM app.admin_health_event
WHERE observed_at::date < current_date - $1::int
GROUP BY observed_at::date, environment, engine_mode, service_key
)
INSERT INTO app.admin_health_daily_rollup (
rollup_date,
environment,
engine_mode,
service_key,
service_name,
sample_count,
ok_samples,
degraded_samples,
down_samples,
first_observed_at,
last_observed_at,
latest_status,
last_down_at,
max_load,
generated_at
)
SELECT
rollup_date,
environment,
engine_mode,
service_key,
service_name,
sample_count,
ok_samples,
degraded_samples,
down_samples,
first_observed_at,
last_observed_at,
latest_status,
last_down_at,
max_load,
now()
FROM rolled
ON CONFLICT (rollup_date, environment, engine_mode, service_key)
DO UPDATE SET
service_name = EXCLUDED.service_name,
sample_count = EXCLUDED.sample_count,
ok_samples = EXCLUDED.ok_samples,
degraded_samples = EXCLUDED.degraded_samples,
down_samples = EXCLUDED.down_samples,
first_observed_at = EXCLUDED.first_observed_at,
last_observed_at = EXCLUDED.last_observed_at,
latest_status = EXCLUDED.latest_status,
last_down_at = EXCLUDED.last_down_at,
max_load = EXCLUDED.max_load,
generated_at = now()
RETURNING 1
""",
rollup_days,
)
upserted_rollups = len(rows)
deleted_events = _deleted_count(
await conn.execute(
"""
DELETE FROM app.admin_health_event
WHERE observed_at::date < current_date - $1::int
""",
retention_days,
)
)
deleted_rollups = _deleted_count(
await conn.execute(
"""
DELETE FROM app.admin_health_daily_rollup
WHERE rollup_date < current_date - $1::int
""",
rollup_retention_days,
)
)
return AdminHealthMaintenanceResult(
applied=apply,
rollup_days=rollup_days,
retention_days=retention_days,
rollup_retention_days=rollup_retention_days,
rollup_event_count=_row_int(rollup_stats, "event_count"),
rollup_bucket_count=_row_int(rollup_stats, "bucket_count"),
upserted_rollups=upserted_rollups,
prunable_event_count=_row_int(prune_stats, "event_count"),
deleted_events=deleted_events,
prunable_rollup_count=_row_int(rollup_prune_stats, "rollup_count"),
deleted_rollups=deleted_rollups,
)
async def _retention_loop() -> None:
delay = settings.admin_health_retention_startup_delay_seconds
if delay:
await asyncio.sleep(delay)
while True:
try:
result = await maintain_admin_health_events(
rollup_days=_RETENTION_ROLLUP_DAYS,
retention_days=_RETENTION_RETENTION_DAYS,
rollup_retention_days=_RETENTION_ROLLUP_RETENTION_DAYS,
apply=True,
)
logger.info(
"관리자 헬스 보존 정책 적용 완료: deleted_events=%d deleted_rollups=%d "
"upserted_rollups=%d",
result.deleted_events,
result.deleted_rollups,
result.upserted_rollups,
)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("관리자 헬스 보존 정책 scheduler cycle 실패")
await asyncio.sleep(settings.admin_health_retention_interval_seconds)
def schedule_admin_health_retention() -> asyncio.Task[None] | None:
global _RETENTION_TASK
if not settings.admin_health_retention_enabled:
return None
if _RETENTION_TASK is not None and not _RETENTION_TASK.done():
return _RETENTION_TASK
_RETENTION_TASK = asyncio.create_task(
_retention_loop(),
name="admin-health-retention",
)
return _RETENTION_TASK
async def stop_admin_health_retention() -> None:
global _RETENTION_TASK
task = _RETENTION_TASK
_RETENTION_TASK = None
if task is None or task.done():
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass