소유자가 2026-10-27 실제 적용과 정기 실행을 승인했다. 원본 90일·집계 365일· 집계 대상 7일 창은 2026-10-20 운영 dry-run과 같은 값으로 고정하고, 데이터를 지우는 작업이라 기본값은 꺼짐이다. DB 풀이 정상일 때만 예약하며 실패는 기록만 하고 다음 회차를 계속한다. NAS compose는 환경변수를 명시 전달하므로 설정 3개를 api 환경에 추가한다.
332 lines
12 KiB
Python
332 lines
12 KiB
Python
"""Maintenance helpers for persisted admin health samples."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from dataclasses import dataclass
|
|
import logging
|
|
import re
|
|
|
|
from ..config import settings
|
|
from ..db import acquire
|
|
|
|
# 2026-10-20 운영 dry-run과 같은 값으로 소유자가 승인한 고정 보존 창이다.
|
|
# 설정으로 노출하지 않는다.
|
|
_RETENTION_ROLLUP_DAYS = 7
|
|
_RETENTION_RETENTION_DAYS = 90
|
|
_RETENTION_ROLLUP_RETENTION_DAYS = 365
|
|
|
|
_RETENTION_TASK: asyncio.Task[None] | None = None
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AdminHealthMaintenanceResult:
|
|
applied: bool
|
|
rollup_days: int
|
|
retention_days: int
|
|
rollup_retention_days: int
|
|
rollup_event_count: int
|
|
rollup_bucket_count: int
|
|
upserted_rollups: int
|
|
prunable_event_count: int
|
|
deleted_events: int
|
|
# apply 전에 이미 존재한 만료 일별 rollup 수다. 이번 집계에서 새로 생긴 행은 포함하지 않는다.
|
|
prunable_rollup_count: int
|
|
deleted_rollups: int
|
|
|
|
|
|
def _validate_windows(
|
|
*,
|
|
rollup_days: int,
|
|
retention_days: int,
|
|
rollup_retention_days: int,
|
|
) -> None:
|
|
if rollup_days < 1:
|
|
raise ValueError("rollup_days must be 1 or greater")
|
|
if retention_days < rollup_days:
|
|
raise ValueError("retention_days must be greater than or equal to rollup_days")
|
|
if rollup_retention_days < retention_days:
|
|
raise ValueError(
|
|
"rollup_retention_days must be greater than or equal to retention_days"
|
|
)
|
|
|
|
|
|
def _deleted_count(command_tag: str) -> int:
|
|
parts = (command_tag or "").split()
|
|
if len(parts) >= 2 and parts[0].upper() == "DELETE":
|
|
try:
|
|
return int(parts[-1])
|
|
except ValueError:
|
|
return 0
|
|
return 0
|
|
|
|
|
|
def _row_int(row, key: str) -> int:
|
|
if row is None:
|
|
return 0
|
|
try:
|
|
return int(row[key] or 0)
|
|
except (KeyError, TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
_EXPECTED_ROLLUP_DELETE_QUAL = "(app.current_role_name()='admin')"
|
|
|
|
|
|
def _normalize_policy_expression(expression: object) -> str:
|
|
if not isinstance(expression, str):
|
|
return ""
|
|
return re.sub(r"\s+", "", expression).replace("::text", "")
|
|
|
|
|
|
def _is_exact_rollup_delete_policy(policy) -> bool:
|
|
try:
|
|
return (
|
|
policy["policy_name"] == "p_admin_health_daily_rollup_delete"
|
|
and policy["command"] == "d"
|
|
and policy["permissive"] is True
|
|
and policy["rls_enabled"] is True
|
|
and _normalize_policy_expression(policy["qual"])
|
|
== _EXPECTED_ROLLUP_DELETE_QUAL
|
|
)
|
|
except (KeyError, TypeError):
|
|
return False
|
|
|
|
|
|
async def _require_rollup_delete_policy(conn) -> None:
|
|
policies = await conn.fetch(
|
|
"""
|
|
SELECT
|
|
policy.polname AS policy_name,
|
|
policy.polcmd::text AS command,
|
|
policy.polpermissive AS permissive,
|
|
relation.relrowsecurity AS rls_enabled,
|
|
pg_get_expr(policy.polqual, policy.polrelid) AS qual
|
|
FROM pg_policy AS policy
|
|
JOIN pg_class AS relation ON relation.oid = policy.polrelid
|
|
JOIN pg_namespace AS namespace ON namespace.oid = relation.relnamespace
|
|
WHERE namespace.nspname = 'app'
|
|
AND relation.relname = 'admin_health_daily_rollup'
|
|
AND policy.polpermissive IS TRUE
|
|
AND policy.polcmd IN ('d', '*')
|
|
ORDER BY policy.polname
|
|
"""
|
|
)
|
|
if len(policies) != 1 or not _is_exact_rollup_delete_policy(policies[0]):
|
|
raise RuntimeError(
|
|
"admin health daily rollup DELETE policy is missing, invalid, or permissive; "
|
|
"owner must apply migration 25_admin_health_rollup_retention.sql"
|
|
)
|
|
|
|
|
|
async def maintain_admin_health_events(
|
|
*,
|
|
rollup_days: int,
|
|
retention_days: int = 90,
|
|
rollup_retention_days: int = 365,
|
|
apply: bool = False,
|
|
) -> AdminHealthMaintenanceResult:
|
|
"""오래된 관리자 헬스 표본을 일별로 집계하고 보존 기간이 지난 행을 정리한다.
|
|
|
|
dry-run은 데이터를 바꾸지 않고 영향받는 표본과 버킷 수만 반환한다.
|
|
prunable_rollup_count는 apply 전에 존재한 만료 일별 rollup 수다. 이번 집계에서
|
|
새로 생긴 만료 rollup은 이 수에 포함하지 않지만, apply에서는 일별 rollup upsert 후
|
|
raw 표본과 만료된 일별 rollup을 같은 트랜잭션에서 순서대로 삭제한다.
|
|
"""
|
|
_validate_windows(
|
|
rollup_days=rollup_days,
|
|
retention_days=retention_days,
|
|
rollup_retention_days=rollup_retention_days,
|
|
)
|
|
async with acquire(role="admin") as conn:
|
|
rollup_stats = await conn.fetchrow(
|
|
"""
|
|
SELECT
|
|
COUNT(*)::int AS event_count,
|
|
COUNT(DISTINCT (observed_at::date, environment, engine_mode, service_key))::int
|
|
AS bucket_count
|
|
FROM app.admin_health_event
|
|
WHERE observed_at::date < current_date - $1::int
|
|
""",
|
|
rollup_days,
|
|
)
|
|
prune_stats = await conn.fetchrow(
|
|
"""
|
|
SELECT COUNT(*)::int AS event_count
|
|
FROM app.admin_health_event
|
|
WHERE observed_at::date < current_date - $1::int
|
|
""",
|
|
retention_days,
|
|
)
|
|
rollup_prune_stats = await conn.fetchrow(
|
|
"""
|
|
SELECT COUNT(*)::int AS rollup_count
|
|
FROM app.admin_health_daily_rollup
|
|
WHERE rollup_date < current_date - $1::int
|
|
""",
|
|
rollup_retention_days,
|
|
)
|
|
upserted_rollups = 0
|
|
deleted_events = 0
|
|
deleted_rollups = 0
|
|
if apply:
|
|
await _require_rollup_delete_policy(conn)
|
|
rows = await conn.fetch(
|
|
"""
|
|
WITH rolled AS (
|
|
SELECT
|
|
observed_at::date AS rollup_date,
|
|
environment,
|
|
engine_mode,
|
|
service_key,
|
|
(array_agg(service_name ORDER BY observed_at DESC, id DESC))[1] AS service_name,
|
|
COUNT(*)::int AS sample_count,
|
|
COUNT(*) FILTER (WHERE service_status = 'ok')::int AS ok_samples,
|
|
COUNT(*) FILTER (WHERE service_status = 'degraded')::int AS degraded_samples,
|
|
COUNT(*) FILTER (WHERE service_status = 'down')::int AS down_samples,
|
|
MIN(observed_at) AS first_observed_at,
|
|
MAX(observed_at) AS last_observed_at,
|
|
(array_agg(service_status ORDER BY observed_at DESC, id DESC))[1] AS latest_status,
|
|
MAX(observed_at) FILTER (WHERE service_status = 'down') AS last_down_at,
|
|
MAX(load)::real AS max_load
|
|
FROM app.admin_health_event
|
|
WHERE observed_at::date < current_date - $1::int
|
|
GROUP BY observed_at::date, environment, engine_mode, service_key
|
|
)
|
|
INSERT INTO app.admin_health_daily_rollup (
|
|
rollup_date,
|
|
environment,
|
|
engine_mode,
|
|
service_key,
|
|
service_name,
|
|
sample_count,
|
|
ok_samples,
|
|
degraded_samples,
|
|
down_samples,
|
|
first_observed_at,
|
|
last_observed_at,
|
|
latest_status,
|
|
last_down_at,
|
|
max_load,
|
|
generated_at
|
|
)
|
|
SELECT
|
|
rollup_date,
|
|
environment,
|
|
engine_mode,
|
|
service_key,
|
|
service_name,
|
|
sample_count,
|
|
ok_samples,
|
|
degraded_samples,
|
|
down_samples,
|
|
first_observed_at,
|
|
last_observed_at,
|
|
latest_status,
|
|
last_down_at,
|
|
max_load,
|
|
now()
|
|
FROM rolled
|
|
ON CONFLICT (rollup_date, environment, engine_mode, service_key)
|
|
DO UPDATE SET
|
|
service_name = EXCLUDED.service_name,
|
|
sample_count = EXCLUDED.sample_count,
|
|
ok_samples = EXCLUDED.ok_samples,
|
|
degraded_samples = EXCLUDED.degraded_samples,
|
|
down_samples = EXCLUDED.down_samples,
|
|
first_observed_at = EXCLUDED.first_observed_at,
|
|
last_observed_at = EXCLUDED.last_observed_at,
|
|
latest_status = EXCLUDED.latest_status,
|
|
last_down_at = EXCLUDED.last_down_at,
|
|
max_load = EXCLUDED.max_load,
|
|
generated_at = now()
|
|
RETURNING 1
|
|
""",
|
|
rollup_days,
|
|
)
|
|
upserted_rollups = len(rows)
|
|
deleted_events = _deleted_count(
|
|
await conn.execute(
|
|
"""
|
|
DELETE FROM app.admin_health_event
|
|
WHERE observed_at::date < current_date - $1::int
|
|
""",
|
|
retention_days,
|
|
)
|
|
)
|
|
deleted_rollups = _deleted_count(
|
|
await conn.execute(
|
|
"""
|
|
DELETE FROM app.admin_health_daily_rollup
|
|
WHERE rollup_date < current_date - $1::int
|
|
""",
|
|
rollup_retention_days,
|
|
)
|
|
)
|
|
|
|
return AdminHealthMaintenanceResult(
|
|
applied=apply,
|
|
rollup_days=rollup_days,
|
|
retention_days=retention_days,
|
|
rollup_retention_days=rollup_retention_days,
|
|
rollup_event_count=_row_int(rollup_stats, "event_count"),
|
|
rollup_bucket_count=_row_int(rollup_stats, "bucket_count"),
|
|
upserted_rollups=upserted_rollups,
|
|
prunable_event_count=_row_int(prune_stats, "event_count"),
|
|
deleted_events=deleted_events,
|
|
prunable_rollup_count=_row_int(rollup_prune_stats, "rollup_count"),
|
|
deleted_rollups=deleted_rollups,
|
|
)
|
|
|
|
|
|
async def _retention_loop() -> None:
|
|
delay = settings.admin_health_retention_startup_delay_seconds
|
|
if delay:
|
|
await asyncio.sleep(delay)
|
|
while True:
|
|
try:
|
|
result = await maintain_admin_health_events(
|
|
rollup_days=_RETENTION_ROLLUP_DAYS,
|
|
retention_days=_RETENTION_RETENTION_DAYS,
|
|
rollup_retention_days=_RETENTION_ROLLUP_RETENTION_DAYS,
|
|
apply=True,
|
|
)
|
|
logger.info(
|
|
"관리자 헬스 보존 정책 적용 완료: deleted_events=%d deleted_rollups=%d "
|
|
"upserted_rollups=%d",
|
|
result.deleted_events,
|
|
result.deleted_rollups,
|
|
result.upserted_rollups,
|
|
)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.exception("관리자 헬스 보존 정책 scheduler cycle 실패")
|
|
await asyncio.sleep(settings.admin_health_retention_interval_seconds)
|
|
|
|
|
|
def schedule_admin_health_retention() -> asyncio.Task[None] | None:
|
|
global _RETENTION_TASK
|
|
if not settings.admin_health_retention_enabled:
|
|
return None
|
|
if _RETENTION_TASK is not None and not _RETENTION_TASK.done():
|
|
return _RETENTION_TASK
|
|
_RETENTION_TASK = asyncio.create_task(
|
|
_retention_loop(),
|
|
name="admin-health-retention",
|
|
)
|
|
return _RETENTION_TASK
|
|
|
|
|
|
async def stop_admin_health_retention() -> None:
|
|
global _RETENTION_TASK
|
|
task = _RETENTION_TASK
|
|
_RETENTION_TASK = None
|
|
if task is None or task.done():
|
|
return
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|