vignette/apps/api/app/routes/admin.py

"""Admin operations routes."""

from __future__ import annotations

import time
from datetime import datetime, timezone
from decimal import Decimal
from typing import Annotated, Literal, cast
from uuid import UUID

from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import BaseModel, Field

from ..auth_types import AccountStatus, RoleName
from ..auth_sessions import (
    ManagedUserPatch,
    ManagedUserUpsertInput,
    active_session_count,
    deactivate_managed_user,
    get_managed_user,
    has_admin_access,
    is_super_admin_email,
    list_managed_users,
    update_managed_user,
    upsert_managed_user,
)
from ..config import settings
from ..contracts.engine_gateway import (
    ENGINE_PROVIDER_DEFAULTS,
    ENGINE_PROVIDERS,
    ENGINE_REASONING_EFFORTS,
    EngineCapabilitiesResponse,
    EngineProvider,
    ReasoningEffort,
)
from ..db import acquire, get_pool, healthcheck
from ..deps import Principal, require_admin_access
from ..engine_client import engine_client
from ..runtime_policy import require_runtime_fallback_allowed
from ..services import evaluator, notifications, rag
from ..services.llm_pricing import (
    estimate_reference_cost,
    provider_uses_reference_cost,
)
from ..services.voice import voice_service
from ..services.voice_runtime import VoiceRuntimeSnapshot, voice_runtime_metrics
from ..store import store

router = APIRouter(prefix="/admin", tags=["admin"])

AdminPrincipal = Annotated[Principal, Depends(require_admin_access())]
HealthStatus = Literal["ok", "degraded", "down"]
UsageBudgetStatus = Literal["disabled", "ok", "warn", "exceeded"]
UsageCostBasis = Literal[
    "provider_estimate",
    "provider_reported",
    "reference_rate",
    "unavailable",
]
TicketCategory = Literal[
    "account_access",
    "session_review",
    "voice_browser",
    "content_scenario",
    "safety",
    "other",
]
TicketPriority = Literal["low", "normal", "high", "urgent"]
TicketStatus = Literal["open", "triaged", "in_progress", "resolved", "closed"]
NotificationDeliveryStatus = Literal["queued", "sending", "sent", "failed", "skipped"]

METERED_CLIENT_TURN_FILTER_SQL = """
    speaker = 'client'
    AND (
      llm_provider IS NOT NULL OR model IS NOT NULL
      OR tokens_in IS NOT NULL OR tokens_out IS NOT NULL
      OR cost_usd IS NOT NULL
    )
"""

USAGE_AGGREGATE_COLUMNS_SQL = """
    COUNT(*) AS turns,
    COUNT(*) FILTER (
      WHERE COALESCE(tokens_in, 0) > 0 OR COALESCE(tokens_out, 0) > 0
    ) AS token_metered_turns,
    COUNT(*) FILTER (
      WHERE COALESCE(tokens_in, 0) <= 0 AND COALESCE(tokens_out, 0) <= 0
    ) AS token_unmetered_turns,
    COALESCE(SUM(tokens_in), 0)::bigint AS tokens_in,
    COALESCE(SUM(tokens_out), 0)::bigint AS tokens_out,
    COALESCE(SUM(cost_usd), 0)::numeric AS cost_usd,
    COUNT(*) FILTER (WHERE COALESCE(cost_usd, 0) <= 0) AS unpriced_turns,
    COALESCE(
      SUM(tokens_in) FILTER (WHERE COALESCE(cost_usd, 0) <= 0),
      0
    )::bigint AS unpriced_tokens_in,
    COALESCE(
      SUM(tokens_out) FILTER (WHERE COALESCE(cost_usd, 0) <= 0),
      0
    )::bigint AS unpriced_tokens_out
"""

SUPPORT_TICKET_DETAIL_FROM_SQL = """
    SELECT
      t.id,
      t.reporter_id,
      t.reporter_email,
      t.reporter_name,
      t.reporter_role,
      t.category,
      t.priority,
      t.status,
      t.subject,
      t.body,
      t.source_path,
      t.fingerprint,
      t.parent_ticket_id,
      t.assigned_group,
      t.resolution_note,
      t.created_at,
      t.updated_at,
      t.resolved_at,
      COALESCE(dup.duplicate_count, 0) AS duplicate_count,
      dup.parent_candidate_id AS duplicate_parent_candidate_id,
      COALESCE(child.child_ticket_count, 0) AS child_ticket_count,
      COALESCE(ev.event_count, 0) AS event_count,
      ev.last_event_at
    FROM app.support_ticket AS t
    LEFT JOIN LATERAL (
      SELECT
        count(*) FILTER (WHERE id <> t.id)::int AS duplicate_count,
        (array_agg(id::text ORDER BY created_at ASC, id ASC))[1] AS parent_candidate_id
      FROM app.support_ticket
      WHERE fingerprint <> ''
        AND fingerprint = t.fingerprint
    ) AS dup ON TRUE
    LEFT JOIN LATERAL (
      SELECT count(*)::int AS child_ticket_count
      FROM app.support_ticket
      WHERE parent_ticket_id = t.id
    ) AS child ON TRUE
    LEFT JOIN LATERAL (
      SELECT count(*)::int AS event_count, max(created_at) AS last_event_at
      FROM audit.audit_log
      WHERE action = 'support_ticket_update'
        AND target_kind = 'support_ticket'
        AND target_id = t.id::text
    ) AS ev ON TRUE
"""


class AdminServiceHealth(BaseModel):
    key: str
    name: str
    status: HealthStatus
    detail: str
    metric: str
    load: float


class AdminHealthResponse(BaseModel):
    status: HealthStatus
    environment: str
    engine_mode: str
    services: list[AdminServiceHealth]


class AdminUsageBreakdown(BaseModel):
    provider: str
    model: str
    turns: int
    token_metered_turns: int = 0
    token_unmetered_turns: int = 0
    tokens_in: int
    tokens_out: int
    cost_usd: float
    recorded_cost_usd: float = 0.0
    estimated_cost_usd: float = 0.0
    cost_basis: UsageCostBasis = "provider_reported"
    rate_label: str | None = None
    rate_source_url: str | None = None


class AdminUsageDailyCost(BaseModel):
    day: str
    turns: int
    tokens_in: int
    tokens_out: int
    cost_usd: float


class AdminUsageBudget(BaseModel):
    limit_usd: float
    used_ratio: float
    remaining_usd: float | None
    status: UsageBudgetStatus


class AdminUsageEvaluatorCache(BaseModel):
    enabled: bool
    entries: int
    hits: int
    misses: int
    stores: int
    evictions: int
    requests: int
    hit_rate: float


class AdminUsageResponse(BaseModel):
    source: Literal["database", "server_session_registry"]
    durable: bool
    window_days: int
    generated_at: float
    total_turns: int
    metered_turns: int
    token_metered_turns: int = 0
    token_unmetered_turns: int = 0
    tokens_in: int
    tokens_out: int
    cost_usd: float
    recorded_cost_usd: float = 0.0
    estimated_cost_usd: float = 0.0
    budget: AdminUsageBudget
    evaluator_cache: AdminUsageEvaluatorCache
    by_provider: list[AdminUsageBreakdown]
    daily_cost: list[AdminUsageDailyCost] = Field(default_factory=list)


class AdminHealthEvent(BaseModel):
    event_id: int
    observed_at: float
    overall_status: HealthStatus
    service_key: str
    service_name: str
    service_status: HealthStatus
    detail: str
    metric: str
    load: float


class AdminUptimeServiceSummary(BaseModel):
    service_key: str
    service_name: str
    samples: int
    ok_samples: int
    degraded_samples: int
    down_samples: int
    latest_status: HealthStatus
    latest_observed_at: float | None = None


class AdminUptimeResponse(BaseModel):
    source: Literal["database", "unavailable"]
    durable: bool
    window_hours: int
    generated_at: float
    sample_count: int
    ok_ratio: float
    degraded_events: int
    down_events: int
    last_down_at: float | None = None
    services: list[AdminUptimeServiceSummary]
    events: list[AdminHealthEvent]


class AdminTicketReporter(BaseModel):
    user_id: str | None = None
    email: str
    display_name: str
    role: str


class AdminSupportTicketResponse(BaseModel):
    ticket_id: str
    reporter: AdminTicketReporter
    category: TicketCategory
    priority: TicketPriority
    status: TicketStatus
    subject: str
    body: str
    source_path: str
    fingerprint: str
    parent_ticket_id: str | None = None
    duplicate_count: int = 0
    duplicate_parent_candidate_id: str | None = None
    child_ticket_count: int = 0
    assigned_group: str
    resolution_note: str
    created_at: float
    updated_at: float
    resolved_at: float | None = None
    event_count: int = 0
    last_event_at: float | None = None


class AdminTicketSummary(BaseModel):
    total: int
    open_count: int
    high_priority_count: int
    stale_count: int
    by_status: dict[str, int]
    by_category: dict[str, int]
    by_priority: dict[str, int]


class AdminTicketsResponse(BaseModel):
    source: Literal["database", "unavailable"]
    durable: bool
    generated_at: float
    tickets: list[AdminSupportTicketResponse]
    summary: AdminTicketSummary


class AdminEngineConfigResponse(BaseModel):
    engine_mode: EngineProvider
    engine_url: str
    model: str
    reasoning_effort: ReasoningEffort | None = None
    updated_by: str | None = None
    updated_at: float | None = None
    durable: bool = False
    source: Literal["database", "runtime_cache", "runtime_default"] = "runtime_default"


class AdminEngineConfigPatch(BaseModel):
    engine_mode: str | None = None
    engine_url: str | None = None
    model: str | None = None
    reasoning_effort: str | None = None


class AdminTicketPatch(BaseModel):
    status: TicketStatus | None = None
    priority: TicketPriority | None = None
    assigned_group: str | None = Field(default=None, max_length=120)
    resolution_note: str | None = Field(default=None, max_length=2000)
    parent_ticket_id: str | None = Field(default=None, max_length=36)


class AdminUserResponse(BaseModel):
    user_id: str
    email: str
    display_name: str
    role: RoleName
    admin_access: bool
    learner_feedback_enabled: bool
    super_admin: bool = False
    account_status: AccountStatus
    cohort_ids: list[str]
    affiliation: str
    active_sessions: int
    created_at: float
    last_seen_at: float
    source: Literal["database", "server_session_registry"]


class AdminUsersResponse(BaseModel):
    source: Literal["database", "server_session_registry"]
    durable: bool
    users: list[AdminUserResponse]


class AdminUserPatch(BaseModel):
    display_name: str | None = Field(default=None, min_length=1, max_length=80)
    role: RoleName | None = None
    admin_access: bool | None = None
    learner_feedback_enabled: bool | None = None
    account_status: AccountStatus | None = None
    affiliation: str | None = Field(default=None, max_length=120)
    cohort_ids: list[str] | None = None


class AdminNotificationDeliveryResponse(BaseModel):
    id: str
    event_id: str
    kind: str
    recipient_email: str
    recipient_name: str
    recipient_role: str
    subject: str
    status: NotificationDeliveryStatus
    attempts: int
    last_error: str | None = None
    provider_message_id: str | None = None
    created_at: float | None = None
    updated_at: float | None = None
    sent_at: float | None = None


class AdminNotificationsResponse(BaseModel):
    provider: str
    smtp_configured: bool
    queued: int
    failed: int
    sent: int
    skipped: int
    deliveries: list[AdminNotificationDeliveryResponse]


class AdminNotificationProcessResponse(BaseModel):
    processed: int
    sent: int
    failed: int
    skipped: int


class AdminNotificationTestResponse(AdminNotificationProcessResponse):
    provider: str
    smtp_configured: bool
    recipients: int


class RuntimeHealthMetrics(BaseModel):
    engine_latency_ms: float | None = None
    db_pool_size: int = 0
    db_pool_idle: int = 0
    db_pool_max: int = 0
    active_users: int = 0
    active_auth_sessions: int = 0
    active_sessions: int = 0
    ended_sessions: int = 0
    pending_reviews: int = 0


def _clamp01(value: float) -> float:
    return round(max(0.0, min(1.0, value)), 3)


def _pool_load(metrics: RuntimeHealthMetrics) -> float:
    if metrics.db_pool_max <= 0:
        return 0.0
    busy = max(0, metrics.db_pool_size - metrics.db_pool_idle)
    return _clamp01(busy / metrics.db_pool_max)


def _workload_load(count: int, expected_capacity: int) -> float:
    if expected_capacity <= 0:
        return 0.0
    return _clamp01(count / expected_capacity)


def _decimal_to_float(value: object) -> float:
    if value is None:
        return 0.0
    if isinstance(value, Decimal):
        return float(value)
    try:
        return float(value)
    except (TypeError, ValueError):
        return 0.0


def _safe_usage_int(value: object) -> int:
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


def _usage_breakdown(
    *,
    provider: str,
    model: str,
    turns: int,
    token_metered_turns: int,
    token_unmetered_turns: int,
    tokens_in: int,
    tokens_out: int,
    stored_cost_usd: float,
    unpriced_tokens_in: int,
    unpriced_tokens_out: int,
) -> AdminUsageBreakdown:
    """저장된 공급자 비용 추정치와 공식 참조단가를 한 원장 행으로 정규화한다."""

    fallback = estimate_reference_cost(
        provider=provider,
        model=model,
        tokens_in=unpriced_tokens_in,
        tokens_out=unpriced_tokens_out,
    )
    rate_info = fallback or estimate_reference_cost(
        provider=provider,
        model=model,
        tokens_in=tokens_in,
        tokens_out=tokens_out,
    )
    fallback_cost = fallback.cost_usd if fallback is not None else 0.0
    effective_cost = max(0.0, stored_cost_usd) + fallback_cost
    reference_basis = provider_uses_reference_cost(provider)

    if reference_basis:
        recorded_cost = 0.0
        estimated_cost = effective_cost
        basis: UsageCostBasis = (
            "reference_rate" if rate_info is not None or effective_cost > 0 else "unavailable"
        )
    else:
        recorded_cost = max(0.0, stored_cost_usd)
        estimated_cost = fallback_cost
        if recorded_cost > 0:
            basis = "provider_estimate" if provider == "claude_cli" else "provider_reported"
        elif fallback is not None:
            basis = "reference_rate"
        else:
            basis = "unavailable"

    return AdminUsageBreakdown(
        provider=provider,
        model=model,
        turns=max(0, turns),
        token_metered_turns=max(0, token_metered_turns),
        token_unmetered_turns=max(0, token_unmetered_turns),
        tokens_in=max(0, tokens_in),
        tokens_out=max(0, tokens_out),
        cost_usd=round(effective_cost, 6),
        recorded_cost_usd=round(recorded_cost, 6),
        estimated_cost_usd=round(estimated_cost, 6),
        cost_basis=basis,
        rate_label=rate_info.rate_label if rate_info is not None else None,
        rate_source_url=rate_info.source_url if rate_info is not None else None,
    )


def _usage_budget(cost_usd: float) -> AdminUsageBudget:
    limit = max(0.0, float(settings.admin_usage_budget_usd or 0.0))
    if limit <= 0:
        return AdminUsageBudget(
            limit_usd=0.0,
            used_ratio=0.0,
            remaining_usd=None,
            status="disabled",
        )
    used_ratio = max(0.0, cost_usd / limit)
    status_value: UsageBudgetStatus = "ok"
    if used_ratio >= 1.0:
        status_value = "exceeded"
    elif used_ratio >= 0.8:
        status_value = "warn"
    return AdminUsageBudget(
        limit_usd=round(limit, 6),
        used_ratio=round(used_ratio, 4),
        remaining_usd=round(max(0.0, limit - cost_usd), 6),
        status=status_value,
    )


def _usage_evaluator_cache() -> AdminUsageEvaluatorCache:
    stats = evaluator.evaluator_semantic_cache_stats()
    hits = _safe_usage_int(stats.get("hits", 0))
    misses = _safe_usage_int(stats.get("misses", 0))
    requests = hits + misses
    hit_rate = round(hits / requests, 4) if requests else 0.0
    enabled = (
        bool(settings.evaluator_semantic_cache_enabled)
        and settings.evaluator_semantic_cache_ttl_seconds > 0
        and settings.evaluator_semantic_cache_max_entries > 0
    )
    return AdminUsageEvaluatorCache(
        enabled=enabled,
        entries=_safe_usage_int(stats.get("entries", 0)),
        hits=hits,
        misses=misses,
        stores=_safe_usage_int(stats.get("stores", 0)),
        evictions=_safe_usage_int(stats.get("evictions", 0)),
        requests=requests,
        hit_rate=hit_rate,
    )


async def _runtime_health_metrics(*, db_ok: bool) -> RuntimeHealthMetrics:
    metrics = RuntimeHealthMetrics()

    try:
        pool = get_pool()
        metrics.db_pool_size = int(pool.get_size())
        metrics.db_pool_idle = int(pool.get_idle_size())
        metrics.db_pool_max = int(pool.get_max_size())
    except Exception:
        pass

    if not db_ok:
        return metrics

    try:
        async with acquire(role="admin") as conn:
            row = await conn.fetchrow(
                """
                SELECT
                  (SELECT COUNT(*) FROM app.app_user WHERE is_active) AS active_users,
                  (
                    SELECT COUNT(*)
                    FROM app.auth_session
                    WHERE revoked_at IS NULL AND expires_at > now()
                  ) AS active_auth_sessions,
                  (
                    SELECT COUNT(*)
                    FROM app.sessions
                    WHERE ended_at IS NULL
                  ) AS active_sessions,
                  (
                    SELECT COUNT(*)
                    FROM app.sessions
                    WHERE ended_at IS NOT NULL
                  ) AS ended_sessions,
                  (
                    SELECT COUNT(*)
                    FROM app.sessions s
                    LEFT JOIN app.session_summary ss ON ss.session_id = s.id
                    WHERE s.ended_at IS NOT NULL AND ss.session_id IS NULL
                  ) AS pending_reviews
                """
            )
        if row is not None:
            metrics.active_users = int(row["active_users"] or 0)
            metrics.active_auth_sessions = int(row["active_auth_sessions"] or 0)
            metrics.active_sessions = int(row["active_sessions"] or 0)
            metrics.ended_sessions = int(row["ended_sessions"] or 0)
            metrics.pending_reviews = int(row["pending_reviews"] or 0)
    except Exception:
        return metrics

    return metrics


async def _usage_from_database(window_days: int) -> AdminUsageResponse:
    async with acquire(role="admin") as conn:
        total_row = await conn.fetchrow(
            f"""
            SELECT
              COUNT(*) FILTER (WHERE speaker = 'client') AS total_turns,
              COUNT(*) FILTER (WHERE {METERED_CLIENT_TURN_FILTER_SQL}) AS metered_turns,
              COUNT(*) FILTER (
                WHERE {METERED_CLIENT_TURN_FILTER_SQL}
                  AND (COALESCE(tokens_in, 0) > 0 OR COALESCE(tokens_out, 0) > 0)
              ) AS token_metered_turns,
              COUNT(*) FILTER (
                WHERE {METERED_CLIENT_TURN_FILTER_SQL}
                  AND COALESCE(tokens_in, 0) <= 0
                  AND COALESCE(tokens_out, 0) <= 0
              ) AS token_unmetered_turns,
              COALESCE(SUM(tokens_in) FILTER (WHERE speaker = 'client'), 0)::bigint AS tokens_in,
              COALESCE(SUM(tokens_out) FILTER (WHERE speaker = 'client'), 0)::bigint AS tokens_out,
              COALESCE(SUM(cost_usd) FILTER (WHERE speaker = 'client'), 0)::numeric AS cost_usd
            FROM app.turns
            WHERE created_at >= now() - ($1::int * interval '1 day')
            """,
            window_days,
        )
        rows = await conn.fetch(
            f"""
            SELECT
              COALESCE(llm_provider, 'unknown') AS provider,
              COALESCE(model, 'unknown') AS model,
              {USAGE_AGGREGATE_COLUMNS_SQL}
            FROM app.turns
            WHERE created_at >= now() - ($1::int * interval '1 day')
              AND {METERED_CLIENT_TURN_FILTER_SQL}
            GROUP BY 1, 2
            """,
            window_days,
        )
        daily_rows = await conn.fetch(
            f"""
            SELECT
              to_char(date_trunc('day', created_at), 'YYYY-MM-DD') AS day,
              COALESCE(llm_provider, 'unknown') AS provider,
              COALESCE(model, 'unknown') AS model,
              {USAGE_AGGREGATE_COLUMNS_SQL}
            FROM app.turns
            WHERE created_at >= now() - ($1::int * interval '1 day')
              AND {METERED_CLIENT_TURN_FILTER_SQL}
            GROUP BY 1, 2, 3
            ORDER BY 1, 2, 3
            """,
            window_days,
        )

    all_breakdowns = [
        _usage_breakdown(
            provider=str(row["provider"] or "unknown"),
            model=str(row["model"] or "unknown"),
            turns=_safe_usage_int(row["turns"]),
            token_metered_turns=_safe_usage_int(row["token_metered_turns"]),
            token_unmetered_turns=_safe_usage_int(row["token_unmetered_turns"]),
            tokens_in=_safe_usage_int(row["tokens_in"]),
            tokens_out=_safe_usage_int(row["tokens_out"]),
            stored_cost_usd=_decimal_to_float(row["cost_usd"]),
            unpriced_tokens_in=_safe_usage_int(row["unpriced_tokens_in"]),
            unpriced_tokens_out=_safe_usage_int(row["unpriced_tokens_out"]),
        )
        for row in rows
    ]
    all_breakdowns.sort(
        key=lambda item: (
            -item.cost_usd,
            -(item.tokens_in + item.tokens_out),
            -item.turns,
            item.provider,
            item.model,
        )
    )
    recorded_cost = round(sum(item.recorded_cost_usd for item in all_breakdowns), 6)
    estimated_cost = round(sum(item.estimated_cost_usd for item in all_breakdowns), 6)
    total_cost = round(recorded_cost + estimated_cost, 6)

    daily_buckets: dict[str, dict[str, int | float]] = {}
    for row in daily_rows:
        breakdown = _usage_breakdown(
            provider=str(row["provider"] or "unknown"),
            model=str(row["model"] or "unknown"),
            turns=_safe_usage_int(row["turns"]),
            token_metered_turns=_safe_usage_int(row["token_metered_turns"]),
            token_unmetered_turns=_safe_usage_int(row["token_unmetered_turns"]),
            tokens_in=_safe_usage_int(row["tokens_in"]),
            tokens_out=_safe_usage_int(row["tokens_out"]),
            stored_cost_usd=_decimal_to_float(row["cost_usd"]),
            unpriced_tokens_in=_safe_usage_int(row["unpriced_tokens_in"]),
            unpriced_tokens_out=_safe_usage_int(row["unpriced_tokens_out"]),
        )
        day = str(row["day"])
        bucket = daily_buckets.setdefault(
            day,
            {"turns": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0},
        )
        bucket["turns"] = int(bucket["turns"]) + breakdown.turns
        bucket["tokens_in"] = int(bucket["tokens_in"]) + breakdown.tokens_in
        bucket["tokens_out"] = int(bucket["tokens_out"]) + breakdown.tokens_out
        bucket["cost_usd"] = float(bucket["cost_usd"]) + breakdown.cost_usd

    return AdminUsageResponse(
        source="database",
        durable=True,
        window_days=window_days,
        generated_at=time.time(),
        total_turns=_safe_usage_int(total_row["total_turns"] if total_row else 0),
        metered_turns=_safe_usage_int(total_row["metered_turns"] if total_row else 0),
        token_metered_turns=_safe_usage_int(
            total_row["token_metered_turns"] if total_row else 0
        ),
        token_unmetered_turns=_safe_usage_int(
            total_row["token_unmetered_turns"] if total_row else 0
        ),
        tokens_in=_safe_usage_int(total_row["tokens_in"] if total_row else 0),
        tokens_out=_safe_usage_int(total_row["tokens_out"] if total_row else 0),
        cost_usd=total_cost,
        recorded_cost_usd=recorded_cost,
        estimated_cost_usd=estimated_cost,
        budget=_usage_budget(total_cost),
        evaluator_cache=_usage_evaluator_cache(),
        by_provider=all_breakdowns[:12],
        daily_cost=[
            AdminUsageDailyCost(
                day=day,
                turns=int(values["turns"]),
                tokens_in=int(values["tokens_in"]),
                tokens_out=int(values["tokens_out"]),
                cost_usd=round(float(values["cost_usd"]), 6),
            )
            for day, values in sorted(daily_buckets.items())
        ],
    )


async def _record_health_events(
    *,
    principal: Principal | None,
    overall_status: HealthStatus,
    environment: str,
    engine_mode: str,
    services: list[AdminServiceHealth],
) -> int:
    if not services:
        return 0
    captured_by = principal.user_id if principal is not None else None
    try:
        async with acquire(role="admin", user_id=captured_by) as conn:
            await conn.executemany(
                """
                INSERT INTO app.admin_health_event (
                  overall_status,
                  environment,
                  engine_mode,
                  service_key,
                  service_name,
                  service_status,
                  detail,
                  metric,
                  load,
                  captured_by
                )
                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::uuid)
                """,
                [
                    (
                        overall_status,
                        environment,
                        engine_mode,
                        service.key,
                        service.name,
                        service.status,
                        service.detail,
                        service.metric,
                        service.load,
                        captured_by,
                    )
                    for service in services
                ],
            )
            return len(services)
    except Exception:
        # 헬스 화면 자체가 장애 확인 경로라, 이력 적재 실패가 응답을 막으면 안 된다.
        return 0


async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
    async with acquire(role="admin") as conn:
        rows = await conn.fetch(
            """
            SELECT
              id,
              observed_at,
              overall_status,
              service_key,
              service_name,
              service_status,
              detail,
              metric,
              load
            FROM app.admin_health_event
            WHERE observed_at >= now() - ($1::int * interval '1 hour')
            ORDER BY observed_at DESC, id DESC
            LIMIT 240
            """,
            window_hours,
        )
        rollup_rows = await conn.fetch(
            """
            SELECT
              r.service_key,
              r.service_name,
              r.sample_count,
              r.ok_samples,
              r.degraded_samples,
              r.down_samples,
              r.latest_status,
              r.last_observed_at,
              r.last_down_at
            FROM app.admin_health_daily_rollup AS r
            WHERE r.rollup_date >= (now() - ($1::int * interval '1 hour'))::date
              AND r.rollup_date < current_date
              AND NOT EXISTS (
                SELECT 1
                FROM app.admin_health_event AS e
                WHERE e.observed_at::date = r.rollup_date
                  AND e.environment = r.environment
                  AND e.engine_mode = r.engine_mode
                  AND e.service_key = r.service_key
              )
            ORDER BY r.last_observed_at DESC, r.service_key
            LIMIT 1000
            """,
            window_hours,
        )

    events = [_health_event_from_row(row) for row in rows]
    service_buckets: dict[str, AdminUptimeServiceSummary] = {}

    def ensure_bucket(
        *,
        service_key: str,
        service_name: str,
        latest_status: HealthStatus,
        latest_observed_at: float | None,
    ) -> AdminUptimeServiceSummary:
        current = service_buckets.get(service_key)
        if current is None:
            current = AdminUptimeServiceSummary(
                service_key=service_key,
                service_name=service_name,
                samples=0,
                ok_samples=0,
                degraded_samples=0,
                down_samples=0,
                latest_status=latest_status,
                latest_observed_at=latest_observed_at,
            )
            service_buckets[service_key] = current
            return current
        if latest_observed_at is not None and (
            current.latest_observed_at is None
            or latest_observed_at > current.latest_observed_at
        ):
            current.latest_status = latest_status
            current.latest_observed_at = latest_observed_at
        return current

    for event in events:
        current = ensure_bucket(
            service_key=event.service_key,
            service_name=event.service_name,
            latest_status=event.service_status,
            latest_observed_at=event.observed_at,
        )
        current.samples += 1
        if event.service_status == "ok":
            current.ok_samples += 1
        elif event.service_status == "degraded":
            current.degraded_samples += 1
        else:
            current.down_samples += 1

    rollup_last_down: list[float] = []
    for row in rollup_rows:
        latest_observed_at = _row_ts(row["last_observed_at"])
        current = ensure_bucket(
            service_key=str(row["service_key"]),
            service_name=str(row["service_name"]),
            latest_status=row["latest_status"],
            latest_observed_at=latest_observed_at,
        )
        current.samples += int(row["sample_count"] or 0)
        current.ok_samples += int(row["ok_samples"] or 0)
        current.degraded_samples += int(row["degraded_samples"] or 0)
        current.down_samples += int(row["down_samples"] or 0)
        last_down = _row_ts(row["last_down_at"])
        if last_down is not None:
            rollup_last_down.append(last_down)

    ok_samples = sum(1 for event in events if event.service_status == "ok") + sum(
        int(row["ok_samples"] or 0) for row in rollup_rows
    )
    sample_count = len(events) + sum(int(row["sample_count"] or 0) for row in rollup_rows)
    down_times = [event.observed_at for event in events if event.service_status == "down"]
    down_times.extend(rollup_last_down)
    return AdminUptimeResponse(
        source="database",
        durable=True,
        window_hours=window_hours,
        generated_at=time.time(),
        sample_count=sample_count,
        ok_ratio=round(ok_samples / sample_count, 4) if sample_count else 0.0,
        degraded_events=sum(1 for event in events if event.service_status == "degraded")
        + sum(int(row["degraded_samples"] or 0) for row in rollup_rows),
        down_events=sum(1 for event in events if event.service_status == "down")
        + sum(int(row["down_samples"] or 0) for row in rollup_rows),
        last_down_at=max(down_times) if down_times else None,
        services=sorted(
            service_buckets.values(),
            key=lambda item: (item.latest_status != "down", item.latest_status != "degraded", item.service_key),
        ),
        events=events,
    )


async def _tickets_from_database(
    *,
    ticket_status: TicketStatus | None,
    category: TicketCategory | None = None,
    priority: TicketPriority | None = None,
    assigned_group: str | None = None,
    source_path: str | None = None,
    stale_only: bool = False,
    search: str = "",
    window_days: int,
) -> AdminTicketsResponse:
    assigned_group_filter = (assigned_group or "").strip()
    source_path_filter = (source_path or "").strip()
    search_filter = search.strip().lower()
    async with acquire(role="admin") as conn:
        rows = await conn.fetch(
            SUPPORT_TICKET_DETAIL_FROM_SQL
            + """
            WHERE ($1::text IS NULL OR status = $1)
              AND ($2::text IS NULL OR category = $2)
              AND ($3::text IS NULL OR priority = $3)
              AND ($4::text = '' OR assigned_group = $4)
              AND ($5::text = '' OR source_path = $5)
              AND (
                NOT $6::bool
                OR (
                  status NOT IN ('resolved', 'closed')
                  AND updated_at < now() - interval '1 day'
                )
              )
              AND (
                $7::text = ''
                OR lower(subject) LIKE '%' || $7 || '%'
                OR lower(body) LIKE '%' || $7 || '%'
                OR lower(source_path) LIKE '%' || $7 || '%'
                OR lower(reporter_email) LIKE '%' || $7 || '%'
              )
              AND created_at >= now() - ($8::int * interval '1 day')
            ORDER BY
              CASE
                WHEN status = 'open' THEN 0
                WHEN status = 'triaged' THEN 1
                WHEN status = 'in_progress' THEN 2
                WHEN status = 'resolved' THEN 3
                ELSE 4
              END,
              CASE
                WHEN priority = 'urgent' THEN 0
                WHEN priority = 'high' THEN 1
                WHEN priority = 'normal' THEN 2
                ELSE 3
              END,
              updated_at DESC
            LIMIT 120
            """,
            ticket_status,
            category,
            priority,
            assigned_group_filter,
            source_path_filter,
            stale_only,
            search_filter,
            window_days,
        )
    tickets = [_ticket_from_row(row) for row in rows]
    return AdminTicketsResponse(
        source="database",
        durable=True,
        generated_at=time.time(),
        tickets=tickets,
        summary=_ticket_summary(tickets),
    )


def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
    window_start = time.time() - (window_days * 86400)
    total_turns = 0
    metered_turns = 0
    token_metered_turns = 0
    token_unmetered_turns = 0
    tokens_in = 0
    tokens_out = 0
    cost_usd = 0.0
    recorded_cost_usd = 0.0
    estimated_cost_usd = 0.0
    buckets: dict[tuple[str, str], dict[str, int | float]] = {}
    daily_buckets: dict[str, dict[str, int | float]] = {}

    for sess in store.list():
        for turn in getattr(sess, "turns", []) or []:
            if getattr(turn, "speaker", "") != "client":
                continue
            created_at = float(getattr(turn, "created_at", 0.0) or 0.0)
            if created_at < window_start:
                continue
            total_turns += 1
            provider = str(getattr(turn, "llm_provider", None) or "unknown")
            model = str(getattr(turn, "model", None) or "unknown")
            turn_tokens_in = _safe_usage_int(getattr(turn, "tokens_in", 0))
            turn_tokens_out = _safe_usage_int(getattr(turn, "tokens_out", 0))
            turn_cost = _decimal_to_float(getattr(turn, "cost_usd", 0.0))
            is_metered = (
                provider != "unknown"
                or model != "unknown"
                or turn_tokens_in > 0
                or turn_tokens_out > 0
                or turn_cost > 0
            )
            if not is_metered:
                continue
            metered_turns += 1
            is_token_metered = turn_tokens_in > 0 or turn_tokens_out > 0
            if is_token_metered:
                token_metered_turns += 1
            else:
                token_unmetered_turns += 1
            tokens_in += turn_tokens_in
            tokens_out += turn_tokens_out
            turn_breakdown = _usage_breakdown(
                provider=provider,
                model=model,
                turns=1,
                token_metered_turns=1 if is_token_metered else 0,
                token_unmetered_turns=0 if is_token_metered else 1,
                tokens_in=turn_tokens_in,
                tokens_out=turn_tokens_out,
                stored_cost_usd=turn_cost,
                unpriced_tokens_in=turn_tokens_in if turn_cost <= 0 else 0,
                unpriced_tokens_out=turn_tokens_out if turn_cost <= 0 else 0,
            )
            cost_usd += turn_breakdown.cost_usd
            recorded_cost_usd += turn_breakdown.recorded_cost_usd
            estimated_cost_usd += turn_breakdown.estimated_cost_usd
            key = (provider, model)
            bucket = buckets.setdefault(
                key,
                {
                    "turns": 0,
                    "token_metered_turns": 0,
                    "token_unmetered_turns": 0,
                    "tokens_in": 0,
                    "tokens_out": 0,
                    "stored_cost_usd": 0.0,
                    "unpriced_tokens_in": 0,
                    "unpriced_tokens_out": 0,
                },
            )
            bucket["turns"] = int(bucket["turns"]) + 1
            if is_token_metered:
                bucket["token_metered_turns"] = int(bucket["token_metered_turns"]) + 1
            else:
                bucket["token_unmetered_turns"] = int(bucket["token_unmetered_turns"]) + 1
            bucket["tokens_in"] = int(bucket["tokens_in"]) + turn_tokens_in
            bucket["tokens_out"] = int(bucket["tokens_out"]) + turn_tokens_out
            bucket["stored_cost_usd"] = float(bucket["stored_cost_usd"]) + turn_cost
            if turn_cost <= 0:
                bucket["unpriced_tokens_in"] = (
                    int(bucket["unpriced_tokens_in"]) + turn_tokens_in
                )
                bucket["unpriced_tokens_out"] = (
                    int(bucket["unpriced_tokens_out"]) + turn_tokens_out
                )
            day = datetime.fromtimestamp(created_at, timezone.utc).strftime("%Y-%m-%d")
            daily_bucket = daily_buckets.setdefault(
                day,
                {"turns": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0},
            )
            daily_bucket["turns"] = int(daily_bucket["turns"]) + 1
            daily_bucket["tokens_in"] = int(daily_bucket["tokens_in"]) + turn_tokens_in
            daily_bucket["tokens_out"] = int(daily_bucket["tokens_out"]) + turn_tokens_out
            daily_bucket["cost_usd"] = (
                float(daily_bucket["cost_usd"]) + turn_breakdown.cost_usd
            )

    by_provider = [
        _usage_breakdown(
            provider=provider,
            model=model,
            turns=int(values["turns"]),
            token_metered_turns=int(values["token_metered_turns"]),
            token_unmetered_turns=int(values["token_unmetered_turns"]),
            tokens_in=int(values["tokens_in"]),
            tokens_out=int(values["tokens_out"]),
            stored_cost_usd=float(values["stored_cost_usd"]),
            unpriced_tokens_in=int(values["unpriced_tokens_in"]),
            unpriced_tokens_out=int(values["unpriced_tokens_out"]),
        )
        for (provider, model), values in buckets.items()
    ]
    by_provider.sort(
        key=lambda item: (
            -item.cost_usd,
            -(item.tokens_in + item.tokens_out),
            -item.turns,
            item.provider,
            item.model,
        )
    )

    total_cost = round(cost_usd, 6)
    return AdminUsageResponse(
        source="server_session_registry",
        durable=False,
        window_days=window_days,
        generated_at=time.time(),
        total_turns=total_turns,
        metered_turns=metered_turns,
        token_metered_turns=token_metered_turns,
        token_unmetered_turns=token_unmetered_turns,
        tokens_in=tokens_in,
        tokens_out=tokens_out,
        cost_usd=total_cost,
        recorded_cost_usd=round(recorded_cost_usd, 6),
        estimated_cost_usd=round(estimated_cost_usd, 6),
        budget=_usage_budget(total_cost),
        evaluator_cache=_usage_evaluator_cache(),
        by_provider=by_provider[:12],
        daily_cost=[
            AdminUsageDailyCost(
                day=day,
                turns=int(values["turns"]),
                tokens_in=int(values["tokens_in"]),
                tokens_out=int(values["tokens_out"]),
                cost_usd=round(float(values["cost_usd"]), 6),
            )
            for day, values in sorted(daily_buckets.items())
        ],
    )


class AdminUserCreate(BaseModel):
    email: str = Field(
        ...,
        min_length=3,
        max_length=254,
        pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$",
    )
    display_name: str = Field(..., min_length=1, max_length=80)
    role: RoleName = "learner"
    admin_access: bool = False
    learner_feedback_enabled: bool = True
    # 외부 연구참여자는 exact-email 사전등록 뒤 별도 승인을 거치게 한다.
    account_status: Literal["pending"] = "pending"
    affiliation: str | None = Field(default=None, max_length=120)
    cohort_ids: list[str] = Field(default_factory=list)


class AdminUserDeleteResponse(BaseModel):
    ok: bool
    user_id: str


_ENGINE_CONFIG: AdminEngineConfigResponse | None = None
ENGINE_MODES = set(ENGINE_PROVIDERS)
ENGINE_MODE_ALIASES = {"messages_api": "claude_api"}


def _normalize_email(value: str) -> str:
    email = value.strip().lower()
    if "@" not in email:
        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
    local, domain = email.rsplit("@", 1)
    if not local or not domain:
        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
    return email


def _default_engine_config() -> AdminEngineConfigResponse:
    default_model, default_effort = ENGINE_PROVIDER_DEFAULTS[settings.engine_mode]
    return AdminEngineConfigResponse(
        engine_mode=settings.engine_mode,
        engine_url=settings.engine_url,
        model=default_model,
        reasoning_effort=default_effort,
        durable=False,
        source="runtime_default",
    )


def _normalize_engine_mode(value: str) -> EngineProvider:
    mode = ENGINE_MODE_ALIASES.get(value.strip(), value.strip())
    if mode not in ENGINE_MODES:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"unsupported engine mode {value}",
        )
    return cast(EngineProvider, mode)


def _normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
    effort = (value or "").strip().lower()
    if not effort:
        return None
    if effort not in ENGINE_REASONING_EFFORTS:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"unsupported reasoning effort {value}",
        )
    return cast(ReasoningEffort, effort)


def _normalize_engine_url(value: str) -> str:
    url = value.strip().rstrip("/")
    if not (url.startswith("http://") or url.startswith("https://")):
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="engine_url must start with http:// or https://",
        )
    return url


def _updated_at_ts(value: datetime | None) -> float | None:
    if value is None:
        return None
    if value.tzinfo is None:
        value = value.replace(tzinfo=timezone.utc)
    return value.timestamp()


def _row_ts(value: object) -> float | None:
    if value is None:
        return None
    if isinstance(value, datetime):
        return _updated_at_ts(value)
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def _row_value(row, key: str, default=None):
    try:
        return row[key]
    except (IndexError, KeyError, TypeError):
        return default


def _notification_delivery_response(row) -> AdminNotificationDeliveryResponse:
    return AdminNotificationDeliveryResponse(
        id=str(row["id"]),
        event_id=str(row["event_id"]),
        kind=str(row["kind"]),
        recipient_email=str(row["recipient_email"]),
        recipient_name=str(row["recipient_name"] or ""),
        recipient_role=str(row["recipient_role"] or ""),
        subject=str(row["subject"] or ""),
        status=row["status"],
        attempts=int(row["attempts"] or 0),
        last_error=str(row["last_error"] or "") or None,
        provider_message_id=str(row["provider_message_id"] or "") or None,
        created_at=_row_ts(row["created_at"]),
        updated_at=_row_ts(row["updated_at"]),
        sent_at=_row_ts(row["sent_at"]),
    )


def _engine_config_from_row(row) -> AdminEngineConfigResponse:
    return AdminEngineConfigResponse(
        engine_mode=_normalize_engine_mode(row["engine_mode"]),
        engine_url=_normalize_engine_url(row["engine_url"]),
        model=row["model"],
        reasoning_effort=_normalize_reasoning_effort(row.get("reasoning_effort")),
        updated_by=row["updated_by"],
        updated_at=_updated_at_ts(row["updated_at"]),
        durable=True,
        source="database",
    )


def _health_event_from_row(row) -> AdminHealthEvent:
    return AdminHealthEvent(
        event_id=int(row["id"]),
        observed_at=_row_ts(row["observed_at"]) or 0.0,
        overall_status=row["overall_status"],
        service_key=row["service_key"],
        service_name=row["service_name"],
        service_status=row["service_status"],
        detail=row["detail"],
        metric=row["metric"],
        load=_clamp01(float(row["load"] or 0.0)),
    )


def _ticket_from_row(row) -> AdminSupportTicketResponse:
    return AdminSupportTicketResponse(
        ticket_id=str(row["id"]),
        reporter=AdminTicketReporter(
            user_id=str(row["reporter_id"]) if row["reporter_id"] else None,
            email=row["reporter_email"],
            display_name=row["reporter_name"],
            role=row["reporter_role"],
        ),
        category=row["category"],
        priority=row["priority"],
        status=row["status"],
        subject=row["subject"],
        body=row["body"],
        source_path=row["source_path"],
        fingerprint=_row_value(row, "fingerprint", "") or "",
        parent_ticket_id=(
            str(_row_value(row, "parent_ticket_id"))
            if _row_value(row, "parent_ticket_id") is not None
            else None
        ),
        duplicate_count=int(_row_value(row, "duplicate_count", 0) or 0),
        duplicate_parent_candidate_id=(
            str(_row_value(row, "duplicate_parent_candidate_id"))
            if _row_value(row, "duplicate_parent_candidate_id") is not None
            else None
        ),
        child_ticket_count=int(_row_value(row, "child_ticket_count", 0) or 0),
        assigned_group=row["assigned_group"],
        resolution_note=row["resolution_note"],
        created_at=_row_ts(row["created_at"]) or 0.0,
        updated_at=_row_ts(row["updated_at"]) or 0.0,
        resolved_at=_row_ts(row["resolved_at"]),
        event_count=int(_row_value(row, "event_count", 0) or 0),
        last_event_at=_row_ts(_row_value(row, "last_event_at")),
    )


def _empty_ticket_summary() -> AdminTicketSummary:
    return AdminTicketSummary(
        total=0,
        open_count=0,
        high_priority_count=0,
        stale_count=0,
        by_status={},
        by_category={},
        by_priority={},
    )


def _ticket_summary(tickets: list[AdminSupportTicketResponse]) -> AdminTicketSummary:
    now = time.time()
    by_status: dict[str, int] = {}
    by_category: dict[str, int] = {}
    by_priority: dict[str, int] = {}
    active_tickets = [ticket for ticket in tickets if ticket.status not in {"resolved", "closed"}]
    for ticket in tickets:
        by_status[ticket.status] = by_status.get(ticket.status, 0) + 1
        by_category[ticket.category] = by_category.get(ticket.category, 0) + 1
        by_priority[ticket.priority] = by_priority.get(ticket.priority, 0) + 1
    return AdminTicketSummary(
        total=len(tickets),
        open_count=len(active_tickets),
        high_priority_count=sum(
            1 for ticket in active_tickets if ticket.priority in {"high", "urgent"}
        ),
        stale_count=sum(
            1
            for ticket in active_tickets
            if now - ticket.updated_at > 86400
        ),
        by_status=by_status,
        by_category=by_category,
        by_priority=by_priority,
    )


def _unavailable_uptime(window_hours: int) -> AdminUptimeResponse:
    return AdminUptimeResponse(
        source="unavailable",
        durable=False,
        window_hours=window_hours,
        generated_at=time.time(),
        sample_count=0,
        ok_ratio=0.0,
        degraded_events=0,
        down_events=0,
        last_down_at=None,
        services=[],
        events=[],
    )


def _unavailable_tickets() -> AdminTicketsResponse:
    return AdminTicketsResponse(
        source="unavailable",
        durable=False,
        generated_at=time.time(),
        tickets=[],
        summary=_empty_ticket_summary(),
    )


def _normalize_parent_ticket_id(value: str | None) -> str | None:
    if value is None:
        return None
    stripped = value.strip()
    if not stripped:
        return None
    try:
        return str(UUID(stripped))
    except ValueError as exc:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="invalid parent ticket id",
        ) from exc


def _ticket_change_detail(old_row, new_row) -> dict[str, object]:
    changed_fields: list[str] = []
    detail: dict[str, object] = {"changed_fields": changed_fields}
    for field in ("status", "priority", "assigned_group", "parent_ticket_id"):
        before = _row_value(old_row, field, "")
        after = _row_value(new_row, field, "")
        if before is not None:
            before = str(before)
        if after is not None:
            after = str(after)
        if before != after:
            changed_fields.append(field)
            detail[field] = {"from": before, "to": after}
    old_note = (_row_value(old_row, "resolution_note", "") or "").strip()
    new_note = (_row_value(new_row, "resolution_note", "") or "").strip()
    if old_note != new_note:
        changed_fields.append("resolution_note")
        detail["resolution_note"] = {
            "from_present": bool(old_note),
            "to_present": bool(new_note),
        }
    detail["category"] = _row_value(new_row, "category", "")
    detail["source_path"] = _row_value(new_row, "source_path", "")
    return detail


async def _record_ticket_update_audit(
    conn,
    *,
    principal: Principal,
    ticket_id: str,
    detail: dict[str, object],
) -> None:
    await conn.execute(
        """
        INSERT INTO audit.audit_log (
          actor_uid, action, target_kind, target_id, detail
        )
        VALUES ($1::uuid, $2, $3, $4, $5::jsonb)
        """,
        principal.user_id,
        "support_ticket_update",
        "support_ticket",
        ticket_id,
        detail,
    )


async def _current_engine_config() -> AdminEngineConfigResponse:
    if _ENGINE_CONFIG is not None:
        return _ENGINE_CONFIG
    try:
        pool = get_pool()
        async with pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
                FROM app.admin_engine_config
                WHERE id = TRUE
                """
            )
            if row is not None:
                return _engine_config_from_row(row)
            return _default_engine_config()
    except Exception:
        require_runtime_fallback_allowed("admin engine config")
    return _default_engine_config()


async def apply_engine_config_from_store() -> AdminEngineConfigResponse:
    """Load admin engine settings and apply them to the live engine client."""
    config = await _current_engine_config()
    await engine_client.configure(
        base_url=config.engine_url,
        engine_mode=config.engine_mode,
        default_model=config.model,
        default_reasoning_effort=config.reasoning_effort,
    )
    return config


def _overall_status(services: list[AdminServiceHealth]) -> HealthStatus:
    if any(s.status == "down" for s in services):
        return "down"
    if any(s.status == "degraded" for s in services):
        return "degraded"
    return "ok"


def _engine_unavailable_detail(detail: str) -> str:
    if detail.lstrip().startswith("{") and '"ok":false' in detail:
        return "Engine readiness failed"
    return detail


def _requires_super_admin_for_privilege_change(
    *,
    current_role: RoleName | None,
    next_role: RoleName | None,
    current_admin_access: bool,
    next_admin_access: bool | None,
) -> bool:
    if next_admin_access is not None and next_admin_access != current_admin_access:
        return True
    if next_role == "admin" and current_role != "admin":
        return True
    if current_role == "admin" and next_role is not None and next_role != "admin":
        return True
    return False


def _assert_super_admin(principal: Principal) -> None:
    if not principal.super_admin:
        raise HTTPException(status.HTTP_403_FORBIDDEN, detail="super admin required")


def _assert_super_admin_target_mutable(email: str, admin_access: bool | None) -> None:
    if is_super_admin_email(email) and admin_access is False:
        raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot revoke super admin access")


async def _admin_user_response(user, *, durable: bool) -> AdminUserResponse:
    effective_admin_access = has_admin_access(user.email, user.role, user.admin_access)
    return AdminUserResponse(
        user_id=user.user_id,
        email=user.email,
        display_name=user.display_name,
        role=user.role,
        admin_access=effective_admin_access,
        learner_feedback_enabled=user.learner_feedback_enabled,
        super_admin=is_super_admin_email(user.email),
        account_status=user.account_status,
        cohort_ids=user.cohort_ids,
        affiliation=user.affiliation,
        active_sessions=await active_session_count(user.user_id),
        created_at=user.created_at,
        last_seen_at=user.last_seen_at,
        source="database" if durable else "server_session_registry",
    )


async def _build_admin_health_response() -> AdminHealthResponse:
    current_engine = await _current_engine_config()
    db_ok = await healthcheck()
    engine_started = time.perf_counter()
    engine = await engine_client.health_detail()
    engine_ok = bool(engine.get("ok"))
    engine_detail = _engine_unavailable_detail(
        str(engine.get("detail") or "engine readiness unavailable")
    )
    engine_latency_ms = (time.perf_counter() - engine_started) * 1000
    voice_ok = voice_service.is_available()
    metrics = await _runtime_health_metrics(db_ok=db_ok)
    metrics.engine_latency_ms = engine_latency_ms if engine_ok else None
    pool_load = _pool_load(metrics)
    session_load = _workload_load(metrics.active_sessions, 50)
    review_load = _workload_load(metrics.pending_reviews, 50)
    runtime_fallback_is_enabled = settings.environment == "dev"
    db_status: HealthStatus = "ok" if db_ok else ("degraded" if runtime_fallback_is_enabled else "down")
    db_detail = (
        "사용자, 세션, 리뷰 저장"
        if db_ok
        else (
            "DB 연결 전까지 비영구 개발 런타임 기록 사용"
            if runtime_fallback_is_enabled
            else "DB 저장소에 연결할 수 없습니다"
        )
    )
    db_metric = (
        f"풀 {max(0, metrics.db_pool_size - metrics.db_pool_idle)}/{metrics.db_pool_max}"
        if db_ok
        else ("비영구 런타임 기록" if runtime_fallback_is_enabled else "저장소 중단")
    )

    services = [
        AdminServiceHealth(
            key="engine",
            name="응답 생성",
            status="ok" if engine_ok else "down",
            detail="AI 엔진 생성 준비 완료" if engine_ok else engine_detail,
            metric=f"{engine_latency_ms:.0f}ms" if engine_ok else "로그인/설정 필요",
            load=_clamp01(engine_latency_ms / 1500) if engine_ok else 0.0,
        ),
        AdminServiceHealth(
            key="db",
            name="영구 저장소",
            status=db_status,
            detail=db_detail,
            metric=db_metric,
            load=max(pool_load, session_load) if db_ok else 0.0,
        ),
        AdminServiceHealth(
            key="voice",
            name="음성 입력",
            status="ok" if voice_ok else "degraded",
            detail="음성 입력과 재생",
            metric="OpenAI 연결" if voice_ok else "설정 필요",
            load=0.05 if voice_ok else 0.0,
        ),
        AdminServiceHealth(
            key="evaluation",
            name="리뷰 생성",
            status="ok" if engine_ok else "degraded",
            detail="회기 종료 후 피드백 생성",
            metric=f"대기 {metrics.pending_reviews}건",
            load=review_load if engine_ok else 0.0,
        ),
        AdminServiceHealth(
            key="kb",
            name="지식 검색",
            status="ok" if db_ok else "degraded",
            detail=f"검색 기준값 {rag.CRAG_TOP1_THRESHOLD}",
            metric=(
                f"활성 세션 {metrics.active_sessions}건"
                if db_ok
                else "대기 중"
            ),
            load=max(pool_load, session_load) if db_ok else 0.0,
        ),
    ]

    response = AdminHealthResponse(
        status=_overall_status(services),
        environment=settings.environment,
        engine_mode=current_engine.engine_mode,
        services=services,
    )
    return response


async def record_admin_health_sample(
    *, principal: Principal | None = None
) -> tuple[AdminHealthResponse, int]:
    """Collect and persist one synthetic/admin health sample."""
    response = await _build_admin_health_response()
    recorded_count = await _record_health_events(
        principal=principal,
        overall_status=response.status,
        environment=response.environment,
        engine_mode=response.engine_mode,
        services=response.services,
    )
    return response, recorded_count


@router.get("/health", response_model=AdminHealthResponse)
async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
    """Return operational health from live backend checks."""
    response, _ = await record_admin_health_sample(principal=principal)
    return response


@router.get("/voice-runtime", response_model=VoiceRuntimeSnapshot)
async def admin_voice_runtime(
    principal: AdminPrincipal,
) -> VoiceRuntimeSnapshot:
    """Return one API worker's metadata-only voice high-water snapshot."""

    return voice_runtime_metrics.snapshot()


@router.get("/usage", response_model=AdminUsageResponse)
async def admin_usage(
    principal: AdminPrincipal,
    window_days: Annotated[int, Query(ge=1, le=90)] = 7,
) -> AdminUsageResponse:
    """Return AI token/cost usage from persisted turns or dev fallback state."""
    try:
        return await _usage_from_database(window_days)
    except Exception:
        require_runtime_fallback_allowed("admin usage")
        return _usage_from_runtime_store(window_days)


@router.get("/uptime", response_model=AdminUptimeResponse)
async def admin_uptime(
    principal: AdminPrincipal,
    window_hours: Annotated[int, Query(ge=1, le=720)] = 24,
) -> AdminUptimeResponse:
    """Return persisted health check samples captured by the admin console."""
    try:
        return await _uptime_from_database(window_hours)
    except Exception:
        return _unavailable_uptime(window_hours)


@router.get("/tickets", response_model=AdminTicketsResponse)
async def list_tickets(
    principal: AdminPrincipal,
    ticket_status: Annotated[TicketStatus | None, Query(alias="status")] = None,
    category: TicketCategory | None = None,
    priority: TicketPriority | None = None,
    assigned_group: Annotated[str | None, Query(max_length=120)] = None,
    source_path: Annotated[str | None, Query(max_length=300)] = None,
    stale_only: bool = False,
    search: Annotated[str, Query(max_length=120)] = "",
    window_days: Annotated[int, Query(ge=1, le=365)] = 30,
) -> AdminTicketsResponse:
    """Return user-submitted operational tickets without synthetic fallback rows."""
    try:
        return await _tickets_from_database(
            ticket_status=ticket_status,
            category=category,
            priority=priority,
            assigned_group=assigned_group,
            source_path=source_path,
            stale_only=stale_only,
            search=search,
            window_days=window_days,
        )
    except Exception:
        return _unavailable_tickets()


@router.get("/notifications", response_model=AdminNotificationsResponse)
async def list_notifications(
    principal: AdminPrincipal,
    limit: Annotated[int, Query(ge=1, le=200)] = 50,
) -> AdminNotificationsResponse:
    """Return recent operational email delivery state."""
    try:
        get_pool()
        async with acquire(role="admin") as conn:
            status_rows = await conn.fetch(
                """
                SELECT status, COUNT(*) AS count
                FROM app.notification_delivery
                GROUP BY status
                """
            )
            rows = await conn.fetch(
                """
                SELECT
                  d.id,
                  d.event_id,
                  e.kind,
                  d.recipient_email,
                  d.recipient_name,
                  d.recipient_role,
                  d.subject,
                  d.status,
                  d.attempts,
                  d.last_error,
                  d.provider_message_id,
                  d.created_at,
                  d.updated_at,
                  d.sent_at
                FROM app.notification_delivery d
                JOIN app.notification_event e ON e.id = d.event_id
                ORDER BY d.created_at DESC
                LIMIT $1
                """,
                limit,
            )
    except Exception:
        require_runtime_fallback_allowed("admin notifications")
        status_rows = []
        rows = []
    counts = {str(row["status"]): int(row["count"] or 0) for row in status_rows}
    return AdminNotificationsResponse(
        provider=settings.notification_email_provider,
        smtp_configured=bool(settings.smtp_host and settings.smtp_from_email),
        queued=counts.get("queued", 0),
        failed=counts.get("failed", 0),
        sent=counts.get("sent", 0),
        skipped=counts.get("skipped", 0),
        deliveries=[_notification_delivery_response(row) for row in rows],
    )


@router.post("/notifications/process", response_model=AdminNotificationProcessResponse)
async def process_notifications(
    principal: AdminPrincipal,
    limit: Annotated[int, Query(ge=1, le=100)] = 25,
) -> AdminNotificationProcessResponse:
    """Drain queued email notifications once from the admin console/API."""
    try:
        result = await notifications.process_queued_email_notifications(limit=limit)
    except Exception:
        require_runtime_fallback_allowed("admin notification processing")
        result = {"processed": 0, "sent": 0, "failed": 0, "skipped": 0}
    return AdminNotificationProcessResponse(**result)


@router.post("/notifications/test", response_model=AdminNotificationTestResponse)
async def send_test_notification(
    principal: AdminPrincipal,
) -> AdminNotificationTestResponse:
    """Queue and process one explicit admin email test."""
    recipient_count = 0
    try:
        recipient_count = await notifications.enqueue_admin_test_email(
            actor_user_id=principal.user_id,
            actor_email=principal.email,
        )
        result = await notifications.process_queued_email_notifications(
            limit=max(1, min(100, recipient_count or 1))
        )
    except Exception:
        require_runtime_fallback_allowed("admin notification test")
        result = {"processed": 0, "sent": 0, "failed": 0, "skipped": 0}
    return AdminNotificationTestResponse(
        provider=settings.notification_email_provider,
        smtp_configured=bool(settings.smtp_host and settings.smtp_from_email),
        recipients=recipient_count,
        **result,
    )


@router.patch("/tickets/{ticket_id}", response_model=AdminSupportTicketResponse)
async def patch_ticket(
    ticket_id: str,
    body: AdminTicketPatch,
    principal: AdminPrincipal,
) -> AdminSupportTicketResponse:
    """Update ticket triage state for administrators."""
    try:
        async with acquire(role="admin", user_id=principal.user_id) as conn:
            old_row = await conn.fetchrow(
                """
                SELECT
                  id,
                  category,
                  priority,
                  status,
                  source_path,
                  assigned_group,
                  resolution_note,
                  parent_ticket_id
                FROM app.support_ticket
                WHERE id = $1::uuid
                """,
                ticket_id,
            )
            if old_row is None:
                raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
            parent_specified = "parent_ticket_id" in body.model_fields_set
            next_parent_id = (
                _normalize_parent_ticket_id(body.parent_ticket_id)
                if parent_specified
                else None
            )
            if parent_specified and next_parent_id is not None:
                if next_parent_id == str(UUID(ticket_id)):
                    raise HTTPException(
                        status.HTTP_422_UNPROCESSABLE_ENTITY,
                        detail="ticket cannot be its own parent",
                    )
                parent_check = await conn.fetchrow(
                    """
                    WITH RECURSIVE ancestors AS (
                      SELECT id, parent_ticket_id
                      FROM app.support_ticket
                      WHERE id = $1::uuid
                      UNION ALL
                      SELECT t.id, t.parent_ticket_id
                      FROM app.support_ticket AS t
                      JOIN ancestors AS a ON t.id = a.parent_ticket_id
                      WHERE a.parent_ticket_id IS NOT NULL
                    )
                    SELECT
                      EXISTS (SELECT 1 FROM ancestors) AS parent_exists,
                      EXISTS (SELECT 1 FROM ancestors WHERE id = $2::uuid) AS creates_cycle
                    """,
                    next_parent_id,
                    ticket_id,
                )
                if not parent_check or not parent_check["parent_exists"]:
                    raise HTTPException(status.HTTP_404_NOT_FOUND, detail="parent ticket not found")
                if parent_check["creates_cycle"]:
                    raise HTTPException(
                        status.HTTP_422_UNPROCESSABLE_ENTITY,
                        detail="ticket parent would create a cycle",
                    )
            row = await conn.fetchrow(
                """
                UPDATE app.support_ticket SET
                  status = COALESCE($2, status),
                  priority = COALESCE($3, priority),
                  assigned_group = COALESCE($4, assigned_group),
                  resolution_note = COALESCE($5, resolution_note),
                  parent_ticket_id = CASE
                    WHEN $6::bool THEN $7::uuid
                    ELSE parent_ticket_id
                  END,
                  resolved_at = CASE
                    WHEN COALESCE($2, status) IN ('resolved', 'closed')
                      THEN COALESCE(resolved_at, now())
                    WHEN $2 IS NOT NULL
                      THEN NULL
                    ELSE resolved_at
                  END,
                  updated_at = now(),
                  last_activity_at = now()
                WHERE id = $1::uuid
                RETURNING
                  id,
                  reporter_id,
                  reporter_email,
                  reporter_name,
                  reporter_role,
                  category,
                  priority,
                  status,
                  subject,
                  body,
                  source_path,
                  fingerprint,
                  parent_ticket_id,
                  assigned_group,
                  resolution_note,
                  created_at,
                  updated_at,
                  resolved_at
                """,
                ticket_id,
                body.status,
                body.priority,
                body.assigned_group.strip() if body.assigned_group is not None else None,
                body.resolution_note.strip() if body.resolution_note is not None else None,
                parent_specified,
                next_parent_id,
            )
            if row is None:
                raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
            detail = _ticket_change_detail(old_row, row)
            if detail["changed_fields"]:
                await _record_ticket_update_audit(
                    conn,
                    principal=principal,
                    ticket_id=ticket_id,
                    detail=detail,
                )
                row = await conn.fetchrow(
                    SUPPORT_TICKET_DETAIL_FROM_SQL + " WHERE t.id = $1::uuid",
                    ticket_id,
                )
    except Exception as exc:
        if isinstance(exc, HTTPException):
            raise
        raise HTTPException(
            status.HTTP_503_SERVICE_UNAVAILABLE,
            detail="ticket persistence unavailable",
        ) from exc
    return _ticket_from_row(row)


@router.get("/engine-config", response_model=AdminEngineConfigResponse)
async def get_engine_config(principal: AdminPrincipal) -> AdminEngineConfigResponse:
    """Return the current admin-managed engine settings."""
    return await _current_engine_config()


@router.get("/engine-capabilities", response_model=EngineCapabilitiesResponse)
async def get_engine_capabilities(
    principal: AdminPrincipal,
    engine_mode: str | None = Query(default=None),
    engine_url: str | None = Query(default=None),
    force: bool = Query(default=False),
) -> EngineCapabilitiesResponse:
    """Return gateway-discovered models and reasoning levels for one provider."""

    current = await _current_engine_config()
    provider = _normalize_engine_mode(engine_mode or current.engine_mode)
    capability_url = (
        _normalize_engine_url(engine_url)
        if engine_url is not None
        else current.engine_url
    )
    try:
        return await engine_client.capabilities(
            provider=provider,
            base_url=capability_url,
            force=force,
        )
    except Exception as exc:
        raise HTTPException(
            status.HTTP_503_SERVICE_UNAVAILABLE,
            detail=f"engine capabilities unavailable: {exc}",
        ) from exc


async def _validate_engine_selection(
    *,
    provider: EngineProvider,
    engine_url: str,
    model: str,
    reasoning_effort: ReasoningEffort | None,
) -> None:
    try:
        capabilities = await engine_client.capabilities(
            provider=provider,
            base_url=engine_url,
            force=True,
        )
    except Exception as exc:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"선택한 엔진의 모델 목록을 검증할 수 없습니다: {exc}",
        ) from exc
    if not capabilities.available:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=capabilities.detail or "선택한 엔진을 사용할 수 없습니다.",
        )
    selected = next((option for option in capabilities.models if option.id == model), None)
    if selected is None:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"{provider}에서 사용할 수 없는 모델입니다: {model}",
        )
    if (
        reasoning_effort is not None
        and reasoning_effort not in selected.reasoning_efforts
    ):
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"{model}에서 사용할 수 없는 추론 강도입니다: {reasoning_effort}",
        )


@router.patch("/engine-config", response_model=AdminEngineConfigResponse)
async def patch_engine_config(
    body: AdminEngineConfigPatch,
    principal: AdminPrincipal,
) -> AdminEngineConfigResponse:
    """Persist engine settings for administrators."""
    global _ENGINE_CONFIG

    current = await _current_engine_config()
    next_mode = _normalize_engine_mode(body.engine_mode or current.engine_mode)
    next_url = _normalize_engine_url(body.engine_url or current.engine_url)
    next_model = (body.model or current.model).strip()
    if not next_model:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="model is required",
        )
    next_effort = _normalize_reasoning_effort(
        body.reasoning_effort
        if "reasoning_effort" in body.model_fields_set
        else current.reasoning_effort
    )
    await _validate_engine_selection(
        provider=next_mode,
        engine_url=next_url,
        model=next_model,
        reasoning_effort=next_effort,
    )
    next_config = AdminEngineConfigResponse(
        engine_mode=next_mode,
        engine_url=next_url,
        model=next_model,
        reasoning_effort=next_effort,
        updated_by=principal.email,
        updated_at=time.time(),
        durable=False,
        source="runtime_cache",
    )
    try:
        pool = get_pool()
        async with pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                INSERT INTO app.admin_engine_config (
                  id, engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
                )
                VALUES (TRUE, $1, $2, $3, $4, $5, now())
                ON CONFLICT (id) DO UPDATE SET
                  engine_mode = EXCLUDED.engine_mode,
                  engine_url = EXCLUDED.engine_url,
                  model = EXCLUDED.model,
                  reasoning_effort = EXCLUDED.reasoning_effort,
                  updated_by = EXCLUDED.updated_by,
                  updated_at = now()
                RETURNING engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
                """,
                next_config.engine_mode,
                next_config.engine_url,
                next_config.model,
                next_config.reasoning_effort,
                principal.email,
            )
            next_config = _engine_config_from_row(row)
    except Exception as exc:
        if settings.environment != "dev":
            raise HTTPException(
                status.HTTP_503_SERVICE_UNAVAILABLE,
                detail="engine config persistence unavailable",
            ) from exc
    _ENGINE_CONFIG = next_config
    await engine_client.configure(
        base_url=next_config.engine_url,
        engine_mode=next_config.engine_mode,
        default_model=next_config.model,
        default_reasoning_effort=next_config.reasoning_effort,
    )
    return next_config


@router.get("/users", response_model=AdminUsersResponse)
async def list_users(principal: AdminPrincipal) -> AdminUsersResponse:
    """Return users observed by the server-side auth/session boundary."""
    users, durable = await list_managed_users()
    if not durable:
        require_runtime_fallback_allowed("admin user list")
    return AdminUsersResponse(
        source="database" if durable else "server_session_registry",
        durable=durable,
        users=[await _admin_user_response(user, durable=durable) for user in users],
    )


@router.post("/users", response_model=AdminUserResponse, status_code=status.HTTP_201_CREATED)
async def create_user(
    body: AdminUserCreate,
    principal: AdminPrincipal,
) -> AdminUserResponse:
    """Create or reactivate a managed user without requiring that user to log in first."""
    if body.role == "admin" or body.admin_access:
        _assert_super_admin(principal)
    user = await upsert_managed_user(
        ManagedUserUpsertInput(
            email=_normalize_email(body.email),
            display_name=body.display_name,
            role=body.role,
            admin_access=body.admin_access,
            learner_feedback_enabled=body.learner_feedback_enabled,
            account_status=body.account_status,
            affiliation=body.affiliation,
            cohort_ids=body.cohort_ids,
            reactivate=True,
        )
    )
    users, durable = await list_managed_users()
    if not durable:
        require_runtime_fallback_allowed("admin user create")
    return await _admin_user_response(user, durable=durable)


@router.patch("/users/{user_id}", response_model=AdminUserResponse)
async def patch_user(
    user_id: str,
    body: AdminUserPatch,
    principal: AdminPrincipal,
) -> AdminUserResponse:
    """Update a server-known user's role/profile for the current API process."""
    current = await get_managed_user(user_id)
    if current is None:
        raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
    if _requires_super_admin_for_privilege_change(
        current_role=current.role,
        next_role=body.role,
        current_admin_access=current.admin_access,
        next_admin_access=body.admin_access,
    ):
        _assert_super_admin(principal)
    _assert_super_admin_target_mutable(current.email, body.admin_access)
    next_user = await update_managed_user(
        user_id,
        ManagedUserPatch(
            display_name=body.display_name,
            role=body.role,
            admin_access=body.admin_access,
            learner_feedback_enabled=body.learner_feedback_enabled,
            account_status=body.account_status,
            affiliation=body.affiliation,
            cohort_ids=body.cohort_ids,
        ),
    )
    if next_user is None:
        raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
    users, durable = await list_managed_users()
    if not durable:
        require_runtime_fallback_allowed("admin user update")
    return await _admin_user_response(next_user, durable=durable)


@router.delete("/users/{user_id}", response_model=AdminUserDeleteResponse)
async def delete_user(
    user_id: str,
    principal: AdminPrincipal,
) -> AdminUserDeleteResponse:
    """Deactivate a managed user and revoke any active browser sessions."""
    if user_id == principal.user_id:
        raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot deactivate current admin")
    current = await get_managed_user(user_id)
    if current is None:
        raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
    if is_super_admin_email(current.email):
        raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot deactivate super admin")
    if not await deactivate_managed_user(user_id):
        raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
    return AdminUserDeleteResponse(ok=True, user_id=user_id)