2398 lines
82 KiB
Python
2398 lines
82 KiB
Python
"""Admin operations routes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from decimal import Decimal
|
|
from typing import Annotated, Iterable, Literal, cast
|
|
from uuid import UUID
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
|
from pydantic import BaseModel, Field
|
|
|
|
from ..auth_types import AccountStatus, RoleName
|
|
from ..auth_sessions import (
|
|
ManagedUserPatch,
|
|
ManagedUserUpsertInput,
|
|
active_session_count,
|
|
deactivate_managed_user,
|
|
get_managed_user,
|
|
has_admin_access,
|
|
is_super_admin_email,
|
|
list_managed_users,
|
|
update_managed_user,
|
|
upsert_managed_user,
|
|
)
|
|
from ..config import settings
|
|
from ..contracts.engine_gateway import (
|
|
ENGINE_PROVIDER_DEFAULTS,
|
|
ENGINE_PROVIDERS,
|
|
ENGINE_REASONING_EFFORTS,
|
|
EngineCapabilitiesResponse,
|
|
EngineProvider,
|
|
ReasoningEffort,
|
|
)
|
|
from ..db import acquire, get_pool, healthcheck
|
|
from ..deps import Principal, require_admin_access
|
|
from ..engine_client import engine_client
|
|
from ..runtime_policy import require_runtime_fallback_allowed
|
|
from ..services import evaluator, notifications, rag
|
|
from ..services.llm_pricing import (
|
|
estimate_reference_cost,
|
|
provider_uses_reference_cost,
|
|
)
|
|
from ..services.voice import voice_service
|
|
from ..services.voice_runtime import VoiceRuntimeSnapshot, voice_runtime_metrics
|
|
from ..store import store
|
|
|
|
router = APIRouter(prefix="/admin", tags=["admin"])
|
|
|
|
AdminPrincipal = Annotated[Principal, Depends(require_admin_access())]
|
|
HealthStatus = Literal["ok", "degraded", "down"]
|
|
UsageBudgetStatus = Literal["disabled", "ok", "warn", "exceeded", "indeterminate"]
|
|
UsageCostBasis = Literal[
|
|
"provider_estimate",
|
|
"provider_reported",
|
|
"reference_rate",
|
|
"reference_upper_bound",
|
|
"partial",
|
|
"partial_upper_bound",
|
|
"unavailable",
|
|
]
|
|
TicketCategory = Literal[
|
|
"account_access",
|
|
"session_review",
|
|
"voice_browser",
|
|
"content_scenario",
|
|
"safety",
|
|
"other",
|
|
]
|
|
TicketPriority = Literal["low", "normal", "high", "urgent"]
|
|
TicketStatus = Literal["open", "triaged", "in_progress", "resolved", "closed"]
|
|
NotificationDeliveryStatus = Literal["queued", "sending", "sent", "failed", "skipped"]
|
|
|
|
SYNTHETIC_USAGE_SIGNATURES = frozenset({("e2e", "fake-client", 1, 1, 0.0)})
|
|
|
|
REPORTABLE_CLIENT_TURN_FILTER_SQL = """
|
|
speaker = 'client'
|
|
AND NOT (
|
|
LOWER(BTRIM(COALESCE(llm_provider, ''))) = 'e2e'
|
|
AND LOWER(BTRIM(COALESCE(model, ''))) = 'fake-client'
|
|
AND COALESCE(tokens_in, 0) = 1
|
|
AND COALESCE(tokens_out, 0) = 1
|
|
AND COALESCE(cost_usd, 0) = 0
|
|
)
|
|
"""
|
|
|
|
METERED_CLIENT_TURN_FILTER_SQL = f"""
|
|
{REPORTABLE_CLIENT_TURN_FILTER_SQL}
|
|
AND (
|
|
llm_provider IS NOT NULL OR model IS NOT NULL
|
|
OR tokens_in IS NOT NULL OR tokens_out IS NOT NULL
|
|
OR cost_usd IS NOT NULL
|
|
)
|
|
"""
|
|
|
|
SUPPORT_TICKET_DETAIL_FROM_SQL = """
|
|
SELECT
|
|
t.id,
|
|
t.reporter_id,
|
|
t.reporter_email,
|
|
t.reporter_name,
|
|
t.reporter_role,
|
|
t.category,
|
|
t.priority,
|
|
t.status,
|
|
t.subject,
|
|
t.body,
|
|
t.source_path,
|
|
t.fingerprint,
|
|
t.parent_ticket_id,
|
|
t.assigned_group,
|
|
t.resolution_note,
|
|
t.created_at,
|
|
t.updated_at,
|
|
t.resolved_at,
|
|
COALESCE(dup.duplicate_count, 0) AS duplicate_count,
|
|
dup.parent_candidate_id AS duplicate_parent_candidate_id,
|
|
COALESCE(child.child_ticket_count, 0) AS child_ticket_count,
|
|
COALESCE(ev.event_count, 0) AS event_count,
|
|
ev.last_event_at
|
|
FROM app.support_ticket AS t
|
|
LEFT JOIN LATERAL (
|
|
SELECT
|
|
count(*) FILTER (WHERE id <> t.id)::int AS duplicate_count,
|
|
(array_agg(id::text ORDER BY created_at ASC, id ASC))[1] AS parent_candidate_id
|
|
FROM app.support_ticket
|
|
WHERE fingerprint <> ''
|
|
AND fingerprint = t.fingerprint
|
|
) AS dup ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT count(*)::int AS child_ticket_count
|
|
FROM app.support_ticket
|
|
WHERE parent_ticket_id = t.id
|
|
) AS child ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT count(*)::int AS event_count, max(created_at) AS last_event_at
|
|
FROM audit.audit_log
|
|
WHERE action = 'support_ticket_update'
|
|
AND target_kind = 'support_ticket'
|
|
AND target_id = t.id::text
|
|
) AS ev ON TRUE
|
|
"""
|
|
|
|
|
|
class AdminServiceHealth(BaseModel):
|
|
key: str
|
|
name: str
|
|
status: HealthStatus
|
|
detail: str
|
|
metric: str
|
|
load: float
|
|
|
|
|
|
class AdminHealthResponse(BaseModel):
|
|
status: HealthStatus
|
|
environment: str
|
|
engine_mode: str
|
|
services: list[AdminServiceHealth]
|
|
|
|
|
|
class AdminUsageBreakdown(BaseModel):
|
|
provider: str
|
|
model: str
|
|
turns: int
|
|
token_metered_turns: int = 0
|
|
token_unmetered_turns: int = 0
|
|
tokens_in: int
|
|
tokens_out: int
|
|
cost_usd: float
|
|
recorded_cost_usd: float = 0.0
|
|
estimated_cost_usd: float = 0.0
|
|
cost_basis: UsageCostBasis = "provider_reported"
|
|
rate_label: str | None = None
|
|
rate_source_url: str | None = None
|
|
|
|
|
|
class AdminUsageDailyCost(BaseModel):
|
|
day: str
|
|
turns: int
|
|
tokens_in: int
|
|
tokens_out: int
|
|
cost_usd: float
|
|
cost_basis: UsageCostBasis = "provider_reported"
|
|
|
|
|
|
class AdminUsageBudget(BaseModel):
|
|
limit_usd: float
|
|
used_ratio: float
|
|
remaining_usd: float | None
|
|
status: UsageBudgetStatus
|
|
cost_basis: UsageCostBasis = "provider_reported"
|
|
|
|
|
|
class AdminUsageEvaluatorCache(BaseModel):
|
|
enabled: bool
|
|
entries: int
|
|
hits: int
|
|
misses: int
|
|
stores: int
|
|
evictions: int
|
|
requests: int
|
|
hit_rate: float
|
|
|
|
|
|
class AdminUsageResponse(BaseModel):
|
|
source: Literal["database", "server_session_registry"]
|
|
durable: bool
|
|
window_days: int
|
|
generated_at: float
|
|
total_turns: int
|
|
metered_turns: int
|
|
token_metered_turns: int = 0
|
|
token_unmetered_turns: int = 0
|
|
tokens_in: int
|
|
tokens_out: int
|
|
cost_usd: float
|
|
recorded_cost_usd: float = 0.0
|
|
estimated_cost_usd: float = 0.0
|
|
cost_basis: UsageCostBasis = "provider_reported"
|
|
budget: AdminUsageBudget
|
|
evaluator_cache: AdminUsageEvaluatorCache
|
|
by_provider: list[AdminUsageBreakdown]
|
|
daily_cost: list[AdminUsageDailyCost] = Field(default_factory=list)
|
|
|
|
|
|
class AdminHealthEvent(BaseModel):
|
|
event_id: int
|
|
observed_at: float
|
|
overall_status: HealthStatus
|
|
service_key: str
|
|
service_name: str
|
|
service_status: HealthStatus
|
|
detail: str
|
|
metric: str
|
|
load: float
|
|
|
|
|
|
class AdminUptimeServiceSummary(BaseModel):
|
|
service_key: str
|
|
service_name: str
|
|
samples: int
|
|
ok_samples: int
|
|
degraded_samples: int
|
|
down_samples: int
|
|
latest_status: HealthStatus
|
|
latest_observed_at: float | None = None
|
|
|
|
|
|
class AdminUptimeResponse(BaseModel):
|
|
source: Literal["database", "unavailable"]
|
|
durable: bool
|
|
window_hours: int
|
|
generated_at: float
|
|
sample_count: int
|
|
ok_ratio: float
|
|
degraded_events: int
|
|
down_events: int
|
|
last_down_at: float | None = None
|
|
services: list[AdminUptimeServiceSummary]
|
|
events: list[AdminHealthEvent]
|
|
|
|
|
|
class AdminTicketReporter(BaseModel):
|
|
user_id: str | None = None
|
|
email: str
|
|
display_name: str
|
|
role: str
|
|
|
|
|
|
class AdminSupportTicketResponse(BaseModel):
|
|
ticket_id: str
|
|
reporter: AdminTicketReporter
|
|
category: TicketCategory
|
|
priority: TicketPriority
|
|
status: TicketStatus
|
|
subject: str
|
|
body: str
|
|
source_path: str
|
|
fingerprint: str
|
|
parent_ticket_id: str | None = None
|
|
duplicate_count: int = 0
|
|
duplicate_parent_candidate_id: str | None = None
|
|
child_ticket_count: int = 0
|
|
assigned_group: str
|
|
resolution_note: str
|
|
created_at: float
|
|
updated_at: float
|
|
resolved_at: float | None = None
|
|
event_count: int = 0
|
|
last_event_at: float | None = None
|
|
|
|
|
|
class AdminTicketSummary(BaseModel):
|
|
total: int
|
|
open_count: int
|
|
high_priority_count: int
|
|
stale_count: int
|
|
by_status: dict[str, int]
|
|
by_category: dict[str, int]
|
|
by_priority: dict[str, int]
|
|
|
|
|
|
class AdminTicketsResponse(BaseModel):
|
|
source: Literal["database", "unavailable"]
|
|
durable: bool
|
|
generated_at: float
|
|
tickets: list[AdminSupportTicketResponse]
|
|
summary: AdminTicketSummary
|
|
|
|
|
|
class AdminEngineConfigResponse(BaseModel):
|
|
engine_mode: EngineProvider
|
|
engine_url: str
|
|
model: str
|
|
reasoning_effort: ReasoningEffort | None = None
|
|
updated_by: str | None = None
|
|
updated_at: float | None = None
|
|
durable: bool = False
|
|
source: Literal["database", "runtime_cache", "runtime_default"] = "runtime_default"
|
|
|
|
|
|
class AdminEngineConfigPatch(BaseModel):
|
|
engine_mode: str | None = None
|
|
engine_url: str | None = None
|
|
model: str | None = None
|
|
reasoning_effort: str | None = None
|
|
|
|
|
|
class AdminTicketPatch(BaseModel):
|
|
status: TicketStatus | None = None
|
|
priority: TicketPriority | None = None
|
|
assigned_group: str | None = Field(default=None, max_length=120)
|
|
resolution_note: str | None = Field(default=None, max_length=2000)
|
|
parent_ticket_id: str | None = Field(default=None, max_length=36)
|
|
|
|
|
|
class AdminUserResponse(BaseModel):
|
|
user_id: str
|
|
email: str
|
|
display_name: str
|
|
role: RoleName
|
|
admin_access: bool
|
|
learner_feedback_enabled: bool
|
|
super_admin: bool = False
|
|
account_status: AccountStatus
|
|
cohort_ids: list[str]
|
|
affiliation: str
|
|
active_sessions: int
|
|
created_at: float
|
|
last_seen_at: float
|
|
source: Literal["database", "server_session_registry"]
|
|
|
|
|
|
class AdminUsersResponse(BaseModel):
|
|
source: Literal["database", "server_session_registry"]
|
|
durable: bool
|
|
users: list[AdminUserResponse]
|
|
|
|
|
|
class AdminUserPatch(BaseModel):
|
|
display_name: str | None = Field(default=None, min_length=1, max_length=80)
|
|
role: RoleName | None = None
|
|
admin_access: bool | None = None
|
|
learner_feedback_enabled: bool | None = None
|
|
account_status: AccountStatus | None = None
|
|
affiliation: str | None = Field(default=None, max_length=120)
|
|
cohort_ids: list[str] | None = None
|
|
|
|
|
|
class AdminNotificationDeliveryResponse(BaseModel):
|
|
id: str
|
|
event_id: str
|
|
kind: str
|
|
recipient_email: str
|
|
recipient_name: str
|
|
recipient_role: str
|
|
subject: str
|
|
status: NotificationDeliveryStatus
|
|
attempts: int
|
|
last_error: str | None = None
|
|
provider_message_id: str | None = None
|
|
created_at: float | None = None
|
|
updated_at: float | None = None
|
|
sent_at: float | None = None
|
|
|
|
|
|
class AdminNotificationsResponse(BaseModel):
|
|
provider: str
|
|
smtp_configured: bool
|
|
queued: int
|
|
failed: int
|
|
sent: int
|
|
skipped: int
|
|
deliveries: list[AdminNotificationDeliveryResponse]
|
|
|
|
|
|
class AdminNotificationProcessResponse(BaseModel):
|
|
processed: int
|
|
sent: int
|
|
failed: int
|
|
skipped: int
|
|
|
|
|
|
class AdminNotificationTestResponse(AdminNotificationProcessResponse):
|
|
provider: str
|
|
smtp_configured: bool
|
|
recipients: int
|
|
|
|
|
|
class RuntimeHealthMetrics(BaseModel):
|
|
engine_latency_ms: float | None = None
|
|
db_pool_size: int = 0
|
|
db_pool_idle: int = 0
|
|
db_pool_max: int = 0
|
|
active_users: int = 0
|
|
active_auth_sessions: int = 0
|
|
active_sessions: int = 0
|
|
ended_sessions: int = 0
|
|
pending_reviews: int = 0
|
|
|
|
|
|
def _clamp01(value: float) -> float:
|
|
return round(max(0.0, min(1.0, value)), 3)
|
|
|
|
|
|
def _pool_load(metrics: RuntimeHealthMetrics) -> float:
|
|
if metrics.db_pool_max <= 0:
|
|
return 0.0
|
|
busy = max(0, metrics.db_pool_size - metrics.db_pool_idle)
|
|
return _clamp01(busy / metrics.db_pool_max)
|
|
|
|
|
|
def _workload_load(count: int, expected_capacity: int) -> float:
|
|
if expected_capacity <= 0:
|
|
return 0.0
|
|
return _clamp01(count / expected_capacity)
|
|
|
|
|
|
def _decimal_to_float(value: object) -> float:
|
|
if value is None:
|
|
return 0.0
|
|
if isinstance(value, Decimal):
|
|
return float(value)
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return 0.0
|
|
|
|
|
|
def _safe_usage_int(value: object) -> int:
|
|
try:
|
|
return int(value or 0)
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _is_reportable_usage(
|
|
provider: str,
|
|
model: str,
|
|
tokens_in: int,
|
|
tokens_out: int,
|
|
cost_usd: float,
|
|
) -> bool:
|
|
signature = (
|
|
provider.strip().lower(),
|
|
model.strip().lower(),
|
|
max(0, tokens_in),
|
|
max(0, tokens_out),
|
|
max(0.0, cost_usd),
|
|
)
|
|
return signature not in SYNTHETIC_USAGE_SIGNATURES
|
|
|
|
|
|
def _usage_breakdown(
|
|
*,
|
|
provider: str,
|
|
model: str,
|
|
turns: int,
|
|
token_metered_turns: int,
|
|
token_unmetered_turns: int,
|
|
tokens_in: int,
|
|
tokens_out: int,
|
|
stored_cost_usd: float,
|
|
unpriced_tokens_in: int,
|
|
unpriced_tokens_out: int,
|
|
priced_at: datetime | int | float | str,
|
|
) -> AdminUsageBreakdown:
|
|
"""저장된 공급자 비용 추정치와 공식 참조단가를 한 원장 행으로 정규화한다."""
|
|
|
|
reference_basis = provider_uses_reference_cost(provider)
|
|
fallback = estimate_reference_cost(
|
|
provider=provider,
|
|
model=model,
|
|
tokens_in=unpriced_tokens_in,
|
|
tokens_out=unpriced_tokens_out,
|
|
priced_at=priced_at,
|
|
)
|
|
rate_info = fallback
|
|
if rate_info is None and not (reference_basis and stored_cost_usd > 0):
|
|
rate_info = estimate_reference_cost(
|
|
provider=provider,
|
|
model=model,
|
|
tokens_in=tokens_in,
|
|
tokens_out=tokens_out,
|
|
priced_at=priced_at,
|
|
)
|
|
fallback_cost = fallback.cost_usd if fallback is not None else 0.0
|
|
effective_cost = max(0.0, stored_cost_usd) + fallback_cost
|
|
|
|
if reference_basis:
|
|
recorded_cost = 0.0
|
|
estimated_cost = effective_cost
|
|
if fallback is not None and stored_cost_usd <= 0:
|
|
basis: UsageCostBasis = "reference_upper_bound"
|
|
else:
|
|
basis = (
|
|
"reference_rate"
|
|
if rate_info is not None or effective_cost > 0
|
|
else "unavailable"
|
|
)
|
|
else:
|
|
recorded_cost = max(0.0, stored_cost_usd)
|
|
estimated_cost = fallback_cost
|
|
if recorded_cost > 0:
|
|
basis = "provider_estimate" if provider == "claude_cli" else "provider_reported"
|
|
elif fallback is not None:
|
|
basis = "reference_rate"
|
|
else:
|
|
basis = "unavailable"
|
|
|
|
rate_label = rate_info.rate_label if rate_info is not None else None
|
|
rate_source_url = rate_info.source_url if rate_info is not None else None
|
|
if reference_basis and stored_cost_usd > 0:
|
|
if fallback is not None:
|
|
rate_label = "호출 시점 저장 추정값 + 미저장분 공식 참조단가 합산"
|
|
else:
|
|
rate_label = "호출 시점에 저장된 참조단가 추정값"
|
|
rate_source_url = None
|
|
elif fallback is not None:
|
|
rate_label = f"{fallback.rate_label} · 캐시 미보존 과거행은 전체 입력 기준"
|
|
|
|
return AdminUsageBreakdown(
|
|
provider=provider,
|
|
model=model,
|
|
turns=max(0, turns),
|
|
token_metered_turns=max(0, token_metered_turns),
|
|
token_unmetered_turns=max(0, token_unmetered_turns),
|
|
tokens_in=max(0, tokens_in),
|
|
tokens_out=max(0, tokens_out),
|
|
cost_usd=effective_cost,
|
|
recorded_cost_usd=recorded_cost,
|
|
estimated_cost_usd=estimated_cost,
|
|
cost_basis=basis,
|
|
rate_label=rate_label,
|
|
rate_source_url=rate_source_url,
|
|
)
|
|
|
|
|
|
def _merge_usage_breakdowns(
|
|
breakdowns: Iterable[AdminUsageBreakdown],
|
|
) -> list[AdminUsageBreakdown]:
|
|
grouped: dict[tuple[str, str], list[AdminUsageBreakdown]] = {}
|
|
for item in breakdowns:
|
|
grouped.setdefault((item.provider, item.model), []).append(item)
|
|
|
|
merged: list[AdminUsageBreakdown] = []
|
|
for (provider, model), items in grouped.items():
|
|
basis = _aggregate_cost_basis(item.cost_basis for item in items)
|
|
labels = {item.rate_label for item in items if item.rate_label}
|
|
source_urls = {item.rate_source_url for item in items if item.rate_source_url}
|
|
rate_label = next(iter(labels)) if len(labels) == 1 else None
|
|
if basis == "partial":
|
|
rate_label = "일부 호출 미산정 · 표시액은 산정 가능분 합계"
|
|
elif basis == "partial_upper_bound":
|
|
rate_label = "일부 호출 미산정 · 산정된 부분도 상한 추정"
|
|
elif len(labels) > 1:
|
|
rate_label = (
|
|
"기간별 공식 참조단가 상한 합산"
|
|
if basis == "reference_upper_bound"
|
|
else "기간별 공식 참조단가 합산"
|
|
)
|
|
merged.append(
|
|
AdminUsageBreakdown(
|
|
provider=provider,
|
|
model=model,
|
|
turns=sum(item.turns for item in items),
|
|
token_metered_turns=sum(item.token_metered_turns for item in items),
|
|
token_unmetered_turns=sum(item.token_unmetered_turns for item in items),
|
|
tokens_in=sum(item.tokens_in for item in items),
|
|
tokens_out=sum(item.tokens_out for item in items),
|
|
cost_usd=round(sum(item.cost_usd for item in items), 6),
|
|
recorded_cost_usd=round(
|
|
sum(item.recorded_cost_usd for item in items), 6
|
|
),
|
|
estimated_cost_usd=round(
|
|
sum(item.estimated_cost_usd for item in items), 6
|
|
),
|
|
cost_basis=cast(UsageCostBasis, basis),
|
|
rate_label=rate_label,
|
|
rate_source_url=(
|
|
next(iter(source_urls)) if len(source_urls) == 1 else None
|
|
),
|
|
)
|
|
)
|
|
return merged
|
|
|
|
|
|
def _aggregate_cost_basis(
|
|
bases: Iterable[UsageCostBasis],
|
|
) -> UsageCostBasis:
|
|
basis_set = set(bases)
|
|
if not basis_set:
|
|
return "provider_reported"
|
|
|
|
has_missing = bool(
|
|
basis_set & {"unavailable", "partial", "partial_upper_bound"}
|
|
)
|
|
has_upper_bound = bool(
|
|
basis_set & {"reference_upper_bound", "partial_upper_bound"}
|
|
)
|
|
has_known_amount = basis_set != {"unavailable"}
|
|
if has_missing:
|
|
if not has_known_amount:
|
|
return "unavailable"
|
|
return "partial_upper_bound" if has_upper_bound else "partial"
|
|
if has_upper_bound:
|
|
return "reference_upper_bound"
|
|
for candidate in (
|
|
"provider_estimate",
|
|
"provider_reported",
|
|
"reference_rate",
|
|
):
|
|
if candidate in basis_set:
|
|
return cast(UsageCostBasis, candidate)
|
|
return "unavailable"
|
|
|
|
|
|
def _scale_usage_breakdown(
|
|
breakdown: AdminUsageBreakdown,
|
|
multiplier: int,
|
|
) -> AdminUsageBreakdown:
|
|
count = max(0, multiplier)
|
|
return AdminUsageBreakdown(
|
|
provider=breakdown.provider,
|
|
model=breakdown.model,
|
|
turns=breakdown.turns * count,
|
|
token_metered_turns=breakdown.token_metered_turns * count,
|
|
token_unmetered_turns=breakdown.token_unmetered_turns * count,
|
|
tokens_in=breakdown.tokens_in * count,
|
|
tokens_out=breakdown.tokens_out * count,
|
|
cost_usd=breakdown.cost_usd * count,
|
|
recorded_cost_usd=breakdown.recorded_cost_usd * count,
|
|
estimated_cost_usd=breakdown.estimated_cost_usd * count,
|
|
cost_basis=breakdown.cost_basis,
|
|
rate_label=breakdown.rate_label,
|
|
rate_source_url=breakdown.rate_source_url,
|
|
)
|
|
|
|
|
|
def _usage_budget(
|
|
cost_usd: float,
|
|
cost_basis: UsageCostBasis,
|
|
) -> AdminUsageBudget:
|
|
limit = max(0.0, float(settings.admin_usage_budget_usd or 0.0))
|
|
if limit <= 0:
|
|
return AdminUsageBudget(
|
|
limit_usd=0.0,
|
|
used_ratio=0.0,
|
|
remaining_usd=None,
|
|
status="disabled",
|
|
cost_basis=cost_basis,
|
|
)
|
|
used_ratio = max(0.0, cost_usd / limit)
|
|
remaining_usd: float | None = round(max(0.0, limit - cost_usd), 6)
|
|
status_value: UsageBudgetStatus
|
|
if cost_basis in {"partial_upper_bound", "unavailable"}:
|
|
status_value = "indeterminate"
|
|
remaining_usd = None
|
|
elif cost_basis == "partial":
|
|
if used_ratio >= 1.0:
|
|
status_value = "exceeded"
|
|
elif used_ratio >= 0.8:
|
|
status_value = "warn"
|
|
else:
|
|
status_value = "indeterminate"
|
|
elif cost_basis == "reference_upper_bound":
|
|
status_value = "ok" if used_ratio < 0.8 else "indeterminate"
|
|
elif used_ratio >= 1.0:
|
|
status_value = "exceeded"
|
|
elif used_ratio >= 0.8:
|
|
status_value = "warn"
|
|
else:
|
|
status_value = "ok"
|
|
return AdminUsageBudget(
|
|
limit_usd=round(limit, 6),
|
|
used_ratio=round(used_ratio, 4),
|
|
remaining_usd=remaining_usd,
|
|
status=status_value,
|
|
cost_basis=cost_basis,
|
|
)
|
|
|
|
|
|
def _usage_evaluator_cache() -> AdminUsageEvaluatorCache:
|
|
stats = evaluator.evaluator_semantic_cache_stats()
|
|
hits = _safe_usage_int(stats.get("hits", 0))
|
|
misses = _safe_usage_int(stats.get("misses", 0))
|
|
requests = hits + misses
|
|
hit_rate = round(hits / requests, 4) if requests else 0.0
|
|
enabled = (
|
|
bool(settings.evaluator_semantic_cache_enabled)
|
|
and settings.evaluator_semantic_cache_ttl_seconds > 0
|
|
and settings.evaluator_semantic_cache_max_entries > 0
|
|
)
|
|
return AdminUsageEvaluatorCache(
|
|
enabled=enabled,
|
|
entries=_safe_usage_int(stats.get("entries", 0)),
|
|
hits=hits,
|
|
misses=misses,
|
|
stores=_safe_usage_int(stats.get("stores", 0)),
|
|
evictions=_safe_usage_int(stats.get("evictions", 0)),
|
|
requests=requests,
|
|
hit_rate=hit_rate,
|
|
)
|
|
|
|
|
|
async def _runtime_health_metrics(*, db_ok: bool) -> RuntimeHealthMetrics:
|
|
metrics = RuntimeHealthMetrics()
|
|
|
|
try:
|
|
pool = get_pool()
|
|
metrics.db_pool_size = int(pool.get_size())
|
|
metrics.db_pool_idle = int(pool.get_idle_size())
|
|
metrics.db_pool_max = int(pool.get_max_size())
|
|
except Exception:
|
|
pass
|
|
|
|
if not db_ok:
|
|
return metrics
|
|
|
|
try:
|
|
async with acquire(role="admin") as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT
|
|
(SELECT COUNT(*) FROM app.app_user WHERE is_active) AS active_users,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM app.auth_session
|
|
WHERE revoked_at IS NULL AND expires_at > now()
|
|
) AS active_auth_sessions,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM app.sessions
|
|
WHERE ended_at IS NULL
|
|
) AS active_sessions,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM app.sessions
|
|
WHERE ended_at IS NOT NULL
|
|
) AS ended_sessions,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM app.sessions s
|
|
LEFT JOIN app.session_summary ss ON ss.session_id = s.id
|
|
WHERE s.ended_at IS NOT NULL AND ss.session_id IS NULL
|
|
) AS pending_reviews
|
|
"""
|
|
)
|
|
if row is not None:
|
|
metrics.active_users = int(row["active_users"] or 0)
|
|
metrics.active_auth_sessions = int(row["active_auth_sessions"] or 0)
|
|
metrics.active_sessions = int(row["active_sessions"] or 0)
|
|
metrics.ended_sessions = int(row["ended_sessions"] or 0)
|
|
metrics.pending_reviews = int(row["pending_reviews"] or 0)
|
|
except Exception:
|
|
return metrics
|
|
|
|
return metrics
|
|
|
|
|
|
def _usage_daily_cost(
|
|
daily_buckets: dict[str, dict[str, int | float | list[UsageCostBasis]]],
|
|
) -> list[AdminUsageDailyCost]:
|
|
return [
|
|
AdminUsageDailyCost(
|
|
day=day,
|
|
turns=int(values["turns"]),
|
|
tokens_in=int(values["tokens_in"]),
|
|
tokens_out=int(values["tokens_out"]),
|
|
cost_usd=round(float(values["cost_usd"]), 6),
|
|
cost_basis=_aggregate_cost_basis(
|
|
cast(list[UsageCostBasis], values["cost_bases"])
|
|
),
|
|
)
|
|
for day, values in sorted(daily_buckets.items())
|
|
]
|
|
|
|
|
|
async def _usage_from_database(window_days: int) -> AdminUsageResponse:
|
|
async with acquire(role="admin") as conn:
|
|
total_row = await conn.fetchrow(
|
|
f"""
|
|
SELECT
|
|
COUNT(*) FILTER (WHERE {REPORTABLE_CLIENT_TURN_FILTER_SQL}) AS total_turns,
|
|
COUNT(*) FILTER (WHERE {METERED_CLIENT_TURN_FILTER_SQL}) AS metered_turns,
|
|
COUNT(*) FILTER (
|
|
WHERE {METERED_CLIENT_TURN_FILTER_SQL}
|
|
AND (COALESCE(tokens_in, 0) > 0 OR COALESCE(tokens_out, 0) > 0)
|
|
) AS token_metered_turns,
|
|
COUNT(*) FILTER (
|
|
WHERE {METERED_CLIENT_TURN_FILTER_SQL}
|
|
AND COALESCE(tokens_in, 0) <= 0
|
|
AND COALESCE(tokens_out, 0) <= 0
|
|
) AS token_unmetered_turns,
|
|
COALESCE(
|
|
SUM(tokens_in) FILTER (WHERE {REPORTABLE_CLIENT_TURN_FILTER_SQL}), 0
|
|
)::bigint AS tokens_in,
|
|
COALESCE(
|
|
SUM(tokens_out) FILTER (WHERE {REPORTABLE_CLIENT_TURN_FILTER_SQL}), 0
|
|
)::bigint AS tokens_out,
|
|
COALESCE(
|
|
SUM(cost_usd) FILTER (WHERE {REPORTABLE_CLIENT_TURN_FILTER_SQL}), 0
|
|
)::numeric AS cost_usd
|
|
FROM app.turns
|
|
WHERE created_at >= now() - ($1::int * interval '1 day')
|
|
AND speaker = 'client'
|
|
""",
|
|
window_days,
|
|
)
|
|
usage_rows = await conn.fetch(
|
|
f"""
|
|
SELECT
|
|
to_char(
|
|
date_trunc('day', created_at AT TIME ZONE 'UTC'),
|
|
'YYYY-MM-DD'
|
|
) AS day,
|
|
COALESCE(llm_provider, 'unknown') AS provider,
|
|
COALESCE(model, 'unknown') AS model,
|
|
COALESCE(tokens_in, 0)::bigint AS tokens_in,
|
|
COALESCE(tokens_out, 0)::bigint AS tokens_out,
|
|
COALESCE(cost_usd, 0)::numeric AS cost_usd,
|
|
COUNT(*)::bigint AS matching_turns
|
|
FROM app.turns
|
|
WHERE created_at >= now() - ($1::int * interval '1 day')
|
|
AND {METERED_CLIENT_TURN_FILTER_SQL}
|
|
GROUP BY 1, 2, 3, 4, 5, 6
|
|
""",
|
|
window_days,
|
|
)
|
|
|
|
# 요청별 입력 크기로 단가 구간이 갈리는 모델이 있어 동일 계량 signature만 묶는다.
|
|
dated_breakdowns: list[AdminUsageBreakdown] = []
|
|
for row in usage_rows:
|
|
tokens_in = _safe_usage_int(row["tokens_in"])
|
|
tokens_out = _safe_usage_int(row["tokens_out"])
|
|
stored_cost_usd = _decimal_to_float(row["cost_usd"])
|
|
is_token_metered = tokens_in > 0 or tokens_out > 0
|
|
matching_turns = max(1, _safe_usage_int(row["matching_turns"]))
|
|
dated_breakdowns.append(
|
|
_scale_usage_breakdown(
|
|
_usage_breakdown(
|
|
provider=str(row["provider"] or "unknown"),
|
|
model=str(row["model"] or "unknown"),
|
|
turns=1,
|
|
token_metered_turns=1 if is_token_metered else 0,
|
|
token_unmetered_turns=0 if is_token_metered else 1,
|
|
tokens_in=tokens_in,
|
|
tokens_out=tokens_out,
|
|
stored_cost_usd=stored_cost_usd,
|
|
unpriced_tokens_in=tokens_in if stored_cost_usd <= 0 else 0,
|
|
unpriced_tokens_out=tokens_out if stored_cost_usd <= 0 else 0,
|
|
priced_at=str(row["day"]),
|
|
),
|
|
matching_turns,
|
|
)
|
|
)
|
|
all_breakdowns = _merge_usage_breakdowns(dated_breakdowns)
|
|
all_breakdowns.sort(
|
|
key=lambda item: (
|
|
-item.cost_usd,
|
|
-(item.tokens_in + item.tokens_out),
|
|
-item.turns,
|
|
item.provider,
|
|
item.model,
|
|
)
|
|
)
|
|
recorded_cost = round(sum(item.recorded_cost_usd for item in dated_breakdowns), 6)
|
|
estimated_cost = round(sum(item.estimated_cost_usd for item in dated_breakdowns), 6)
|
|
total_cost = round(recorded_cost + estimated_cost, 6)
|
|
total_cost_basis = _aggregate_cost_basis(
|
|
item.cost_basis for item in dated_breakdowns
|
|
)
|
|
|
|
daily_buckets: dict[str, dict[str, int | float | list[UsageCostBasis]]] = {}
|
|
for row, breakdown in zip(usage_rows, dated_breakdowns, strict=True):
|
|
day = str(row["day"])
|
|
bucket = daily_buckets.setdefault(
|
|
day,
|
|
{
|
|
"turns": 0,
|
|
"tokens_in": 0,
|
|
"tokens_out": 0,
|
|
"cost_usd": 0.0,
|
|
"cost_bases": [],
|
|
},
|
|
)
|
|
bucket["turns"] = int(bucket["turns"]) + breakdown.turns
|
|
bucket["tokens_in"] = int(bucket["tokens_in"]) + breakdown.tokens_in
|
|
bucket["tokens_out"] = int(bucket["tokens_out"]) + breakdown.tokens_out
|
|
bucket["cost_usd"] = float(bucket["cost_usd"]) + breakdown.cost_usd
|
|
cost_bases = cast(list[UsageCostBasis], bucket["cost_bases"])
|
|
cost_bases.append(breakdown.cost_basis)
|
|
|
|
return AdminUsageResponse(
|
|
source="database",
|
|
durable=True,
|
|
window_days=window_days,
|
|
generated_at=time.time(),
|
|
total_turns=_safe_usage_int(total_row["total_turns"] if total_row else 0),
|
|
metered_turns=_safe_usage_int(total_row["metered_turns"] if total_row else 0),
|
|
token_metered_turns=_safe_usage_int(
|
|
total_row["token_metered_turns"] if total_row else 0
|
|
),
|
|
token_unmetered_turns=_safe_usage_int(
|
|
total_row["token_unmetered_turns"] if total_row else 0
|
|
),
|
|
tokens_in=_safe_usage_int(total_row["tokens_in"] if total_row else 0),
|
|
tokens_out=_safe_usage_int(total_row["tokens_out"] if total_row else 0),
|
|
cost_usd=total_cost,
|
|
recorded_cost_usd=recorded_cost,
|
|
estimated_cost_usd=estimated_cost,
|
|
cost_basis=total_cost_basis,
|
|
budget=_usage_budget(total_cost, total_cost_basis),
|
|
evaluator_cache=_usage_evaluator_cache(),
|
|
by_provider=all_breakdowns[:12],
|
|
daily_cost=_usage_daily_cost(daily_buckets),
|
|
)
|
|
|
|
|
|
async def _record_health_events(
|
|
*,
|
|
principal: Principal | None,
|
|
overall_status: HealthStatus,
|
|
environment: str,
|
|
engine_mode: str,
|
|
services: list[AdminServiceHealth],
|
|
) -> int:
|
|
if not services:
|
|
return 0
|
|
captured_by = principal.user_id if principal is not None else None
|
|
try:
|
|
async with acquire(role="admin", user_id=captured_by) as conn:
|
|
await conn.executemany(
|
|
"""
|
|
INSERT INTO app.admin_health_event (
|
|
overall_status,
|
|
environment,
|
|
engine_mode,
|
|
service_key,
|
|
service_name,
|
|
service_status,
|
|
detail,
|
|
metric,
|
|
load,
|
|
captured_by
|
|
)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::uuid)
|
|
""",
|
|
[
|
|
(
|
|
overall_status,
|
|
environment,
|
|
engine_mode,
|
|
service.key,
|
|
service.name,
|
|
service.status,
|
|
service.detail,
|
|
service.metric,
|
|
service.load,
|
|
captured_by,
|
|
)
|
|
for service in services
|
|
],
|
|
)
|
|
return len(services)
|
|
except Exception:
|
|
# 헬스 화면 자체가 장애 확인 경로라, 이력 적재 실패가 응답을 막으면 안 된다.
|
|
return 0
|
|
|
|
|
|
async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
|
|
async with acquire(role="admin") as conn:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT
|
|
id,
|
|
observed_at,
|
|
overall_status,
|
|
service_key,
|
|
service_name,
|
|
service_status,
|
|
detail,
|
|
metric,
|
|
load
|
|
FROM app.admin_health_event
|
|
WHERE observed_at >= now() - ($1::int * interval '1 hour')
|
|
ORDER BY observed_at DESC, id DESC
|
|
LIMIT 240
|
|
""",
|
|
window_hours,
|
|
)
|
|
rollup_rows = await conn.fetch(
|
|
"""
|
|
SELECT
|
|
r.service_key,
|
|
r.service_name,
|
|
r.sample_count,
|
|
r.ok_samples,
|
|
r.degraded_samples,
|
|
r.down_samples,
|
|
r.latest_status,
|
|
r.last_observed_at,
|
|
r.last_down_at
|
|
FROM app.admin_health_daily_rollup AS r
|
|
WHERE r.rollup_date >= (now() - ($1::int * interval '1 hour'))::date
|
|
AND r.rollup_date < current_date
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM app.admin_health_event AS e
|
|
WHERE e.observed_at::date = r.rollup_date
|
|
AND e.environment = r.environment
|
|
AND e.engine_mode = r.engine_mode
|
|
AND e.service_key = r.service_key
|
|
)
|
|
ORDER BY r.last_observed_at DESC, r.service_key
|
|
LIMIT 1000
|
|
""",
|
|
window_hours,
|
|
)
|
|
|
|
events = [_health_event_from_row(row) for row in rows]
|
|
service_buckets: dict[str, AdminUptimeServiceSummary] = {}
|
|
|
|
def ensure_bucket(
|
|
*,
|
|
service_key: str,
|
|
service_name: str,
|
|
latest_status: HealthStatus,
|
|
latest_observed_at: float | None,
|
|
) -> AdminUptimeServiceSummary:
|
|
current = service_buckets.get(service_key)
|
|
if current is None:
|
|
current = AdminUptimeServiceSummary(
|
|
service_key=service_key,
|
|
service_name=service_name,
|
|
samples=0,
|
|
ok_samples=0,
|
|
degraded_samples=0,
|
|
down_samples=0,
|
|
latest_status=latest_status,
|
|
latest_observed_at=latest_observed_at,
|
|
)
|
|
service_buckets[service_key] = current
|
|
return current
|
|
if latest_observed_at is not None and (
|
|
current.latest_observed_at is None
|
|
or latest_observed_at > current.latest_observed_at
|
|
):
|
|
current.latest_status = latest_status
|
|
current.latest_observed_at = latest_observed_at
|
|
return current
|
|
|
|
for event in events:
|
|
current = ensure_bucket(
|
|
service_key=event.service_key,
|
|
service_name=event.service_name,
|
|
latest_status=event.service_status,
|
|
latest_observed_at=event.observed_at,
|
|
)
|
|
current.samples += 1
|
|
if event.service_status == "ok":
|
|
current.ok_samples += 1
|
|
elif event.service_status == "degraded":
|
|
current.degraded_samples += 1
|
|
else:
|
|
current.down_samples += 1
|
|
|
|
rollup_last_down: list[float] = []
|
|
for row in rollup_rows:
|
|
latest_observed_at = _row_ts(row["last_observed_at"])
|
|
current = ensure_bucket(
|
|
service_key=str(row["service_key"]),
|
|
service_name=str(row["service_name"]),
|
|
latest_status=row["latest_status"],
|
|
latest_observed_at=latest_observed_at,
|
|
)
|
|
current.samples += int(row["sample_count"] or 0)
|
|
current.ok_samples += int(row["ok_samples"] or 0)
|
|
current.degraded_samples += int(row["degraded_samples"] or 0)
|
|
current.down_samples += int(row["down_samples"] or 0)
|
|
last_down = _row_ts(row["last_down_at"])
|
|
if last_down is not None:
|
|
rollup_last_down.append(last_down)
|
|
|
|
ok_samples = sum(1 for event in events if event.service_status == "ok") + sum(
|
|
int(row["ok_samples"] or 0) for row in rollup_rows
|
|
)
|
|
sample_count = len(events) + sum(int(row["sample_count"] or 0) for row in rollup_rows)
|
|
down_times = [event.observed_at for event in events if event.service_status == "down"]
|
|
down_times.extend(rollup_last_down)
|
|
return AdminUptimeResponse(
|
|
source="database",
|
|
durable=True,
|
|
window_hours=window_hours,
|
|
generated_at=time.time(),
|
|
sample_count=sample_count,
|
|
ok_ratio=round(ok_samples / sample_count, 4) if sample_count else 0.0,
|
|
degraded_events=sum(1 for event in events if event.service_status == "degraded")
|
|
+ sum(int(row["degraded_samples"] or 0) for row in rollup_rows),
|
|
down_events=sum(1 for event in events if event.service_status == "down")
|
|
+ sum(int(row["down_samples"] or 0) for row in rollup_rows),
|
|
last_down_at=max(down_times) if down_times else None,
|
|
services=sorted(
|
|
service_buckets.values(),
|
|
key=lambda item: (item.latest_status != "down", item.latest_status != "degraded", item.service_key),
|
|
),
|
|
events=events,
|
|
)
|
|
|
|
|
|
async def _tickets_from_database(
|
|
*,
|
|
ticket_status: TicketStatus | None,
|
|
category: TicketCategory | None = None,
|
|
priority: TicketPriority | None = None,
|
|
assigned_group: str | None = None,
|
|
source_path: str | None = None,
|
|
stale_only: bool = False,
|
|
search: str = "",
|
|
window_days: int,
|
|
) -> AdminTicketsResponse:
|
|
assigned_group_filter = (assigned_group or "").strip()
|
|
source_path_filter = (source_path or "").strip()
|
|
search_filter = search.strip().lower()
|
|
async with acquire(role="admin") as conn:
|
|
rows = await conn.fetch(
|
|
SUPPORT_TICKET_DETAIL_FROM_SQL
|
|
+ """
|
|
WHERE ($1::text IS NULL OR status = $1)
|
|
AND ($2::text IS NULL OR category = $2)
|
|
AND ($3::text IS NULL OR priority = $3)
|
|
AND ($4::text = '' OR assigned_group = $4)
|
|
AND ($5::text = '' OR source_path = $5)
|
|
AND (
|
|
NOT $6::bool
|
|
OR (
|
|
status NOT IN ('resolved', 'closed')
|
|
AND updated_at < now() - interval '1 day'
|
|
)
|
|
)
|
|
AND (
|
|
$7::text = ''
|
|
OR lower(subject) LIKE '%' || $7 || '%'
|
|
OR lower(body) LIKE '%' || $7 || '%'
|
|
OR lower(source_path) LIKE '%' || $7 || '%'
|
|
OR lower(reporter_email) LIKE '%' || $7 || '%'
|
|
)
|
|
AND created_at >= now() - ($8::int * interval '1 day')
|
|
ORDER BY
|
|
CASE
|
|
WHEN status = 'open' THEN 0
|
|
WHEN status = 'triaged' THEN 1
|
|
WHEN status = 'in_progress' THEN 2
|
|
WHEN status = 'resolved' THEN 3
|
|
ELSE 4
|
|
END,
|
|
CASE
|
|
WHEN priority = 'urgent' THEN 0
|
|
WHEN priority = 'high' THEN 1
|
|
WHEN priority = 'normal' THEN 2
|
|
ELSE 3
|
|
END,
|
|
updated_at DESC
|
|
LIMIT 120
|
|
""",
|
|
ticket_status,
|
|
category,
|
|
priority,
|
|
assigned_group_filter,
|
|
source_path_filter,
|
|
stale_only,
|
|
search_filter,
|
|
window_days,
|
|
)
|
|
tickets = [_ticket_from_row(row) for row in rows]
|
|
return AdminTicketsResponse(
|
|
source="database",
|
|
durable=True,
|
|
generated_at=time.time(),
|
|
tickets=tickets,
|
|
summary=_ticket_summary(tickets),
|
|
)
|
|
|
|
|
|
def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
|
|
window_start = time.time() - (window_days * 86400)
|
|
total_turns = 0
|
|
metered_turns = 0
|
|
token_metered_turns = 0
|
|
token_unmetered_turns = 0
|
|
tokens_in = 0
|
|
tokens_out = 0
|
|
cost_usd = 0.0
|
|
recorded_cost_usd = 0.0
|
|
estimated_cost_usd = 0.0
|
|
turn_breakdowns: list[AdminUsageBreakdown] = []
|
|
daily_buckets: dict[str, dict[str, int | float | list[UsageCostBasis]]] = {}
|
|
|
|
for sess in store.list():
|
|
for turn in getattr(sess, "turns", []) or []:
|
|
if getattr(turn, "speaker", "") != "client":
|
|
continue
|
|
created_at = float(getattr(turn, "created_at", 0.0) or 0.0)
|
|
if created_at < window_start:
|
|
continue
|
|
provider = str(getattr(turn, "llm_provider", None) or "unknown")
|
|
model = str(getattr(turn, "model", None) or "unknown")
|
|
turn_tokens_in = _safe_usage_int(getattr(turn, "tokens_in", 0))
|
|
turn_tokens_out = _safe_usage_int(getattr(turn, "tokens_out", 0))
|
|
turn_cost = _decimal_to_float(getattr(turn, "cost_usd", 0.0))
|
|
if not _is_reportable_usage(
|
|
provider,
|
|
model,
|
|
turn_tokens_in,
|
|
turn_tokens_out,
|
|
turn_cost,
|
|
):
|
|
continue
|
|
total_turns += 1
|
|
is_metered = (
|
|
provider != "unknown"
|
|
or model != "unknown"
|
|
or turn_tokens_in > 0
|
|
or turn_tokens_out > 0
|
|
or turn_cost > 0
|
|
)
|
|
if not is_metered:
|
|
continue
|
|
metered_turns += 1
|
|
is_token_metered = turn_tokens_in > 0 or turn_tokens_out > 0
|
|
if is_token_metered:
|
|
token_metered_turns += 1
|
|
else:
|
|
token_unmetered_turns += 1
|
|
tokens_in += turn_tokens_in
|
|
tokens_out += turn_tokens_out
|
|
turn_breakdown = _usage_breakdown(
|
|
provider=provider,
|
|
model=model,
|
|
turns=1,
|
|
token_metered_turns=1 if is_token_metered else 0,
|
|
token_unmetered_turns=0 if is_token_metered else 1,
|
|
tokens_in=turn_tokens_in,
|
|
tokens_out=turn_tokens_out,
|
|
stored_cost_usd=turn_cost,
|
|
unpriced_tokens_in=turn_tokens_in if turn_cost <= 0 else 0,
|
|
unpriced_tokens_out=turn_tokens_out if turn_cost <= 0 else 0,
|
|
priced_at=created_at,
|
|
)
|
|
turn_breakdowns.append(turn_breakdown)
|
|
cost_usd += turn_breakdown.cost_usd
|
|
recorded_cost_usd += turn_breakdown.recorded_cost_usd
|
|
estimated_cost_usd += turn_breakdown.estimated_cost_usd
|
|
day = datetime.fromtimestamp(created_at, timezone.utc).strftime("%Y-%m-%d")
|
|
daily_bucket = daily_buckets.setdefault(
|
|
day,
|
|
{
|
|
"turns": 0,
|
|
"tokens_in": 0,
|
|
"tokens_out": 0,
|
|
"cost_usd": 0.0,
|
|
"cost_bases": [],
|
|
},
|
|
)
|
|
daily_bucket["turns"] = int(daily_bucket["turns"]) + 1
|
|
daily_bucket["tokens_in"] = int(daily_bucket["tokens_in"]) + turn_tokens_in
|
|
daily_bucket["tokens_out"] = int(daily_bucket["tokens_out"]) + turn_tokens_out
|
|
daily_bucket["cost_usd"] = (
|
|
float(daily_bucket["cost_usd"]) + turn_breakdown.cost_usd
|
|
)
|
|
daily_cost_bases = cast(
|
|
list[UsageCostBasis], daily_bucket["cost_bases"]
|
|
)
|
|
daily_cost_bases.append(turn_breakdown.cost_basis)
|
|
|
|
by_provider = _merge_usage_breakdowns(turn_breakdowns)
|
|
by_provider.sort(
|
|
key=lambda item: (
|
|
-item.cost_usd,
|
|
-(item.tokens_in + item.tokens_out),
|
|
-item.turns,
|
|
item.provider,
|
|
item.model,
|
|
)
|
|
)
|
|
|
|
total_cost = round(cost_usd, 6)
|
|
total_cost_basis = _aggregate_cost_basis(
|
|
item.cost_basis for item in turn_breakdowns
|
|
)
|
|
return AdminUsageResponse(
|
|
source="server_session_registry",
|
|
durable=False,
|
|
window_days=window_days,
|
|
generated_at=time.time(),
|
|
total_turns=total_turns,
|
|
metered_turns=metered_turns,
|
|
token_metered_turns=token_metered_turns,
|
|
token_unmetered_turns=token_unmetered_turns,
|
|
tokens_in=tokens_in,
|
|
tokens_out=tokens_out,
|
|
cost_usd=total_cost,
|
|
recorded_cost_usd=round(recorded_cost_usd, 6),
|
|
estimated_cost_usd=round(estimated_cost_usd, 6),
|
|
cost_basis=total_cost_basis,
|
|
budget=_usage_budget(total_cost, total_cost_basis),
|
|
evaluator_cache=_usage_evaluator_cache(),
|
|
by_provider=by_provider[:12],
|
|
daily_cost=_usage_daily_cost(daily_buckets),
|
|
)
|
|
|
|
|
|
class AdminUserCreate(BaseModel):
|
|
email: str = Field(
|
|
...,
|
|
min_length=3,
|
|
max_length=254,
|
|
pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$",
|
|
)
|
|
display_name: str = Field(..., min_length=1, max_length=80)
|
|
role: RoleName = "learner"
|
|
admin_access: bool = False
|
|
learner_feedback_enabled: bool = True
|
|
# 외부 연구참여자는 exact-email 사전등록 뒤 별도 승인을 거치게 한다.
|
|
account_status: Literal["pending"] = "pending"
|
|
affiliation: str | None = Field(default=None, max_length=120)
|
|
cohort_ids: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class AdminUserDeleteResponse(BaseModel):
|
|
ok: bool
|
|
user_id: str
|
|
|
|
|
|
_ENGINE_CONFIG: AdminEngineConfigResponse | None = None
|
|
ENGINE_MODES = set(ENGINE_PROVIDERS)
|
|
ENGINE_MODE_ALIASES = {"messages_api": "claude_api"}
|
|
|
|
|
|
def _normalize_email(value: str) -> str:
|
|
email = value.strip().lower()
|
|
if "@" not in email:
|
|
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
|
|
local, domain = email.rsplit("@", 1)
|
|
if not local or not domain:
|
|
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
|
|
return email
|
|
|
|
|
|
def _default_engine_config() -> AdminEngineConfigResponse:
|
|
default_model, default_effort = ENGINE_PROVIDER_DEFAULTS[settings.engine_mode]
|
|
return AdminEngineConfigResponse(
|
|
engine_mode=settings.engine_mode,
|
|
engine_url=settings.engine_url,
|
|
model=default_model,
|
|
reasoning_effort=default_effort,
|
|
durable=False,
|
|
source="runtime_default",
|
|
)
|
|
|
|
|
|
def _normalize_engine_mode(value: str) -> EngineProvider:
|
|
mode = ENGINE_MODE_ALIASES.get(value.strip(), value.strip())
|
|
if mode not in ENGINE_MODES:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=f"unsupported engine mode {value}",
|
|
)
|
|
return cast(EngineProvider, mode)
|
|
|
|
|
|
def _normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
|
|
effort = (value or "").strip().lower()
|
|
if not effort:
|
|
return None
|
|
if effort not in ENGINE_REASONING_EFFORTS:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=f"unsupported reasoning effort {value}",
|
|
)
|
|
return cast(ReasoningEffort, effort)
|
|
|
|
|
|
def _normalize_engine_url(value: str) -> str:
|
|
url = value.strip().rstrip("/")
|
|
if not (url.startswith("http://") or url.startswith("https://")):
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="engine_url must start with http:// or https://",
|
|
)
|
|
return url
|
|
|
|
|
|
def _updated_at_ts(value: datetime | None) -> float | None:
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
value = value.replace(tzinfo=timezone.utc)
|
|
return value.timestamp()
|
|
|
|
|
|
def _row_ts(value: object) -> float | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, datetime):
|
|
return _updated_at_ts(value)
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _row_value(row, key: str, default=None):
|
|
try:
|
|
return row[key]
|
|
except (IndexError, KeyError, TypeError):
|
|
return default
|
|
|
|
|
|
def _notification_delivery_response(row) -> AdminNotificationDeliveryResponse:
|
|
return AdminNotificationDeliveryResponse(
|
|
id=str(row["id"]),
|
|
event_id=str(row["event_id"]),
|
|
kind=str(row["kind"]),
|
|
recipient_email=str(row["recipient_email"]),
|
|
recipient_name=str(row["recipient_name"] or ""),
|
|
recipient_role=str(row["recipient_role"] or ""),
|
|
subject=str(row["subject"] or ""),
|
|
status=row["status"],
|
|
attempts=int(row["attempts"] or 0),
|
|
last_error=str(row["last_error"] or "") or None,
|
|
provider_message_id=str(row["provider_message_id"] or "") or None,
|
|
created_at=_row_ts(row["created_at"]),
|
|
updated_at=_row_ts(row["updated_at"]),
|
|
sent_at=_row_ts(row["sent_at"]),
|
|
)
|
|
|
|
|
|
def _engine_config_from_row(row) -> AdminEngineConfigResponse:
|
|
return AdminEngineConfigResponse(
|
|
engine_mode=_normalize_engine_mode(row["engine_mode"]),
|
|
engine_url=_normalize_engine_url(row["engine_url"]),
|
|
model=row["model"],
|
|
reasoning_effort=_normalize_reasoning_effort(row.get("reasoning_effort")),
|
|
updated_by=row["updated_by"],
|
|
updated_at=_updated_at_ts(row["updated_at"]),
|
|
durable=True,
|
|
source="database",
|
|
)
|
|
|
|
|
|
def _health_event_from_row(row) -> AdminHealthEvent:
|
|
return AdminHealthEvent(
|
|
event_id=int(row["id"]),
|
|
observed_at=_row_ts(row["observed_at"]) or 0.0,
|
|
overall_status=row["overall_status"],
|
|
service_key=row["service_key"],
|
|
service_name=row["service_name"],
|
|
service_status=row["service_status"],
|
|
detail=row["detail"],
|
|
metric=row["metric"],
|
|
load=_clamp01(float(row["load"] or 0.0)),
|
|
)
|
|
|
|
|
|
def _ticket_from_row(row) -> AdminSupportTicketResponse:
|
|
return AdminSupportTicketResponse(
|
|
ticket_id=str(row["id"]),
|
|
reporter=AdminTicketReporter(
|
|
user_id=str(row["reporter_id"]) if row["reporter_id"] else None,
|
|
email=row["reporter_email"],
|
|
display_name=row["reporter_name"],
|
|
role=row["reporter_role"],
|
|
),
|
|
category=row["category"],
|
|
priority=row["priority"],
|
|
status=row["status"],
|
|
subject=row["subject"],
|
|
body=row["body"],
|
|
source_path=row["source_path"],
|
|
fingerprint=_row_value(row, "fingerprint", "") or "",
|
|
parent_ticket_id=(
|
|
str(_row_value(row, "parent_ticket_id"))
|
|
if _row_value(row, "parent_ticket_id") is not None
|
|
else None
|
|
),
|
|
duplicate_count=int(_row_value(row, "duplicate_count", 0) or 0),
|
|
duplicate_parent_candidate_id=(
|
|
str(_row_value(row, "duplicate_parent_candidate_id"))
|
|
if _row_value(row, "duplicate_parent_candidate_id") is not None
|
|
else None
|
|
),
|
|
child_ticket_count=int(_row_value(row, "child_ticket_count", 0) or 0),
|
|
assigned_group=row["assigned_group"],
|
|
resolution_note=row["resolution_note"],
|
|
created_at=_row_ts(row["created_at"]) or 0.0,
|
|
updated_at=_row_ts(row["updated_at"]) or 0.0,
|
|
resolved_at=_row_ts(row["resolved_at"]),
|
|
event_count=int(_row_value(row, "event_count", 0) or 0),
|
|
last_event_at=_row_ts(_row_value(row, "last_event_at")),
|
|
)
|
|
|
|
|
|
def _empty_ticket_summary() -> AdminTicketSummary:
|
|
return AdminTicketSummary(
|
|
total=0,
|
|
open_count=0,
|
|
high_priority_count=0,
|
|
stale_count=0,
|
|
by_status={},
|
|
by_category={},
|
|
by_priority={},
|
|
)
|
|
|
|
|
|
def _ticket_summary(tickets: list[AdminSupportTicketResponse]) -> AdminTicketSummary:
|
|
now = time.time()
|
|
by_status: dict[str, int] = {}
|
|
by_category: dict[str, int] = {}
|
|
by_priority: dict[str, int] = {}
|
|
active_tickets = [ticket for ticket in tickets if ticket.status not in {"resolved", "closed"}]
|
|
for ticket in tickets:
|
|
by_status[ticket.status] = by_status.get(ticket.status, 0) + 1
|
|
by_category[ticket.category] = by_category.get(ticket.category, 0) + 1
|
|
by_priority[ticket.priority] = by_priority.get(ticket.priority, 0) + 1
|
|
return AdminTicketSummary(
|
|
total=len(tickets),
|
|
open_count=len(active_tickets),
|
|
high_priority_count=sum(
|
|
1 for ticket in active_tickets if ticket.priority in {"high", "urgent"}
|
|
),
|
|
stale_count=sum(
|
|
1
|
|
for ticket in active_tickets
|
|
if now - ticket.updated_at > 86400
|
|
),
|
|
by_status=by_status,
|
|
by_category=by_category,
|
|
by_priority=by_priority,
|
|
)
|
|
|
|
|
|
def _unavailable_uptime(window_hours: int) -> AdminUptimeResponse:
|
|
return AdminUptimeResponse(
|
|
source="unavailable",
|
|
durable=False,
|
|
window_hours=window_hours,
|
|
generated_at=time.time(),
|
|
sample_count=0,
|
|
ok_ratio=0.0,
|
|
degraded_events=0,
|
|
down_events=0,
|
|
last_down_at=None,
|
|
services=[],
|
|
events=[],
|
|
)
|
|
|
|
|
|
def _unavailable_tickets() -> AdminTicketsResponse:
|
|
return AdminTicketsResponse(
|
|
source="unavailable",
|
|
durable=False,
|
|
generated_at=time.time(),
|
|
tickets=[],
|
|
summary=_empty_ticket_summary(),
|
|
)
|
|
|
|
|
|
def _normalize_parent_ticket_id(value: str | None) -> str | None:
|
|
if value is None:
|
|
return None
|
|
stripped = value.strip()
|
|
if not stripped:
|
|
return None
|
|
try:
|
|
return str(UUID(stripped))
|
|
except ValueError as exc:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="invalid parent ticket id",
|
|
) from exc
|
|
|
|
|
|
def _ticket_change_detail(old_row, new_row) -> dict[str, object]:
|
|
changed_fields: list[str] = []
|
|
detail: dict[str, object] = {"changed_fields": changed_fields}
|
|
for field in ("status", "priority", "assigned_group", "parent_ticket_id"):
|
|
before = _row_value(old_row, field, "")
|
|
after = _row_value(new_row, field, "")
|
|
if before is not None:
|
|
before = str(before)
|
|
if after is not None:
|
|
after = str(after)
|
|
if before != after:
|
|
changed_fields.append(field)
|
|
detail[field] = {"from": before, "to": after}
|
|
old_note = (_row_value(old_row, "resolution_note", "") or "").strip()
|
|
new_note = (_row_value(new_row, "resolution_note", "") or "").strip()
|
|
if old_note != new_note:
|
|
changed_fields.append("resolution_note")
|
|
detail["resolution_note"] = {
|
|
"from_present": bool(old_note),
|
|
"to_present": bool(new_note),
|
|
}
|
|
detail["category"] = _row_value(new_row, "category", "")
|
|
detail["source_path"] = _row_value(new_row, "source_path", "")
|
|
return detail
|
|
|
|
|
|
async def _record_ticket_update_audit(
|
|
conn,
|
|
*,
|
|
principal: Principal,
|
|
ticket_id: str,
|
|
detail: dict[str, object],
|
|
) -> None:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO audit.audit_log (
|
|
actor_uid, action, target_kind, target_id, detail
|
|
)
|
|
VALUES ($1::uuid, $2, $3, $4, $5::jsonb)
|
|
""",
|
|
principal.user_id,
|
|
"support_ticket_update",
|
|
"support_ticket",
|
|
ticket_id,
|
|
detail,
|
|
)
|
|
|
|
|
|
async def _current_engine_config() -> AdminEngineConfigResponse:
|
|
if _ENGINE_CONFIG is not None:
|
|
return _ENGINE_CONFIG
|
|
try:
|
|
pool = get_pool()
|
|
async with pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
|
|
FROM app.admin_engine_config
|
|
WHERE id = TRUE
|
|
"""
|
|
)
|
|
if row is not None:
|
|
return _engine_config_from_row(row)
|
|
return _default_engine_config()
|
|
except Exception:
|
|
require_runtime_fallback_allowed("admin engine config")
|
|
return _default_engine_config()
|
|
|
|
|
|
async def apply_engine_config_from_store() -> AdminEngineConfigResponse:
|
|
"""Load admin engine settings and apply them to the live engine client."""
|
|
config = await _current_engine_config()
|
|
await engine_client.configure(
|
|
base_url=config.engine_url,
|
|
engine_mode=config.engine_mode,
|
|
default_model=config.model,
|
|
default_reasoning_effort=config.reasoning_effort,
|
|
)
|
|
return config
|
|
|
|
|
|
def _overall_status(services: list[AdminServiceHealth]) -> HealthStatus:
|
|
if any(s.status == "down" for s in services):
|
|
return "down"
|
|
if any(s.status == "degraded" for s in services):
|
|
return "degraded"
|
|
return "ok"
|
|
|
|
|
|
def _engine_unavailable_detail(detail: str) -> str:
|
|
if detail.lstrip().startswith("{") and '"ok":false' in detail:
|
|
return "Engine readiness failed"
|
|
return detail
|
|
|
|
|
|
def _requires_super_admin_for_privilege_change(
|
|
*,
|
|
current_role: RoleName | None,
|
|
next_role: RoleName | None,
|
|
current_admin_access: bool,
|
|
next_admin_access: bool | None,
|
|
) -> bool:
|
|
if next_admin_access is not None and next_admin_access != current_admin_access:
|
|
return True
|
|
if next_role == "admin" and current_role != "admin":
|
|
return True
|
|
if current_role == "admin" and next_role is not None and next_role != "admin":
|
|
return True
|
|
return False
|
|
|
|
|
|
def _assert_super_admin(principal: Principal) -> None:
|
|
if not principal.super_admin:
|
|
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="super admin required")
|
|
|
|
|
|
def _assert_super_admin_target_mutable(email: str, admin_access: bool | None) -> None:
|
|
if is_super_admin_email(email) and admin_access is False:
|
|
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot revoke super admin access")
|
|
|
|
|
|
async def _admin_user_response(user, *, durable: bool) -> AdminUserResponse:
|
|
effective_admin_access = has_admin_access(user.email, user.role, user.admin_access)
|
|
return AdminUserResponse(
|
|
user_id=user.user_id,
|
|
email=user.email,
|
|
display_name=user.display_name,
|
|
role=user.role,
|
|
admin_access=effective_admin_access,
|
|
learner_feedback_enabled=user.learner_feedback_enabled,
|
|
super_admin=is_super_admin_email(user.email),
|
|
account_status=user.account_status,
|
|
cohort_ids=user.cohort_ids,
|
|
affiliation=user.affiliation,
|
|
active_sessions=await active_session_count(user.user_id),
|
|
created_at=user.created_at,
|
|
last_seen_at=user.last_seen_at,
|
|
source="database" if durable else "server_session_registry",
|
|
)
|
|
|
|
|
|
async def _build_admin_health_response() -> AdminHealthResponse:
|
|
current_engine = await _current_engine_config()
|
|
db_ok = await healthcheck()
|
|
engine_started = time.perf_counter()
|
|
engine = await engine_client.health_detail()
|
|
engine_ok = bool(engine.get("ok"))
|
|
engine_detail = _engine_unavailable_detail(
|
|
str(engine.get("detail") or "engine readiness unavailable")
|
|
)
|
|
engine_latency_ms = (time.perf_counter() - engine_started) * 1000
|
|
voice_ok = voice_service.is_available()
|
|
metrics = await _runtime_health_metrics(db_ok=db_ok)
|
|
metrics.engine_latency_ms = engine_latency_ms if engine_ok else None
|
|
pool_load = _pool_load(metrics)
|
|
session_load = _workload_load(metrics.active_sessions, 50)
|
|
review_load = _workload_load(metrics.pending_reviews, 50)
|
|
runtime_fallback_is_enabled = settings.environment == "dev"
|
|
db_status: HealthStatus = "ok" if db_ok else ("degraded" if runtime_fallback_is_enabled else "down")
|
|
db_detail = (
|
|
"사용자, 세션, 리뷰 저장"
|
|
if db_ok
|
|
else (
|
|
"DB 연결 전까지 비영구 개발 런타임 기록 사용"
|
|
if runtime_fallback_is_enabled
|
|
else "DB 저장소에 연결할 수 없습니다"
|
|
)
|
|
)
|
|
db_metric = (
|
|
f"풀 {max(0, metrics.db_pool_size - metrics.db_pool_idle)}/{metrics.db_pool_max}"
|
|
if db_ok
|
|
else ("비영구 런타임 기록" if runtime_fallback_is_enabled else "저장소 중단")
|
|
)
|
|
|
|
services = [
|
|
AdminServiceHealth(
|
|
key="engine",
|
|
name="응답 생성",
|
|
status="ok" if engine_ok else "down",
|
|
detail="AI 엔진 생성 준비 완료" if engine_ok else engine_detail,
|
|
metric=f"{engine_latency_ms:.0f}ms" if engine_ok else "로그인/설정 필요",
|
|
load=_clamp01(engine_latency_ms / 1500) if engine_ok else 0.0,
|
|
),
|
|
AdminServiceHealth(
|
|
key="db",
|
|
name="영구 저장소",
|
|
status=db_status,
|
|
detail=db_detail,
|
|
metric=db_metric,
|
|
load=max(pool_load, session_load) if db_ok else 0.0,
|
|
),
|
|
AdminServiceHealth(
|
|
key="voice",
|
|
name="음성 입력",
|
|
status="ok" if voice_ok else "degraded",
|
|
detail="음성 입력과 재생",
|
|
metric="OpenAI 연결" if voice_ok else "설정 필요",
|
|
load=0.05 if voice_ok else 0.0,
|
|
),
|
|
AdminServiceHealth(
|
|
key="evaluation",
|
|
name="리뷰 생성",
|
|
status="ok" if engine_ok else "degraded",
|
|
detail="회기 종료 후 피드백 생성",
|
|
metric=f"대기 {metrics.pending_reviews}건",
|
|
load=review_load if engine_ok else 0.0,
|
|
),
|
|
AdminServiceHealth(
|
|
key="kb",
|
|
name="지식 검색",
|
|
status="ok" if db_ok else "degraded",
|
|
detail=f"검색 기준값 {rag.CRAG_TOP1_THRESHOLD}",
|
|
metric=(
|
|
f"활성 세션 {metrics.active_sessions}건"
|
|
if db_ok
|
|
else "대기 중"
|
|
),
|
|
load=max(pool_load, session_load) if db_ok else 0.0,
|
|
),
|
|
]
|
|
|
|
response = AdminHealthResponse(
|
|
status=_overall_status(services),
|
|
environment=settings.environment,
|
|
engine_mode=current_engine.engine_mode,
|
|
services=services,
|
|
)
|
|
return response
|
|
|
|
|
|
async def record_admin_health_sample(
|
|
*, principal: Principal | None = None
|
|
) -> tuple[AdminHealthResponse, int]:
|
|
"""Collect and persist one synthetic/admin health sample."""
|
|
response = await _build_admin_health_response()
|
|
recorded_count = await _record_health_events(
|
|
principal=principal,
|
|
overall_status=response.status,
|
|
environment=response.environment,
|
|
engine_mode=response.engine_mode,
|
|
services=response.services,
|
|
)
|
|
return response, recorded_count
|
|
|
|
|
|
@router.get("/health", response_model=AdminHealthResponse)
|
|
async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
|
|
"""Return operational health from live backend checks."""
|
|
response, _ = await record_admin_health_sample(principal=principal)
|
|
return response
|
|
|
|
|
|
@router.get("/voice-runtime", response_model=VoiceRuntimeSnapshot)
|
|
async def admin_voice_runtime(
|
|
principal: AdminPrincipal,
|
|
) -> VoiceRuntimeSnapshot:
|
|
"""Return one API worker's metadata-only voice high-water snapshot."""
|
|
|
|
return voice_runtime_metrics.snapshot()
|
|
|
|
|
|
@router.get("/usage", response_model=AdminUsageResponse)
|
|
async def admin_usage(
|
|
principal: AdminPrincipal,
|
|
window_days: Annotated[int, Query(ge=1, le=90)] = 7,
|
|
) -> AdminUsageResponse:
|
|
"""Return AI token/cost usage from persisted turns or dev fallback state."""
|
|
try:
|
|
return await _usage_from_database(window_days)
|
|
except Exception:
|
|
require_runtime_fallback_allowed("admin usage")
|
|
return _usage_from_runtime_store(window_days)
|
|
|
|
|
|
@router.get("/uptime", response_model=AdminUptimeResponse)
|
|
async def admin_uptime(
|
|
principal: AdminPrincipal,
|
|
window_hours: Annotated[int, Query(ge=1, le=720)] = 24,
|
|
) -> AdminUptimeResponse:
|
|
"""Return persisted health check samples captured by the admin console."""
|
|
try:
|
|
return await _uptime_from_database(window_hours)
|
|
except Exception:
|
|
return _unavailable_uptime(window_hours)
|
|
|
|
|
|
@router.get("/tickets", response_model=AdminTicketsResponse)
|
|
async def list_tickets(
|
|
principal: AdminPrincipal,
|
|
ticket_status: Annotated[TicketStatus | None, Query(alias="status")] = None,
|
|
category: TicketCategory | None = None,
|
|
priority: TicketPriority | None = None,
|
|
assigned_group: Annotated[str | None, Query(max_length=120)] = None,
|
|
source_path: Annotated[str | None, Query(max_length=300)] = None,
|
|
stale_only: bool = False,
|
|
search: Annotated[str, Query(max_length=120)] = "",
|
|
window_days: Annotated[int, Query(ge=1, le=365)] = 30,
|
|
) -> AdminTicketsResponse:
|
|
"""Return user-submitted operational tickets without synthetic fallback rows."""
|
|
try:
|
|
return await _tickets_from_database(
|
|
ticket_status=ticket_status,
|
|
category=category,
|
|
priority=priority,
|
|
assigned_group=assigned_group,
|
|
source_path=source_path,
|
|
stale_only=stale_only,
|
|
search=search,
|
|
window_days=window_days,
|
|
)
|
|
except Exception:
|
|
return _unavailable_tickets()
|
|
|
|
|
|
@router.get("/notifications", response_model=AdminNotificationsResponse)
|
|
async def list_notifications(
|
|
principal: AdminPrincipal,
|
|
limit: Annotated[int, Query(ge=1, le=200)] = 50,
|
|
) -> AdminNotificationsResponse:
|
|
"""Return recent operational email delivery state."""
|
|
try:
|
|
get_pool()
|
|
async with acquire(role="admin") as conn:
|
|
status_rows = await conn.fetch(
|
|
"""
|
|
SELECT status, COUNT(*) AS count
|
|
FROM app.notification_delivery
|
|
GROUP BY status
|
|
"""
|
|
)
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT
|
|
d.id,
|
|
d.event_id,
|
|
e.kind,
|
|
d.recipient_email,
|
|
d.recipient_name,
|
|
d.recipient_role,
|
|
d.subject,
|
|
d.status,
|
|
d.attempts,
|
|
d.last_error,
|
|
d.provider_message_id,
|
|
d.created_at,
|
|
d.updated_at,
|
|
d.sent_at
|
|
FROM app.notification_delivery d
|
|
JOIN app.notification_event e ON e.id = d.event_id
|
|
ORDER BY d.created_at DESC
|
|
LIMIT $1
|
|
""",
|
|
limit,
|
|
)
|
|
except Exception:
|
|
require_runtime_fallback_allowed("admin notifications")
|
|
status_rows = []
|
|
rows = []
|
|
counts = {str(row["status"]): int(row["count"] or 0) for row in status_rows}
|
|
return AdminNotificationsResponse(
|
|
provider=settings.notification_email_provider,
|
|
smtp_configured=bool(settings.smtp_host and settings.smtp_from_email),
|
|
queued=counts.get("queued", 0),
|
|
failed=counts.get("failed", 0),
|
|
sent=counts.get("sent", 0),
|
|
skipped=counts.get("skipped", 0),
|
|
deliveries=[_notification_delivery_response(row) for row in rows],
|
|
)
|
|
|
|
|
|
@router.post("/notifications/process", response_model=AdminNotificationProcessResponse)
|
|
async def process_notifications(
|
|
principal: AdminPrincipal,
|
|
limit: Annotated[int, Query(ge=1, le=100)] = 25,
|
|
) -> AdminNotificationProcessResponse:
|
|
"""Drain queued email notifications once from the admin console/API."""
|
|
try:
|
|
result = await notifications.process_queued_email_notifications(limit=limit)
|
|
except Exception:
|
|
require_runtime_fallback_allowed("admin notification processing")
|
|
result = {"processed": 0, "sent": 0, "failed": 0, "skipped": 0}
|
|
return AdminNotificationProcessResponse(**result)
|
|
|
|
|
|
@router.post("/notifications/test", response_model=AdminNotificationTestResponse)
|
|
async def send_test_notification(
|
|
principal: AdminPrincipal,
|
|
) -> AdminNotificationTestResponse:
|
|
"""Queue and process one explicit admin email test."""
|
|
recipient_count = 0
|
|
try:
|
|
recipient_count = await notifications.enqueue_admin_test_email(
|
|
actor_user_id=principal.user_id,
|
|
actor_email=principal.email,
|
|
)
|
|
result = await notifications.process_queued_email_notifications(
|
|
limit=max(1, min(100, recipient_count or 1))
|
|
)
|
|
except Exception:
|
|
require_runtime_fallback_allowed("admin notification test")
|
|
result = {"processed": 0, "sent": 0, "failed": 0, "skipped": 0}
|
|
return AdminNotificationTestResponse(
|
|
provider=settings.notification_email_provider,
|
|
smtp_configured=bool(settings.smtp_host and settings.smtp_from_email),
|
|
recipients=recipient_count,
|
|
**result,
|
|
)
|
|
|
|
|
|
@router.patch("/tickets/{ticket_id}", response_model=AdminSupportTicketResponse)
|
|
async def patch_ticket(
|
|
ticket_id: str,
|
|
body: AdminTicketPatch,
|
|
principal: AdminPrincipal,
|
|
) -> AdminSupportTicketResponse:
|
|
"""Update ticket triage state for administrators."""
|
|
try:
|
|
async with acquire(role="admin", user_id=principal.user_id) as conn:
|
|
old_row = await conn.fetchrow(
|
|
"""
|
|
SELECT
|
|
id,
|
|
category,
|
|
priority,
|
|
status,
|
|
source_path,
|
|
assigned_group,
|
|
resolution_note,
|
|
parent_ticket_id
|
|
FROM app.support_ticket
|
|
WHERE id = $1::uuid
|
|
""",
|
|
ticket_id,
|
|
)
|
|
if old_row is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
|
|
parent_specified = "parent_ticket_id" in body.model_fields_set
|
|
next_parent_id = (
|
|
_normalize_parent_ticket_id(body.parent_ticket_id)
|
|
if parent_specified
|
|
else None
|
|
)
|
|
if parent_specified and next_parent_id is not None:
|
|
if next_parent_id == str(UUID(ticket_id)):
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="ticket cannot be its own parent",
|
|
)
|
|
parent_check = await conn.fetchrow(
|
|
"""
|
|
WITH RECURSIVE ancestors AS (
|
|
SELECT id, parent_ticket_id
|
|
FROM app.support_ticket
|
|
WHERE id = $1::uuid
|
|
UNION ALL
|
|
SELECT t.id, t.parent_ticket_id
|
|
FROM app.support_ticket AS t
|
|
JOIN ancestors AS a ON t.id = a.parent_ticket_id
|
|
WHERE a.parent_ticket_id IS NOT NULL
|
|
)
|
|
SELECT
|
|
EXISTS (SELECT 1 FROM ancestors) AS parent_exists,
|
|
EXISTS (SELECT 1 FROM ancestors WHERE id = $2::uuid) AS creates_cycle
|
|
""",
|
|
next_parent_id,
|
|
ticket_id,
|
|
)
|
|
if not parent_check or not parent_check["parent_exists"]:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="parent ticket not found")
|
|
if parent_check["creates_cycle"]:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="ticket parent would create a cycle",
|
|
)
|
|
row = await conn.fetchrow(
|
|
"""
|
|
UPDATE app.support_ticket SET
|
|
status = COALESCE($2, status),
|
|
priority = COALESCE($3, priority),
|
|
assigned_group = COALESCE($4, assigned_group),
|
|
resolution_note = COALESCE($5, resolution_note),
|
|
parent_ticket_id = CASE
|
|
WHEN $6::bool THEN $7::uuid
|
|
ELSE parent_ticket_id
|
|
END,
|
|
resolved_at = CASE
|
|
WHEN COALESCE($2, status) IN ('resolved', 'closed')
|
|
THEN COALESCE(resolved_at, now())
|
|
WHEN $2 IS NOT NULL
|
|
THEN NULL
|
|
ELSE resolved_at
|
|
END,
|
|
updated_at = now(),
|
|
last_activity_at = now()
|
|
WHERE id = $1::uuid
|
|
RETURNING
|
|
id,
|
|
reporter_id,
|
|
reporter_email,
|
|
reporter_name,
|
|
reporter_role,
|
|
category,
|
|
priority,
|
|
status,
|
|
subject,
|
|
body,
|
|
source_path,
|
|
fingerprint,
|
|
parent_ticket_id,
|
|
assigned_group,
|
|
resolution_note,
|
|
created_at,
|
|
updated_at,
|
|
resolved_at
|
|
""",
|
|
ticket_id,
|
|
body.status,
|
|
body.priority,
|
|
body.assigned_group.strip() if body.assigned_group is not None else None,
|
|
body.resolution_note.strip() if body.resolution_note is not None else None,
|
|
parent_specified,
|
|
next_parent_id,
|
|
)
|
|
if row is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
|
|
detail = _ticket_change_detail(old_row, row)
|
|
if detail["changed_fields"]:
|
|
await _record_ticket_update_audit(
|
|
conn,
|
|
principal=principal,
|
|
ticket_id=ticket_id,
|
|
detail=detail,
|
|
)
|
|
row = await conn.fetchrow(
|
|
SUPPORT_TICKET_DETAIL_FROM_SQL + " WHERE t.id = $1::uuid",
|
|
ticket_id,
|
|
)
|
|
except Exception as exc:
|
|
if isinstance(exc, HTTPException):
|
|
raise
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail="ticket persistence unavailable",
|
|
) from exc
|
|
return _ticket_from_row(row)
|
|
|
|
|
|
@router.get("/engine-config", response_model=AdminEngineConfigResponse)
|
|
async def get_engine_config(principal: AdminPrincipal) -> AdminEngineConfigResponse:
|
|
"""Return the current admin-managed engine settings."""
|
|
return await _current_engine_config()
|
|
|
|
|
|
@router.get("/engine-capabilities", response_model=EngineCapabilitiesResponse)
|
|
async def get_engine_capabilities(
|
|
principal: AdminPrincipal,
|
|
engine_mode: str | None = Query(default=None),
|
|
engine_url: str | None = Query(default=None),
|
|
force: bool = Query(default=False),
|
|
) -> EngineCapabilitiesResponse:
|
|
"""Return gateway-discovered models and reasoning levels for one provider."""
|
|
|
|
current = await _current_engine_config()
|
|
provider = _normalize_engine_mode(engine_mode or current.engine_mode)
|
|
capability_url = (
|
|
_normalize_engine_url(engine_url)
|
|
if engine_url is not None
|
|
else current.engine_url
|
|
)
|
|
try:
|
|
return await engine_client.capabilities(
|
|
provider=provider,
|
|
base_url=capability_url,
|
|
force=force,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=f"engine capabilities unavailable: {exc}",
|
|
) from exc
|
|
|
|
|
|
async def _validate_engine_selection(
|
|
*,
|
|
provider: EngineProvider,
|
|
engine_url: str,
|
|
model: str,
|
|
reasoning_effort: ReasoningEffort | None,
|
|
) -> None:
|
|
try:
|
|
capabilities = await engine_client.capabilities(
|
|
provider=provider,
|
|
base_url=engine_url,
|
|
force=True,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=f"선택한 엔진의 모델 목록을 검증할 수 없습니다: {exc}",
|
|
) from exc
|
|
if not capabilities.available:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=capabilities.detail or "선택한 엔진을 사용할 수 없습니다.",
|
|
)
|
|
selected = next((option for option in capabilities.models if option.id == model), None)
|
|
if selected is None:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=f"{provider}에서 사용할 수 없는 모델입니다: {model}",
|
|
)
|
|
if (
|
|
reasoning_effort is not None
|
|
and reasoning_effort not in selected.reasoning_efforts
|
|
):
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=f"{model}에서 사용할 수 없는 추론 강도입니다: {reasoning_effort}",
|
|
)
|
|
|
|
|
|
@router.patch("/engine-config", response_model=AdminEngineConfigResponse)
|
|
async def patch_engine_config(
|
|
body: AdminEngineConfigPatch,
|
|
principal: AdminPrincipal,
|
|
) -> AdminEngineConfigResponse:
|
|
"""Persist engine settings for administrators."""
|
|
global _ENGINE_CONFIG
|
|
|
|
current = await _current_engine_config()
|
|
next_mode = _normalize_engine_mode(body.engine_mode or current.engine_mode)
|
|
next_url = _normalize_engine_url(body.engine_url or current.engine_url)
|
|
next_model = (body.model or current.model).strip()
|
|
if not next_model:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="model is required",
|
|
)
|
|
next_effort = _normalize_reasoning_effort(
|
|
body.reasoning_effort
|
|
if "reasoning_effort" in body.model_fields_set
|
|
else current.reasoning_effort
|
|
)
|
|
await _validate_engine_selection(
|
|
provider=next_mode,
|
|
engine_url=next_url,
|
|
model=next_model,
|
|
reasoning_effort=next_effort,
|
|
)
|
|
next_config = AdminEngineConfigResponse(
|
|
engine_mode=next_mode,
|
|
engine_url=next_url,
|
|
model=next_model,
|
|
reasoning_effort=next_effort,
|
|
updated_by=principal.email,
|
|
updated_at=time.time(),
|
|
durable=False,
|
|
source="runtime_cache",
|
|
)
|
|
try:
|
|
pool = get_pool()
|
|
async with pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.admin_engine_config (
|
|
id, engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
|
|
)
|
|
VALUES (TRUE, $1, $2, $3, $4, $5, now())
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
engine_mode = EXCLUDED.engine_mode,
|
|
engine_url = EXCLUDED.engine_url,
|
|
model = EXCLUDED.model,
|
|
reasoning_effort = EXCLUDED.reasoning_effort,
|
|
updated_by = EXCLUDED.updated_by,
|
|
updated_at = now()
|
|
RETURNING engine_mode, engine_url, model, reasoning_effort, updated_by, updated_at
|
|
""",
|
|
next_config.engine_mode,
|
|
next_config.engine_url,
|
|
next_config.model,
|
|
next_config.reasoning_effort,
|
|
principal.email,
|
|
)
|
|
next_config = _engine_config_from_row(row)
|
|
except Exception as exc:
|
|
if settings.environment != "dev":
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail="engine config persistence unavailable",
|
|
) from exc
|
|
_ENGINE_CONFIG = next_config
|
|
await engine_client.configure(
|
|
base_url=next_config.engine_url,
|
|
engine_mode=next_config.engine_mode,
|
|
default_model=next_config.model,
|
|
default_reasoning_effort=next_config.reasoning_effort,
|
|
)
|
|
return next_config
|
|
|
|
|
|
@router.get("/users", response_model=AdminUsersResponse)
|
|
async def list_users(principal: AdminPrincipal) -> AdminUsersResponse:
|
|
"""Return users observed by the server-side auth/session boundary."""
|
|
users, durable = await list_managed_users()
|
|
if not durable:
|
|
require_runtime_fallback_allowed("admin user list")
|
|
return AdminUsersResponse(
|
|
source="database" if durable else "server_session_registry",
|
|
durable=durable,
|
|
users=[await _admin_user_response(user, durable=durable) for user in users],
|
|
)
|
|
|
|
|
|
@router.post("/users", response_model=AdminUserResponse, status_code=status.HTTP_201_CREATED)
|
|
async def create_user(
|
|
body: AdminUserCreate,
|
|
principal: AdminPrincipal,
|
|
) -> AdminUserResponse:
|
|
"""Create or reactivate a managed user without requiring that user to log in first."""
|
|
if body.role == "admin" or body.admin_access:
|
|
_assert_super_admin(principal)
|
|
user = await upsert_managed_user(
|
|
ManagedUserUpsertInput(
|
|
email=_normalize_email(body.email),
|
|
display_name=body.display_name,
|
|
role=body.role,
|
|
admin_access=body.admin_access,
|
|
learner_feedback_enabled=body.learner_feedback_enabled,
|
|
account_status=body.account_status,
|
|
affiliation=body.affiliation,
|
|
cohort_ids=body.cohort_ids,
|
|
reactivate=True,
|
|
)
|
|
)
|
|
users, durable = await list_managed_users()
|
|
if not durable:
|
|
require_runtime_fallback_allowed("admin user create")
|
|
return await _admin_user_response(user, durable=durable)
|
|
|
|
|
|
@router.patch("/users/{user_id}", response_model=AdminUserResponse)
|
|
async def patch_user(
|
|
user_id: str,
|
|
body: AdminUserPatch,
|
|
principal: AdminPrincipal,
|
|
) -> AdminUserResponse:
|
|
"""Update a server-known user's role/profile for the current API process."""
|
|
current = await get_managed_user(user_id)
|
|
if current is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
|
if _requires_super_admin_for_privilege_change(
|
|
current_role=current.role,
|
|
next_role=body.role,
|
|
current_admin_access=current.admin_access,
|
|
next_admin_access=body.admin_access,
|
|
):
|
|
_assert_super_admin(principal)
|
|
_assert_super_admin_target_mutable(current.email, body.admin_access)
|
|
next_user = await update_managed_user(
|
|
user_id,
|
|
ManagedUserPatch(
|
|
display_name=body.display_name,
|
|
role=body.role,
|
|
admin_access=body.admin_access,
|
|
learner_feedback_enabled=body.learner_feedback_enabled,
|
|
account_status=body.account_status,
|
|
affiliation=body.affiliation,
|
|
cohort_ids=body.cohort_ids,
|
|
),
|
|
)
|
|
if next_user is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
|
users, durable = await list_managed_users()
|
|
if not durable:
|
|
require_runtime_fallback_allowed("admin user update")
|
|
return await _admin_user_response(next_user, durable=durable)
|
|
|
|
|
|
@router.delete("/users/{user_id}", response_model=AdminUserDeleteResponse)
|
|
async def delete_user(
|
|
user_id: str,
|
|
principal: AdminPrincipal,
|
|
) -> AdminUserDeleteResponse:
|
|
"""Deactivate a managed user and revoke any active browser sessions."""
|
|
if user_id == principal.user_id:
|
|
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot deactivate current admin")
|
|
current = await get_managed_user(user_id)
|
|
if current is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
|
if is_super_admin_email(current.email):
|
|
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot deactivate super admin")
|
|
if not await deactivate_managed_user(user_id):
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
|
return AdminUserDeleteResponse(ok=True, user_id=user_id)
|