yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

운영 지표와 지원 요청 저장소 추가

Browse source
This commit is contained in:
Yun Chan 2026-06-28 20:13:06 +09:00
parent 50fa4ad432
commit e7ebb38177
20 changed files with 3038 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

358
apps/api/app/routes/admin.py
View file

@ -6,6 +6,7 @@ import time
from datetime import datetime, timezone
from decimal import Decimal
from typing import Annotated, Literal
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import BaseModel, Field
@ -28,7 +29,7 @@ from ..deps import Principal, require_admin_access
from ..engine_client import engine_client
from ..runtime_policy import require_runtime_fallback_allowed
from ..services.voice import voice_service
from ..services import rag
from ..services import evaluator, rag
from ..store import store
router = APIRouter(prefix="/admin", tags=["admin"])
@ -73,6 +74,14 @@ class AdminUsageBreakdown(BaseModel):
cost_usd: float
class AdminUsageDailyCost(BaseModel):
day: str
turns: int
tokens_in: int
tokens_out: int
cost_usd: float
class AdminUsageBudget(BaseModel):
limit_usd: float
used_ratio: float
@ -80,6 +89,17 @@ class AdminUsageBudget(BaseModel):
status: UsageBudgetStatus
class AdminUsageEvaluatorCache(BaseModel):
enabled: bool
entries: int
hits: int
misses: int
stores: int
evictions: int
requests: int
hit_rate: float
class AdminUsageResponse(BaseModel):
source: Literal["database", "server_session_registry"]
durable: bool
@ -91,7 +111,9 @@ class AdminUsageResponse(BaseModel):
tokens_out: int
cost_usd: float
budget: AdminUsageBudget
evaluator_cache: AdminUsageEvaluatorCache
by_provider: list[AdminUsageBreakdown]
daily_cost: list[AdminUsageDailyCost] = Field(default_factory=list)
class AdminHealthEvent(BaseModel):
@ -147,6 +169,11 @@ class AdminSupportTicketResponse(BaseModel):
subject: str
body: str
source_path: str
fingerprint: str
parent_ticket_id: str | None = None
duplicate_count: int = 0
duplicate_parent_candidate_id: str | None = None
child_ticket_count: int = 0
assigned_group: str
resolution_note: str
created_at: float
@ -195,6 +222,7 @@ class AdminTicketPatch(BaseModel):
priority: TicketPriority | None = None
assigned_group: str | None = Field(default=None, max_length=120)
resolution_note: str | None = Field(default=None, max_length=2000)
parent_ticket_id: str | None = Field(default=None, max_length=36)
class AdminUserResponse(BaseModel):
@ -298,6 +326,29 @@ def _usage_budget(cost_usd: float) -> AdminUsageBudget:
)
def _usage_evaluator_cache() -> AdminUsageEvaluatorCache:
stats = evaluator.evaluator_semantic_cache_stats()
hits = _safe_usage_int(stats.get("hits", 0))
misses = _safe_usage_int(stats.get("misses", 0))
requests = hits + misses
hit_rate = round(hits / requests, 4) if requests else 0.0
enabled = (
bool(settings.evaluator_semantic_cache_enabled)
and settings.evaluator_semantic_cache_ttl_seconds > 0
and settings.evaluator_semantic_cache_max_entries > 0
)
return AdminUsageEvaluatorCache(
enabled=enabled,
entries=_safe_usage_int(stats.get("entries", 0)),
hits=hits,
misses=misses,
stores=_safe_usage_int(stats.get("stores", 0)),
evictions=_safe_usage_int(stats.get("evictions", 0)),
requests=requests,
hit_rate=hit_rate,
)
async def _runtime_health_metrics(*, db_ok: bool) -> RuntimeHealthMetrics:
metrics = RuntimeHealthMetrics()
@ -401,6 +452,27 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
""",
window_days,
)
daily_rows = await conn.fetch(
"""
SELECT
to_char(date_trunc('day', created_at), 'YYYY-MM-DD') AS day,
COUNT(*) AS turns,
COALESCE(SUM(tokens_in), 0)::bigint AS tokens_in,
COALESCE(SUM(tokens_out), 0)::bigint AS tokens_out,
COALESCE(SUM(cost_usd), 0)::numeric AS cost_usd
FROM app.turns
WHERE created_at >= now() - ($1::int * interval '1 day')
AND speaker = 'client'
AND (
llm_provider IS NOT NULL OR model IS NOT NULL
OR tokens_in IS NOT NULL OR tokens_out IS NOT NULL
OR cost_usd IS NOT NULL
)
GROUP BY 1
ORDER BY 1
""",
window_days,
)
total_cost = round(_decimal_to_float(total_row["cost_usd"] if total_row else 0), 6)
return AdminUsageResponse(
@ -414,6 +486,7 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
tokens_out=_safe_usage_int(total_row["tokens_out"] if total_row else 0),
cost_usd=total_cost,
budget=_usage_budget(total_cost),
evaluator_cache=_usage_evaluator_cache(),
by_provider=[
AdminUsageBreakdown(
provider=str(row["provider"] or "unknown"),
@ -425,21 +498,32 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
)
for row in rows
],
daily_cost=[
AdminUsageDailyCost(
day=str(row["day"]),
turns=_safe_usage_int(row["turns"]),
tokens_in=_safe_usage_int(row["tokens_in"]),
tokens_out=_safe_usage_int(row["tokens_out"]),
cost_usd=round(_decimal_to_float(row["cost_usd"]), 6),
)
for row in daily_rows
],
)
async def _record_health_events(
*,
principal: Principal,
principal: Principal | None,
overall_status: HealthStatus,
environment: str,
engine_mode: str,
services: list[AdminServiceHealth],
) -> None:
) -> int:
if not services:
return
return 0
captured_by = principal.user_id if principal is not None else None
try:
async with acquire(role="admin", user_id=principal.user_id) as conn:
async with acquire(role="admin", user_id=captured_by) as conn:
await conn.executemany(
"""
INSERT INTO app.admin_health_event (
@ -467,14 +551,15 @@ async def _record_health_events(
service.detail,
service.metric,
service.load,
principal.user_id,
captured_by,
)
for service in services
],
)
return len(services)
except Exception:
# 헬스 화면 자체가 장애 확인 경로라, 이력 적재 실패가 응답을 막으면 안 된다.
return
return 0
async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
@ -498,23 +583,74 @@ async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
""",
window_hours,
)
rollup_rows = await conn.fetch(
"""
SELECT
r.service_key,
r.service_name,
r.sample_count,
r.ok_samples,
r.degraded_samples,
r.down_samples,
r.latest_status,
r.last_observed_at,
r.last_down_at
FROM app.admin_health_daily_rollup AS r
WHERE r.rollup_date >= (now() - ($1::int * interval '1 hour'))::date
AND r.rollup_date < current_date
AND NOT EXISTS (
SELECT 1
FROM app.admin_health_event AS e
WHERE e.observed_at::date = r.rollup_date
AND e.environment = r.environment
AND e.engine_mode = r.engine_mode
AND e.service_key = r.service_key
)
ORDER BY r.last_observed_at DESC, r.service_key
LIMIT 1000
""",
window_hours,
)
events = [_health_event_from_row(row) for row in rows]
service_buckets: dict[str, AdminUptimeServiceSummary] = {}
for event in events:
current = service_buckets.get(event.service_key)
def ensure_bucket(
*,
service_key: str,
service_name: str,
latest_status: HealthStatus,
latest_observed_at: float | None,
) -> AdminUptimeServiceSummary:
current = service_buckets.get(service_key)
if current is None:
current = AdminUptimeServiceSummary(
service_key=event.service_key,
service_name=event.service_name,
service_key=service_key,
service_name=service_name,
samples=0,
ok_samples=0,
degraded_samples=0,
down_samples=0,
latest_status=event.service_status,
latest_observed_at=event.observed_at,
latest_status=latest_status,
latest_observed_at=latest_observed_at,
)
service_buckets[event.service_key] = current
service_buckets[service_key] = current
return current
if latest_observed_at is not None and (
current.latest_observed_at is None
or latest_observed_at > current.latest_observed_at
):
current.latest_status = latest_status
current.latest_observed_at = latest_observed_at
return current
for event in events:
current = ensure_bucket(
service_key=event.service_key,
service_name=event.service_name,
latest_status=event.service_status,
latest_observed_at=event.observed_at,
)
current.samples += 1
if event.service_status == "ok":
current.ok_samples += 1
@ -523,9 +659,29 @@ async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
else:
current.down_samples += 1
ok_samples = sum(1 for event in events if event.service_status == "ok")
sample_count = len(events)
rollup_last_down: list[float] = []
for row in rollup_rows:
latest_observed_at = _row_ts(row["last_observed_at"])
current = ensure_bucket(
service_key=str(row["service_key"]),
service_name=str(row["service_name"]),
latest_status=row["latest_status"],
latest_observed_at=latest_observed_at,
)
current.samples += int(row["sample_count"] or 0)
current.ok_samples += int(row["ok_samples"] or 0)
current.degraded_samples += int(row["degraded_samples"] or 0)
current.down_samples += int(row["down_samples"] or 0)
last_down = _row_ts(row["last_down_at"])
if last_down is not None:
rollup_last_down.append(last_down)
ok_samples = sum(1 for event in events if event.service_status == "ok") + sum(
int(row["ok_samples"] or 0) for row in rollup_rows
)
sample_count = len(events) + sum(int(row["sample_count"] or 0) for row in rollup_rows)
down_times = [event.observed_at for event in events if event.service_status == "down"]
down_times.extend(rollup_last_down)
return AdminUptimeResponse(
source="database",
durable=True,
@ -533,8 +689,10 @@ async def _uptime_from_database(window_hours: int) -> AdminUptimeResponse:
generated_at=time.time(),
sample_count=sample_count,
ok_ratio=round(ok_samples / sample_count, 4) if sample_count else 0.0,
degraded_events=sum(1 for event in events if event.service_status == "degraded"),
down_events=sum(1 for event in events if event.service_status == "down"),
degraded_events=sum(1 for event in events if event.service_status == "degraded")
+ sum(int(row["degraded_samples"] or 0) for row in rollup_rows),
down_events=sum(1 for event in events if event.service_status == "down")
+ sum(int(row["down_samples"] or 0) for row in rollup_rows),
last_down_at=max(down_times) if down_times else None,
services=sorted(
service_buckets.values(),
@ -573,14 +731,32 @@ async def _tickets_from_database(
t.subject,
t.body,
t.source_path,
t.fingerprint,
t.parent_ticket_id,
t.assigned_group,
t.resolution_note,
t.created_at,
t.updated_at,
t.resolved_at,
COALESCE(dup.duplicate_count, 0) AS duplicate_count,
dup.parent_candidate_id AS duplicate_parent_candidate_id,
COALESCE(child.child_ticket_count, 0) AS child_ticket_count,
COALESCE(ev.event_count, 0) AS event_count,
ev.last_event_at
FROM app.support_ticket AS t
LEFT JOIN LATERAL (
SELECT
count(*) FILTER (WHERE id <> t.id)::int AS duplicate_count,
(array_agg(id::text ORDER BY created_at ASC, id ASC))[1] AS parent_candidate_id
FROM app.support_ticket
WHERE fingerprint <> ''
AND fingerprint = t.fingerprint
) AS dup ON TRUE
LEFT JOIN LATERAL (
SELECT count(*)::int AS child_ticket_count
FROM app.support_ticket
WHERE parent_ticket_id = t.id
) AS child ON TRUE
LEFT JOIN LATERAL (
SELECT count(*)::int AS event_count, max(created_at) AS last_event_at
FROM audit.audit_log
@ -652,12 +828,14 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
tokens_out = 0
cost_usd = 0.0
buckets: dict[tuple[str, str], dict[str, int | float]] = {}
daily_buckets: dict[str, dict[str, int | float]] = {}
for sess in store.list():
for turn in getattr(sess, "turns", []) or []:
if getattr(turn, "speaker", "") != "client":
continue
if float(getattr(turn, "created_at", 0.0) or 0.0) < window_start:
created_at = float(getattr(turn, "created_at", 0.0) or 0.0)
if created_at < window_start:
continue
total_turns += 1
provider = str(getattr(turn, "llm_provider", None) or "unknown")
@ -687,6 +865,15 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
bucket["tokens_in"] = int(bucket["tokens_in"]) + turn_tokens_in
bucket["tokens_out"] = int(bucket["tokens_out"]) + turn_tokens_out
bucket["cost_usd"] = float(bucket["cost_usd"]) + turn_cost
day = datetime.fromtimestamp(created_at, timezone.utc).strftime("%Y-%m-%d")
daily_bucket = daily_buckets.setdefault(
day,
{"turns": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0},
)
daily_bucket["turns"] = int(daily_bucket["turns"]) + 1
daily_bucket["tokens_in"] = int(daily_bucket["tokens_in"]) + turn_tokens_in
daily_bucket["tokens_out"] = int(daily_bucket["tokens_out"]) + turn_tokens_out
daily_bucket["cost_usd"] = float(daily_bucket["cost_usd"]) + turn_cost
by_provider = [
AdminUsageBreakdown(
@ -719,7 +906,18 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
tokens_out=tokens_out,
cost_usd=total_cost,
budget=_usage_budget(total_cost),
evaluator_cache=_usage_evaluator_cache(),
by_provider=by_provider,
daily_cost=[
AdminUsageDailyCost(
day=day,
turns=int(values["turns"]),
tokens_in=int(values["tokens_in"]),
tokens_out=int(values["tokens_out"]),
cost_usd=round(float(values["cost_usd"]), 6),
)
for day, values in sorted(daily_buckets.items())
],
)
@ -850,6 +1048,19 @@ def _ticket_from_row(row) -> AdminSupportTicketResponse:
subject=row["subject"],
body=row["body"],
source_path=row["source_path"],
fingerprint=_row_value(row, "fingerprint", "") or "",
parent_ticket_id=(
str(_row_value(row, "parent_ticket_id"))
if _row_value(row, "parent_ticket_id") is not None
else None
),
duplicate_count=int(_row_value(row, "duplicate_count", 0) or 0),
duplicate_parent_candidate_id=(
str(_row_value(row, "duplicate_parent_candidate_id"))
if _row_value(row, "duplicate_parent_candidate_id") is not None
else None
),
child_ticket_count=int(_row_value(row, "child_ticket_count", 0) or 0),
assigned_group=row["assigned_group"],
resolution_note=row["resolution_note"],
created_at=_row_ts(row["created_at"]) or 0.0,
@ -925,12 +1136,31 @@ def _unavailable_tickets() -> AdminTicketsResponse:
)
def _normalize_parent_ticket_id(value: str | None) -> str | None:
if value is None:
return None
stripped = value.strip()
if not stripped:
return None
try:
return str(UUID(stripped))
except ValueError as exc:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="invalid parent ticket id",
) from exc
def _ticket_change_detail(old_row, new_row) -> dict[str, object]:
changed_fields: list[str] = []
detail: dict[str, object] = {"changed_fields": changed_fields}
for field in ("status", "priority", "assigned_group"):
for field in ("status", "priority", "assigned_group", "parent_ticket_id"):
before = _row_value(old_row, field, "")
after = _row_value(new_row, field, "")
if before is not None:
before = str(before)
if after is not None:
after = str(after)
if before != after:
changed_fields.append(field)
detail[field] = {"from": before, "to": after}
@ -1060,9 +1290,7 @@ async def _admin_user_response(user, *, durable: bool) -> AdminUserResponse:
)
@router.get("/health", response_model=AdminHealthResponse)
async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
"""Return operational health from live backend checks."""
async def _build_admin_health_response() -> AdminHealthResponse:
current_engine = await _current_engine_config()
db_ok = await healthcheck()
engine_started = time.perf_counter()
@ -1148,13 +1376,28 @@ async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
engine_mode=current_engine.engine_mode,
services=services,
)
await _record_health_events(
return response
async def record_admin_health_sample(
*, principal: Principal | None = None
) -> tuple[AdminHealthResponse, int]:
"""Collect and persist one synthetic/admin health sample."""
response = await _build_admin_health_response()
recorded_count = await _record_health_events(
principal=principal,
overall_status=response.status,
environment=response.environment,
engine_mode=response.engine_mode,
services=response.services,
)
return response, recorded_count
@router.get("/health", response_model=AdminHealthResponse)
async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
"""Return operational health from live backend checks."""
response, _ = await record_admin_health_sample(principal=principal)
return response
@ -1229,7 +1472,8 @@ async def patch_ticket(
status,
source_path,
assigned_group,
resolution_note
resolution_note,
parent_ticket_id
FROM app.support_ticket
WHERE id = $1::uuid
""",
@ -1237,6 +1481,44 @@ async def patch_ticket(
)
if old_row is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
parent_specified = "parent_ticket_id" in body.model_fields_set
next_parent_id = (
_normalize_parent_ticket_id(body.parent_ticket_id)
if parent_specified
else None
)
if parent_specified and next_parent_id is not None:
if next_parent_id == str(UUID(ticket_id)):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="ticket cannot be its own parent",
)
parent_check = await conn.fetchrow(
"""
WITH RECURSIVE ancestors AS (
SELECT id, parent_ticket_id
FROM app.support_ticket
WHERE id = $1::uuid
UNION ALL
SELECT t.id, t.parent_ticket_id
FROM app.support_ticket AS t
JOIN ancestors AS a ON t.id = a.parent_ticket_id
WHERE a.parent_ticket_id IS NOT NULL
)
SELECT
EXISTS (SELECT 1 FROM ancestors) AS parent_exists,
EXISTS (SELECT 1 FROM ancestors WHERE id = $2::uuid) AS creates_cycle
""",
next_parent_id,
ticket_id,
)
if not parent_check or not parent_check["parent_exists"]:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="parent ticket not found")
if parent_check["creates_cycle"]:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="ticket parent would create a cycle",
)
row = await conn.fetchrow(
"""
UPDATE app.support_ticket SET
@ -1244,6 +1526,10 @@ async def patch_ticket(
priority = COALESCE($3, priority),
assigned_group = COALESCE($4, assigned_group),
resolution_note = COALESCE($5, resolution_note),
parent_ticket_id = CASE
WHEN $6::bool THEN $7::uuid
ELSE parent_ticket_id
END,
resolved_at = CASE
WHEN COALESCE($2, status) IN ('resolved', 'closed')
THEN COALESCE(resolved_at, now())
@ -1266,6 +1552,8 @@ async def patch_ticket(
subject,
body,
source_path,
fingerprint,
parent_ticket_id,
assigned_group,
resolution_note,
created_at,
@ -1277,6 +1565,8 @@ async def patch_ticket(
body.priority,
body.assigned_group.strip() if body.assigned_group is not None else None,
body.resolution_note.strip() if body.resolution_note is not None else None,
parent_specified,
next_parent_id,
)
if row is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="ticket not found")
@ -1302,14 +1592,32 @@ async def patch_ticket(
t.subject,
t.body,
t.source_path,
t.fingerprint,
t.parent_ticket_id,
t.assigned_group,
t.resolution_note,
t.created_at,
t.updated_at,
t.resolved_at,
COALESCE(dup.duplicate_count, 0) AS duplicate_count,
dup.parent_candidate_id AS duplicate_parent_candidate_id,
COALESCE(child.child_ticket_count, 0) AS child_ticket_count,
COALESCE(ev.event_count, 0) AS event_count,
ev.last_event_at
FROM app.support_ticket AS t
LEFT JOIN LATERAL (
SELECT
count(*) FILTER (WHERE id <> t.id)::int AS duplicate_count,
(array_agg(id::text ORDER BY created_at ASC, id ASC))[1] AS parent_candidate_id
FROM app.support_ticket
WHERE fingerprint <> ''
AND fingerprint = t.fingerprint
) AS dup ON TRUE
LEFT JOIN LATERAL (
SELECT count(*)::int AS child_ticket_count
FROM app.support_ticket
WHERE parent_ticket_id = t.id
) AS child ON TRUE
LEFT JOIN LATERAL (
SELECT count(*)::int AS event_count, max(created_at) AS last_event_at
FROM audit.audit_log