Stabilize runtime auth and E2E coverage
This commit is contained in:
Yun Chan
parent
6a3e3b541c
commit
188e899394
133 changed files with 55987 additions and 6775 deletions
539
apps/api/app/routes/admin.py
Normal file
539
apps/api/app/routes/admin.py
Normal file
|
|
@ -0,0 +1,539 @@
|
|||
"""Admin operations routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..auth_sessions import (
|
||||
active_session_count,
|
||||
deactivate_managed_user,
|
||||
list_managed_users,
|
||||
update_managed_user,
|
||||
upsert_managed_user,
|
||||
)
|
||||
from ..config import settings
|
||||
from ..db import acquire, get_pool, healthcheck
|
||||
from ..deps import Principal, Role, require_role
|
||||
from ..engine_client import engine_client
|
||||
from ..runtime_policy import require_runtime_fallback_allowed
|
||||
from ..services.voice import voice_service
|
||||
from ..services import rag
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
AdminPrincipal = Annotated[Principal, Depends(require_role(Role.ADMIN))]
|
||||
HealthStatus = Literal["ok", "degraded", "down"]
|
||||
|
||||
|
||||
class AdminServiceHealth(BaseModel):
|
||||
key: str
|
||||
name: str
|
||||
status: HealthStatus
|
||||
detail: str
|
||||
metric: str
|
||||
load: float
|
||||
|
||||
|
||||
class AdminHealthResponse(BaseModel):
|
||||
status: HealthStatus
|
||||
environment: str
|
||||
engine_mode: str
|
||||
services: list[AdminServiceHealth]
|
||||
|
||||
|
||||
class AdminEngineConfigResponse(BaseModel):
|
||||
engine_mode: str
|
||||
engine_url: str
|
||||
model: str
|
||||
updated_by: str | None = None
|
||||
updated_at: float | None = None
|
||||
durable: bool = False
|
||||
source: Literal["database", "runtime_cache", "runtime_default"] = "runtime_default"
|
||||
|
||||
|
||||
class AdminEngineConfigPatch(BaseModel):
|
||||
engine_mode: str | None = None
|
||||
engine_url: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
|
||||
RoleName = Literal["learner", "teacher", "admin"]
|
||||
|
||||
|
||||
class AdminUserResponse(BaseModel):
|
||||
user_id: str
|
||||
email: str
|
||||
display_name: str
|
||||
role: RoleName
|
||||
cohort_ids: list[str]
|
||||
affiliation: str
|
||||
active_sessions: int
|
||||
created_at: float
|
||||
last_seen_at: float
|
||||
source: Literal["database", "server_session_registry"]
|
||||
|
||||
|
||||
class AdminUsersResponse(BaseModel):
|
||||
source: Literal["database", "server_session_registry"]
|
||||
durable: bool
|
||||
users: list[AdminUserResponse]
|
||||
|
||||
|
||||
class AdminUserPatch(BaseModel):
|
||||
display_name: str | None = Field(default=None, min_length=1, max_length=80)
|
||||
role: RoleName | None = None
|
||||
affiliation: str | None = Field(default=None, max_length=120)
|
||||
cohort_ids: list[str] | None = None
|
||||
|
||||
|
||||
class RuntimeHealthMetrics(BaseModel):
|
||||
engine_latency_ms: float | None = None
|
||||
db_pool_size: int = 0
|
||||
db_pool_idle: int = 0
|
||||
db_pool_max: int = 0
|
||||
active_users: int = 0
|
||||
active_auth_sessions: int = 0
|
||||
active_sessions: int = 0
|
||||
ended_sessions: int = 0
|
||||
pending_reviews: int = 0
|
||||
|
||||
|
||||
def _clamp01(value: float) -> float:
|
||||
return round(max(0.0, min(1.0, value)), 3)
|
||||
|
||||
|
||||
def _pool_load(metrics: RuntimeHealthMetrics) -> float:
|
||||
if metrics.db_pool_max <= 0:
|
||||
return 0.0
|
||||
busy = max(0, metrics.db_pool_size - metrics.db_pool_idle)
|
||||
return _clamp01(busy / metrics.db_pool_max)
|
||||
|
||||
|
||||
def _workload_load(count: int, expected_capacity: int) -> float:
|
||||
if expected_capacity <= 0:
|
||||
return 0.0
|
||||
return _clamp01(count / expected_capacity)
|
||||
|
||||
|
||||
async def _runtime_health_metrics(*, db_ok: bool) -> RuntimeHealthMetrics:
|
||||
metrics = RuntimeHealthMetrics()
|
||||
|
||||
try:
|
||||
pool = get_pool()
|
||||
metrics.db_pool_size = int(pool.get_size())
|
||||
metrics.db_pool_idle = int(pool.get_idle_size())
|
||||
metrics.db_pool_max = int(pool.get_max_size())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not db_ok:
|
||||
return metrics
|
||||
|
||||
try:
|
||||
async with acquire(role="admin") as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM app.app_user WHERE is_active) AS active_users,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM app.auth_session
|
||||
WHERE revoked_at IS NULL AND expires_at > now()
|
||||
) AS active_auth_sessions,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM app.sessions
|
||||
WHERE ended_at IS NULL
|
||||
) AS active_sessions,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM app.sessions
|
||||
WHERE ended_at IS NOT NULL
|
||||
) AS ended_sessions,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM app.sessions s
|
||||
LEFT JOIN app.session_summary ss ON ss.session_id = s.id
|
||||
WHERE s.ended_at IS NOT NULL AND ss.session_id IS NULL
|
||||
) AS pending_reviews
|
||||
"""
|
||||
)
|
||||
if row is not None:
|
||||
metrics.active_users = int(row["active_users"] or 0)
|
||||
metrics.active_auth_sessions = int(row["active_auth_sessions"] or 0)
|
||||
metrics.active_sessions = int(row["active_sessions"] or 0)
|
||||
metrics.ended_sessions = int(row["ended_sessions"] or 0)
|
||||
metrics.pending_reviews = int(row["pending_reviews"] or 0)
|
||||
except Exception:
|
||||
return metrics
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
class AdminUserCreate(BaseModel):
|
||||
email: str = Field(..., min_length=3, max_length=254)
|
||||
display_name: str = Field(..., min_length=1, max_length=80)
|
||||
role: RoleName = "learner"
|
||||
affiliation: str | None = Field(default=None, max_length=120)
|
||||
cohort_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AdminUserDeleteResponse(BaseModel):
|
||||
ok: bool
|
||||
user_id: str
|
||||
|
||||
|
||||
_ENGINE_CONFIG: AdminEngineConfigResponse | None = None
|
||||
ENGINE_MODES = {"claude_api", "claude_cli", "openai", "solar"}
|
||||
ENGINE_MODE_ALIASES = {"messages_api": "claude_api"}
|
||||
|
||||
|
||||
def _normalize_email(value: str) -> str:
|
||||
email = value.strip().lower()
|
||||
if "@" not in email:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
|
||||
local, domain = email.rsplit("@", 1)
|
||||
if not local or not domain:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail="email is invalid")
|
||||
allowed = {item.strip().lower().lstrip("@") for item in settings.auth_allowed_email_domains if item.strip()}
|
||||
if domain not in allowed:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="email domain is not allowed")
|
||||
return email
|
||||
|
||||
|
||||
def _default_engine_config() -> AdminEngineConfigResponse:
|
||||
return AdminEngineConfigResponse(
|
||||
engine_mode=settings.engine_mode,
|
||||
engine_url=settings.engine_url,
|
||||
model="gateway-default",
|
||||
durable=False,
|
||||
source="runtime_default",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_engine_mode(value: str) -> str:
|
||||
mode = ENGINE_MODE_ALIASES.get(value.strip(), value.strip())
|
||||
if mode not in ENGINE_MODES:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"unsupported engine mode {value}",
|
||||
)
|
||||
return mode
|
||||
|
||||
|
||||
def _normalize_engine_url(value: str) -> str:
|
||||
url = value.strip().rstrip("/")
|
||||
if not (url.startswith("http://") or url.startswith("https://")):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="engine_url must start with http:// or https://",
|
||||
)
|
||||
return url
|
||||
|
||||
|
||||
def _updated_at_ts(value: datetime | None) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.timestamp()
|
||||
|
||||
|
||||
def _engine_config_from_row(row) -> AdminEngineConfigResponse:
|
||||
return AdminEngineConfigResponse(
|
||||
engine_mode=_normalize_engine_mode(row["engine_mode"]),
|
||||
engine_url=_normalize_engine_url(row["engine_url"]),
|
||||
model=row["model"],
|
||||
updated_by=row["updated_by"],
|
||||
updated_at=_updated_at_ts(row["updated_at"]),
|
||||
durable=True,
|
||||
source="database",
|
||||
)
|
||||
|
||||
|
||||
async def _current_engine_config() -> AdminEngineConfigResponse:
|
||||
if _ENGINE_CONFIG is not None:
|
||||
return _ENGINE_CONFIG
|
||||
try:
|
||||
pool = get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT engine_mode, engine_url, model, updated_by, updated_at
|
||||
FROM app.admin_engine_config
|
||||
WHERE id = TRUE
|
||||
"""
|
||||
)
|
||||
if row is not None:
|
||||
return _engine_config_from_row(row)
|
||||
except Exception:
|
||||
require_runtime_fallback_allowed("admin engine config")
|
||||
require_runtime_fallback_allowed("admin engine config")
|
||||
return _default_engine_config()
|
||||
|
||||
|
||||
async def apply_engine_config_from_store() -> AdminEngineConfigResponse:
|
||||
"""Load admin engine settings and apply them to the live engine client."""
|
||||
config = await _current_engine_config()
|
||||
await engine_client.configure(
|
||||
base_url=config.engine_url,
|
||||
engine_mode=config.engine_mode,
|
||||
default_model=config.model,
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _overall_status(services: list[AdminServiceHealth]) -> HealthStatus:
|
||||
if any(s.status == "down" for s in services):
|
||||
return "down"
|
||||
if any(s.status == "degraded" for s in services):
|
||||
return "degraded"
|
||||
return "ok"
|
||||
|
||||
|
||||
def _engine_unavailable_detail(detail: str) -> str:
|
||||
if detail.lstrip().startswith("{") and '"ok":false' in detail:
|
||||
return "Engine readiness failed"
|
||||
return detail
|
||||
|
||||
|
||||
async def _admin_user_response(user, *, durable: bool) -> AdminUserResponse:
|
||||
return AdminUserResponse(
|
||||
user_id=user.user_id,
|
||||
email=user.email,
|
||||
display_name=user.display_name,
|
||||
role=user.role,
|
||||
cohort_ids=user.cohort_ids,
|
||||
affiliation=user.affiliation,
|
||||
active_sessions=await active_session_count(user.user_id),
|
||||
created_at=user.created_at,
|
||||
last_seen_at=user.last_seen_at,
|
||||
source="database" if durable else "server_session_registry",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/health", response_model=AdminHealthResponse)
|
||||
async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
|
||||
"""Return operational health from live backend checks."""
|
||||
current_engine = await _current_engine_config()
|
||||
db_ok = await healthcheck()
|
||||
engine_started = time.perf_counter()
|
||||
engine = await engine_client.health_detail()
|
||||
engine_ok = bool(engine.get("ok"))
|
||||
engine_detail = _engine_unavailable_detail(
|
||||
str(engine.get("detail") or "engine readiness unavailable")
|
||||
)
|
||||
engine_latency_ms = (time.perf_counter() - engine_started) * 1000
|
||||
voice_ok = voice_service.is_available()
|
||||
metrics = await _runtime_health_metrics(db_ok=db_ok)
|
||||
metrics.engine_latency_ms = engine_latency_ms if engine_ok else None
|
||||
pool_load = _pool_load(metrics)
|
||||
session_load = _workload_load(metrics.active_sessions, 50)
|
||||
review_load = _workload_load(metrics.pending_reviews, 50)
|
||||
runtime_fallback_is_enabled = settings.environment == "dev"
|
||||
db_status: HealthStatus = "ok" if db_ok else ("degraded" if runtime_fallback_is_enabled else "down")
|
||||
db_detail = (
|
||||
"사용자, 세션, 리뷰 저장"
|
||||
if db_ok
|
||||
else (
|
||||
"DB 연결 전까지 비영구 개발 런타임 기록 사용"
|
||||
if runtime_fallback_is_enabled
|
||||
else "DB 저장소에 연결할 수 없습니다"
|
||||
)
|
||||
)
|
||||
db_metric = (
|
||||
f"풀 {max(0, metrics.db_pool_size - metrics.db_pool_idle)}/{metrics.db_pool_max}"
|
||||
if db_ok
|
||||
else ("비영구 런타임 기록" if runtime_fallback_is_enabled else "저장소 중단")
|
||||
)
|
||||
|
||||
services = [
|
||||
AdminServiceHealth(
|
||||
key="engine",
|
||||
name="응답 생성",
|
||||
status="ok" if engine_ok else "down",
|
||||
detail="AI 엔진 생성 준비 완료" if engine_ok else engine_detail,
|
||||
metric=f"{engine_latency_ms:.0f}ms" if engine_ok else "로그인/설정 필요",
|
||||
load=_clamp01(engine_latency_ms / 1500) if engine_ok else 0.0,
|
||||
),
|
||||
AdminServiceHealth(
|
||||
key="db",
|
||||
name="영구 저장소",
|
||||
status=db_status,
|
||||
detail=db_detail,
|
||||
metric=db_metric,
|
||||
load=max(pool_load, session_load) if db_ok else 0.0,
|
||||
),
|
||||
AdminServiceHealth(
|
||||
key="voice",
|
||||
name="음성 입력",
|
||||
status="ok" if voice_ok else "degraded",
|
||||
detail="음성 입력과 재생",
|
||||
metric="OpenAI 연결" if voice_ok else "설정 필요",
|
||||
load=0.05 if voice_ok else 0.0,
|
||||
),
|
||||
AdminServiceHealth(
|
||||
key="evaluation",
|
||||
name="리뷰 생성",
|
||||
status="ok" if engine_ok else "degraded",
|
||||
detail="회기 종료 후 피드백 생성",
|
||||
metric=f"대기 {metrics.pending_reviews}건",
|
||||
load=review_load if engine_ok else 0.0,
|
||||
),
|
||||
AdminServiceHealth(
|
||||
key="kb",
|
||||
name="지식 검색",
|
||||
status="ok" if db_ok else "degraded",
|
||||
detail=f"검색 기준값 {rag.CRAG_TOP1_THRESHOLD}",
|
||||
metric=(
|
||||
f"활성 세션 {metrics.active_sessions}건"
|
||||
if db_ok
|
||||
else "대기 중"
|
||||
),
|
||||
load=max(pool_load, session_load) if db_ok else 0.0,
|
||||
),
|
||||
]
|
||||
|
||||
return AdminHealthResponse(
|
||||
status=_overall_status(services),
|
||||
environment=settings.environment,
|
||||
engine_mode=current_engine.engine_mode,
|
||||
services=services,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/engine-config", response_model=AdminEngineConfigResponse)
|
||||
async def get_engine_config(principal: AdminPrincipal) -> AdminEngineConfigResponse:
|
||||
"""Return the current admin-managed engine settings."""
|
||||
return await _current_engine_config()
|
||||
|
||||
|
||||
@router.patch("/engine-config", response_model=AdminEngineConfigResponse)
|
||||
async def patch_engine_config(
|
||||
body: AdminEngineConfigPatch,
|
||||
principal: AdminPrincipal,
|
||||
) -> AdminEngineConfigResponse:
|
||||
"""Persist engine settings for administrators."""
|
||||
global _ENGINE_CONFIG
|
||||
|
||||
current = await _current_engine_config()
|
||||
next_mode = _normalize_engine_mode(body.engine_mode or current.engine_mode)
|
||||
next_url = _normalize_engine_url(body.engine_url or current.engine_url)
|
||||
next_config = AdminEngineConfigResponse(
|
||||
engine_mode=next_mode,
|
||||
engine_url=next_url,
|
||||
model=(body.model or current.model).strip(),
|
||||
updated_by=principal.email,
|
||||
updated_at=time.time(),
|
||||
durable=False,
|
||||
source="runtime_cache",
|
||||
)
|
||||
try:
|
||||
pool = get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
INSERT INTO app.admin_engine_config (
|
||||
id, engine_mode, engine_url, model, updated_by, updated_at
|
||||
)
|
||||
VALUES (TRUE, $1, $2, $3, $4, now())
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
engine_mode = EXCLUDED.engine_mode,
|
||||
engine_url = EXCLUDED.engine_url,
|
||||
model = EXCLUDED.model,
|
||||
updated_by = EXCLUDED.updated_by,
|
||||
updated_at = now()
|
||||
RETURNING engine_mode, engine_url, model, updated_by, updated_at
|
||||
""",
|
||||
next_config.engine_mode,
|
||||
next_config.engine_url,
|
||||
next_config.model,
|
||||
principal.email,
|
||||
)
|
||||
next_config = _engine_config_from_row(row)
|
||||
except Exception as exc:
|
||||
if settings.environment != "dev":
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="engine config persistence unavailable",
|
||||
) from exc
|
||||
_ENGINE_CONFIG = next_config
|
||||
await engine_client.configure(
|
||||
base_url=next_config.engine_url,
|
||||
engine_mode=next_config.engine_mode,
|
||||
default_model=next_config.model,
|
||||
)
|
||||
return next_config
|
||||
|
||||
|
||||
@router.get("/users", response_model=AdminUsersResponse)
|
||||
async def list_users(principal: AdminPrincipal) -> AdminUsersResponse:
|
||||
"""Return users observed by the server-side auth/session boundary."""
|
||||
users, durable = await list_managed_users()
|
||||
if not durable:
|
||||
require_runtime_fallback_allowed("admin user list")
|
||||
return AdminUsersResponse(
|
||||
source="database" if durable else "server_session_registry",
|
||||
durable=durable,
|
||||
users=[await _admin_user_response(user, durable=durable) for user in users],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/users", response_model=AdminUserResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_user(
|
||||
body: AdminUserCreate,
|
||||
principal: AdminPrincipal,
|
||||
) -> AdminUserResponse:
|
||||
"""Create or reactivate a managed user without requiring that user to log in first."""
|
||||
user = await upsert_managed_user(
|
||||
email=_normalize_email(body.email),
|
||||
display_name=body.display_name,
|
||||
role=body.role,
|
||||
affiliation=body.affiliation,
|
||||
cohort_ids=body.cohort_ids,
|
||||
reactivate=True,
|
||||
)
|
||||
users, durable = await list_managed_users()
|
||||
if not durable:
|
||||
require_runtime_fallback_allowed("admin user create")
|
||||
return await _admin_user_response(user, durable=durable)
|
||||
|
||||
|
||||
@router.patch("/users/{user_id}", response_model=AdminUserResponse)
|
||||
async def patch_user(
|
||||
user_id: str,
|
||||
body: AdminUserPatch,
|
||||
principal: AdminPrincipal,
|
||||
) -> AdminUserResponse:
|
||||
"""Update a server-known user's role/profile for the current API process."""
|
||||
next_user = await update_managed_user(
|
||||
user_id,
|
||||
display_name=body.display_name,
|
||||
role=body.role,
|
||||
affiliation=body.affiliation,
|
||||
cohort_ids=body.cohort_ids,
|
||||
)
|
||||
if next_user is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
||||
users, durable = await list_managed_users()
|
||||
if not durable:
|
||||
require_runtime_fallback_allowed("admin user update")
|
||||
return await _admin_user_response(next_user, durable=durable)
|
||||
|
||||
|
||||
@router.delete("/users/{user_id}", response_model=AdminUserDeleteResponse)
|
||||
async def delete_user(
|
||||
user_id: str,
|
||||
principal: AdminPrincipal,
|
||||
) -> AdminUserDeleteResponse:
|
||||
"""Deactivate a managed user and revoke any active browser sessions."""
|
||||
if user_id == principal.user_id:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="cannot deactivate current admin")
|
||||
if not await deactivate_managed_user(user_id):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="user not found")
|
||||
return AdminUserDeleteResponse(ok=True, user_id=user_id)
|
||||
Loading…
Add table
Add a link
Reference in a new issue