G0~G8 성과·동맹 측정 OS 작업 일괄 고정

8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본
폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을
git 이력으로 고정하는 것이 목적이다.

- contracts/routes/services: measurement, outcome_trajectory, rupture_repair,
  deliberate_practice, calibration_transfer, supervision_research,
  multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트
- infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행)
- apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가
- docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG
- scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트

engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan 2026-08-08 01:30:53 +09:00
parent 93dd8f82d7
commit 16e791e044
390 changed files with 243188 additions and 499 deletions

View file

@ -37,8 +37,13 @@ from ..db import acquire, get_pool, healthcheck
from ..deps import Principal, require_admin_access
from ..engine_client import engine_client
from ..runtime_policy import require_runtime_fallback_allowed
from ..services.voice import voice_service
from ..services import evaluator, notifications, rag
from ..services.llm_pricing import (
estimate_reference_cost,
provider_uses_reference_cost,
)
from ..services.voice import voice_service
from ..services.voice_runtime import VoiceRuntimeSnapshot, voice_runtime_metrics
from ..store import store
router = APIRouter(prefix="/admin", tags=["admin"])
@ -46,6 +51,12 @@ router = APIRouter(prefix="/admin", tags=["admin"])
AdminPrincipal = Annotated[Principal, Depends(require_admin_access())]
HealthStatus = Literal["ok", "degraded", "down"]
UsageBudgetStatus = Literal["disabled", "ok", "warn", "exceeded"]
UsageCostBasis = Literal[
"provider_estimate",
"provider_reported",
"reference_rate",
"unavailable",
]
TicketCategory = Literal[
"account_access",
"session_review",
@ -69,9 +80,24 @@ METERED_CLIENT_TURN_FILTER_SQL = """
USAGE_AGGREGATE_COLUMNS_SQL = """
COUNT(*) AS turns,
COUNT(*) FILTER (
WHERE COALESCE(tokens_in, 0) > 0 OR COALESCE(tokens_out, 0) > 0
) AS token_metered_turns,
COUNT(*) FILTER (
WHERE COALESCE(tokens_in, 0) <= 0 AND COALESCE(tokens_out, 0) <= 0
) AS token_unmetered_turns,
COALESCE(SUM(tokens_in), 0)::bigint AS tokens_in,
COALESCE(SUM(tokens_out), 0)::bigint AS tokens_out,
COALESCE(SUM(cost_usd), 0)::numeric AS cost_usd
COALESCE(SUM(cost_usd), 0)::numeric AS cost_usd,
COUNT(*) FILTER (WHERE COALESCE(cost_usd, 0) <= 0) AS unpriced_turns,
COALESCE(
SUM(tokens_in) FILTER (WHERE COALESCE(cost_usd, 0) <= 0),
0
)::bigint AS unpriced_tokens_in,
COALESCE(
SUM(tokens_out) FILTER (WHERE COALESCE(cost_usd, 0) <= 0),
0
)::bigint AS unpriced_tokens_out
"""
SUPPORT_TICKET_DETAIL_FROM_SQL = """
@ -143,9 +169,16 @@ class AdminUsageBreakdown(BaseModel):
provider: str
model: str
turns: int
token_metered_turns: int = 0
token_unmetered_turns: int = 0
tokens_in: int
tokens_out: int
cost_usd: float
recorded_cost_usd: float = 0.0
estimated_cost_usd: float = 0.0
cost_basis: UsageCostBasis = "provider_reported"
rate_label: str | None = None
rate_source_url: str | None = None
class AdminUsageDailyCost(BaseModel):
@ -181,9 +214,13 @@ class AdminUsageResponse(BaseModel):
generated_at: float
total_turns: int
metered_turns: int
token_metered_turns: int = 0
token_unmetered_turns: int = 0
tokens_in: int
tokens_out: int
cost_usd: float
recorded_cost_usd: float = 0.0
estimated_cost_usd: float = 0.0
budget: AdminUsageBudget
evaluator_cache: AdminUsageEvaluatorCache
by_provider: list[AdminUsageBreakdown]
@ -419,6 +456,70 @@ def _safe_usage_int(value: object) -> int:
return 0
def _usage_breakdown(
*,
provider: str,
model: str,
turns: int,
token_metered_turns: int,
token_unmetered_turns: int,
tokens_in: int,
tokens_out: int,
stored_cost_usd: float,
unpriced_tokens_in: int,
unpriced_tokens_out: int,
) -> AdminUsageBreakdown:
"""저장된 공급자 비용 추정치와 공식 참조단가를 한 원장 행으로 정규화한다."""
fallback = estimate_reference_cost(
provider=provider,
model=model,
tokens_in=unpriced_tokens_in,
tokens_out=unpriced_tokens_out,
)
rate_info = fallback or estimate_reference_cost(
provider=provider,
model=model,
tokens_in=tokens_in,
tokens_out=tokens_out,
)
fallback_cost = fallback.cost_usd if fallback is not None else 0.0
effective_cost = max(0.0, stored_cost_usd) + fallback_cost
reference_basis = provider_uses_reference_cost(provider)
if reference_basis:
recorded_cost = 0.0
estimated_cost = effective_cost
basis: UsageCostBasis = (
"reference_rate" if rate_info is not None or effective_cost > 0 else "unavailable"
)
else:
recorded_cost = max(0.0, stored_cost_usd)
estimated_cost = fallback_cost
if recorded_cost > 0:
basis = "provider_estimate" if provider == "claude_cli" else "provider_reported"
elif fallback is not None:
basis = "reference_rate"
else:
basis = "unavailable"
return AdminUsageBreakdown(
provider=provider,
model=model,
turns=max(0, turns),
token_metered_turns=max(0, token_metered_turns),
token_unmetered_turns=max(0, token_unmetered_turns),
tokens_in=max(0, tokens_in),
tokens_out=max(0, tokens_out),
cost_usd=round(effective_cost, 6),
recorded_cost_usd=round(recorded_cost, 6),
estimated_cost_usd=round(estimated_cost, 6),
cost_basis=basis,
rate_label=rate_info.rate_label if rate_info is not None else None,
rate_source_url=rate_info.source_url if rate_info is not None else None,
)
def _usage_budget(cost_usd: float) -> AdminUsageBudget:
limit = max(0.0, float(settings.admin_usage_budget_usd or 0.0))
if limit <= 0:
@ -527,6 +628,15 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
SELECT
COUNT(*) FILTER (WHERE speaker = 'client') AS total_turns,
COUNT(*) FILTER (WHERE {METERED_CLIENT_TURN_FILTER_SQL}) AS metered_turns,
COUNT(*) FILTER (
WHERE {METERED_CLIENT_TURN_FILTER_SQL}
AND (COALESCE(tokens_in, 0) > 0 OR COALESCE(tokens_out, 0) > 0)
) AS token_metered_turns,
COUNT(*) FILTER (
WHERE {METERED_CLIENT_TURN_FILTER_SQL}
AND COALESCE(tokens_in, 0) <= 0
AND COALESCE(tokens_out, 0) <= 0
) AS token_unmetered_turns,
COALESCE(SUM(tokens_in) FILTER (WHERE speaker = 'client'), 0)::bigint AS tokens_in,
COALESCE(SUM(tokens_out) FILTER (WHERE speaker = 'client'), 0)::bigint AS tokens_out,
COALESCE(SUM(cost_usd) FILTER (WHERE speaker = 'client'), 0)::numeric AS cost_usd
@ -545,11 +655,6 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
WHERE created_at >= now() - ($1::int * interval '1 day')
AND {METERED_CLIENT_TURN_FILTER_SQL}
GROUP BY 1, 2
ORDER BY
cost_usd DESC,
COALESCE(SUM(tokens_in), 0) + COALESCE(SUM(tokens_out), 0) DESC,
turns DESC
LIMIT 12
""",
window_days,
)
@ -557,17 +662,70 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
f"""
SELECT
to_char(date_trunc('day', created_at), 'YYYY-MM-DD') AS day,
COALESCE(llm_provider, 'unknown') AS provider,
COALESCE(model, 'unknown') AS model,
{USAGE_AGGREGATE_COLUMNS_SQL}
FROM app.turns
WHERE created_at >= now() - ($1::int * interval '1 day')
AND {METERED_CLIENT_TURN_FILTER_SQL}
GROUP BY 1
ORDER BY 1
GROUP BY 1, 2, 3
ORDER BY 1, 2, 3
""",
window_days,
)
total_cost = round(_decimal_to_float(total_row["cost_usd"] if total_row else 0), 6)
all_breakdowns = [
_usage_breakdown(
provider=str(row["provider"] or "unknown"),
model=str(row["model"] or "unknown"),
turns=_safe_usage_int(row["turns"]),
token_metered_turns=_safe_usage_int(row["token_metered_turns"]),
token_unmetered_turns=_safe_usage_int(row["token_unmetered_turns"]),
tokens_in=_safe_usage_int(row["tokens_in"]),
tokens_out=_safe_usage_int(row["tokens_out"]),
stored_cost_usd=_decimal_to_float(row["cost_usd"]),
unpriced_tokens_in=_safe_usage_int(row["unpriced_tokens_in"]),
unpriced_tokens_out=_safe_usage_int(row["unpriced_tokens_out"]),
)
for row in rows
]
all_breakdowns.sort(
key=lambda item: (
-item.cost_usd,
-(item.tokens_in + item.tokens_out),
-item.turns,
item.provider,
item.model,
)
)
recorded_cost = round(sum(item.recorded_cost_usd for item in all_breakdowns), 6)
estimated_cost = round(sum(item.estimated_cost_usd for item in all_breakdowns), 6)
total_cost = round(recorded_cost + estimated_cost, 6)
daily_buckets: dict[str, dict[str, int | float]] = {}
for row in daily_rows:
breakdown = _usage_breakdown(
provider=str(row["provider"] or "unknown"),
model=str(row["model"] or "unknown"),
turns=_safe_usage_int(row["turns"]),
token_metered_turns=_safe_usage_int(row["token_metered_turns"]),
token_unmetered_turns=_safe_usage_int(row["token_unmetered_turns"]),
tokens_in=_safe_usage_int(row["tokens_in"]),
tokens_out=_safe_usage_int(row["tokens_out"]),
stored_cost_usd=_decimal_to_float(row["cost_usd"]),
unpriced_tokens_in=_safe_usage_int(row["unpriced_tokens_in"]),
unpriced_tokens_out=_safe_usage_int(row["unpriced_tokens_out"]),
)
day = str(row["day"])
bucket = daily_buckets.setdefault(
day,
{"turns": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0},
)
bucket["turns"] = int(bucket["turns"]) + breakdown.turns
bucket["tokens_in"] = int(bucket["tokens_in"]) + breakdown.tokens_in
bucket["tokens_out"] = int(bucket["tokens_out"]) + breakdown.tokens_out
bucket["cost_usd"] = float(bucket["cost_usd"]) + breakdown.cost_usd
return AdminUsageResponse(
source="database",
durable=True,
@ -575,31 +733,29 @@ async def _usage_from_database(window_days: int) -> AdminUsageResponse:
generated_at=time.time(),
total_turns=_safe_usage_int(total_row["total_turns"] if total_row else 0),
metered_turns=_safe_usage_int(total_row["metered_turns"] if total_row else 0),
token_metered_turns=_safe_usage_int(
total_row["token_metered_turns"] if total_row else 0
),
token_unmetered_turns=_safe_usage_int(
total_row["token_unmetered_turns"] if total_row else 0
),
tokens_in=_safe_usage_int(total_row["tokens_in"] if total_row else 0),
tokens_out=_safe_usage_int(total_row["tokens_out"] if total_row else 0),
cost_usd=total_cost,
recorded_cost_usd=recorded_cost,
estimated_cost_usd=estimated_cost,
budget=_usage_budget(total_cost),
evaluator_cache=_usage_evaluator_cache(),
by_provider=[
AdminUsageBreakdown(
provider=str(row["provider"] or "unknown"),
model=str(row["model"] or "unknown"),
turns=_safe_usage_int(row["turns"]),
tokens_in=_safe_usage_int(row["tokens_in"]),
tokens_out=_safe_usage_int(row["tokens_out"]),
cost_usd=round(_decimal_to_float(row["cost_usd"]), 6),
)
for row in rows
],
by_provider=all_breakdowns[:12],
daily_cost=[
AdminUsageDailyCost(
day=str(row["day"]),
turns=_safe_usage_int(row["turns"]),
tokens_in=_safe_usage_int(row["tokens_in"]),
tokens_out=_safe_usage_int(row["tokens_out"]),
cost_usd=round(_decimal_to_float(row["cost_usd"]), 6),
day=day,
turns=int(values["turns"]),
tokens_in=int(values["tokens_in"]),
tokens_out=int(values["tokens_out"]),
cost_usd=round(float(values["cost_usd"]), 6),
)
for row in daily_rows
for day, values in sorted(daily_buckets.items())
],
)
@ -873,9 +1029,13 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
window_start = time.time() - (window_days * 86400)
total_turns = 0
metered_turns = 0
token_metered_turns = 0
token_unmetered_turns = 0
tokens_in = 0
tokens_out = 0
cost_usd = 0.0
recorded_cost_usd = 0.0
estimated_cost_usd = 0.0
buckets: dict[tuple[str, str], dict[str, int | float]] = {}
daily_buckets: dict[str, dict[str, int | float]] = {}
@ -902,18 +1062,57 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
if not is_metered:
continue
metered_turns += 1
is_token_metered = turn_tokens_in > 0 or turn_tokens_out > 0
if is_token_metered:
token_metered_turns += 1
else:
token_unmetered_turns += 1
tokens_in += turn_tokens_in
tokens_out += turn_tokens_out
cost_usd += turn_cost
turn_breakdown = _usage_breakdown(
provider=provider,
model=model,
turns=1,
token_metered_turns=1 if is_token_metered else 0,
token_unmetered_turns=0 if is_token_metered else 1,
tokens_in=turn_tokens_in,
tokens_out=turn_tokens_out,
stored_cost_usd=turn_cost,
unpriced_tokens_in=turn_tokens_in if turn_cost <= 0 else 0,
unpriced_tokens_out=turn_tokens_out if turn_cost <= 0 else 0,
)
cost_usd += turn_breakdown.cost_usd
recorded_cost_usd += turn_breakdown.recorded_cost_usd
estimated_cost_usd += turn_breakdown.estimated_cost_usd
key = (provider, model)
bucket = buckets.setdefault(
key,
{"turns": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0},
{
"turns": 0,
"token_metered_turns": 0,
"token_unmetered_turns": 0,
"tokens_in": 0,
"tokens_out": 0,
"stored_cost_usd": 0.0,
"unpriced_tokens_in": 0,
"unpriced_tokens_out": 0,
},
)
bucket["turns"] = int(bucket["turns"]) + 1
if is_token_metered:
bucket["token_metered_turns"] = int(bucket["token_metered_turns"]) + 1
else:
bucket["token_unmetered_turns"] = int(bucket["token_unmetered_turns"]) + 1
bucket["tokens_in"] = int(bucket["tokens_in"]) + turn_tokens_in
bucket["tokens_out"] = int(bucket["tokens_out"]) + turn_tokens_out
bucket["cost_usd"] = float(bucket["cost_usd"]) + turn_cost
bucket["stored_cost_usd"] = float(bucket["stored_cost_usd"]) + turn_cost
if turn_cost <= 0:
bucket["unpriced_tokens_in"] = (
int(bucket["unpriced_tokens_in"]) + turn_tokens_in
)
bucket["unpriced_tokens_out"] = (
int(bucket["unpriced_tokens_out"]) + turn_tokens_out
)
day = datetime.fromtimestamp(created_at, timezone.utc).strftime("%Y-%m-%d")
daily_bucket = daily_buckets.setdefault(
day,
@ -922,26 +1121,34 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
daily_bucket["turns"] = int(daily_bucket["turns"]) + 1
daily_bucket["tokens_in"] = int(daily_bucket["tokens_in"]) + turn_tokens_in
daily_bucket["tokens_out"] = int(daily_bucket["tokens_out"]) + turn_tokens_out
daily_bucket["cost_usd"] = float(daily_bucket["cost_usd"]) + turn_cost
daily_bucket["cost_usd"] = (
float(daily_bucket["cost_usd"]) + turn_breakdown.cost_usd
)
by_provider = [
AdminUsageBreakdown(
_usage_breakdown(
provider=provider,
model=model,
turns=int(values["turns"]),
token_metered_turns=int(values["token_metered_turns"]),
token_unmetered_turns=int(values["token_unmetered_turns"]),
tokens_in=int(values["tokens_in"]),
tokens_out=int(values["tokens_out"]),
cost_usd=round(float(values["cost_usd"]), 6),
stored_cost_usd=float(values["stored_cost_usd"]),
unpriced_tokens_in=int(values["unpriced_tokens_in"]),
unpriced_tokens_out=int(values["unpriced_tokens_out"]),
)
for (provider, model), values in sorted(
buckets.items(),
key=lambda item: (
-float(item[1]["cost_usd"]),
-(int(item[1]["tokens_in"]) + int(item[1]["tokens_out"])),
-int(item[1]["turns"]),
),
)[:12]
for (provider, model), values in buckets.items()
]
by_provider.sort(
key=lambda item: (
-item.cost_usd,
-(item.tokens_in + item.tokens_out),
-item.turns,
item.provider,
item.model,
)
)
total_cost = round(cost_usd, 6)
return AdminUsageResponse(
@ -951,12 +1158,16 @@ def _usage_from_runtime_store(window_days: int) -> AdminUsageResponse:
generated_at=time.time(),
total_turns=total_turns,
metered_turns=metered_turns,
token_metered_turns=token_metered_turns,
token_unmetered_turns=token_unmetered_turns,
tokens_in=tokens_in,
tokens_out=tokens_out,
cost_usd=total_cost,
recorded_cost_usd=round(recorded_cost_usd, 6),
estimated_cost_usd=round(estimated_cost_usd, 6),
budget=_usage_budget(total_cost),
evaluator_cache=_usage_evaluator_cache(),
by_provider=by_provider,
by_provider=by_provider[:12],
daily_cost=[
AdminUsageDailyCost(
day=day,
@ -1485,6 +1696,15 @@ async def admin_health(principal: AdminPrincipal) -> AdminHealthResponse:
return response
@router.get("/voice-runtime", response_model=VoiceRuntimeSnapshot)
async def admin_voice_runtime(
principal: AdminPrincipal,
) -> VoiceRuntimeSnapshot:
"""Return one API worker's metadata-only voice high-water snapshot."""
return voice_runtime_metrics.snapshot()
@router.get("/usage", response_model=AdminUsageResponse)
async def admin_usage(
principal: AdminPrincipal,