G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
|
|
@ -11,12 +11,14 @@ Vignette 엔진 게이트웨이 — 로컬 claude -p(Opus 4.8) 상주 멀티턴
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
|
@ -58,6 +60,28 @@ SESSION_IDLE_TTL_SECONDS = float(os.environ.get("ENGINE_SESSION_IDLE_TTL_SECONDS
|
|||
MAX_RESIDENT_SESSIONS = max(1, int(os.environ.get("ENGINE_MAX_RESIDENT_SESSIONS", "24")))
|
||||
GATEWAY_PROVIDER = "claude_cli"
|
||||
GATEWAY_FALLBACK_MODEL_NAME = "claude-opus-4-8"
|
||||
# claude -p 자식의 stderr 보존량. 인증 만료/플래그 오류처럼 CLI가 stdout 한 줄도 못 내고
|
||||
# 죽는 경우 원인은 stderr에만 남는다. 이걸 버리면 게이트웨이는 "empty engine response"만
|
||||
# 보고하고 운영자는 프로세스 환경을 재현해야 원인을 안다(2026-08-07 공개 런타임 사고).
|
||||
STDERR_TAIL_LINES = 8
|
||||
STDERR_TAIL_CHARS = 400
|
||||
ENGINE_TOKEN_HEADER = "X-Vignette-Engine-Token"
|
||||
_SECRET_PLACEHOLDER_PREFIXES = ("change-me", "replace-with", "dummy", "example")
|
||||
|
||||
|
||||
def _load_gateway_shared_secret() -> str:
|
||||
value = os.environ.get("ENGINE_GATEWAY_SHARED_SECRET", "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
if len(value) < 32 or value.lower().startswith(_SECRET_PLACEHOLDER_PREFIXES):
|
||||
raise RuntimeError(
|
||||
"ENGINE_GATEWAY_SHARED_SECRET must be a non-placeholder value "
|
||||
"containing at least 32 characters"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
ENGINE_GATEWAY_SHARED_SECRET = _load_gateway_shared_secret()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -83,6 +107,54 @@ BASE_ARGS = [
|
|||
"--exclude-dynamic-system-prompt-sections",
|
||||
]
|
||||
|
||||
|
||||
def _usage_int(payload: dict[str, Any], *keys: str) -> int:
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
try:
|
||||
return max(0, int(value))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return 0
|
||||
|
||||
|
||||
def _claude_result_tokens(result: dict[str, Any]) -> tuple[int, int]:
|
||||
"""Claude CLI result의 전체 agent-tree 토큰을 기존 원장 두 컬럼으로 정규화한다.
|
||||
|
||||
modelUsage/model_usage가 있으면 서브에이전트를 포함한 모델별 사용량을 합산한다.
|
||||
입력 토큰은 공식 계약에 맞춰 비캐시 입력 + cache read + cache creation이다.
|
||||
"""
|
||||
|
||||
raw_model_usage = result.get("modelUsage") or result.get("model_usage")
|
||||
usage_rows = (
|
||||
[row for row in raw_model_usage.values() if isinstance(row, dict)]
|
||||
if isinstance(raw_model_usage, dict)
|
||||
else []
|
||||
)
|
||||
if not usage_rows:
|
||||
usage = result.get("usage")
|
||||
usage_rows = [usage] if isinstance(usage, dict) else []
|
||||
|
||||
tokens_in = 0
|
||||
tokens_out = 0
|
||||
for usage in usage_rows:
|
||||
tokens_in += _usage_int(usage, "inputTokens", "input_tokens")
|
||||
tokens_in += _usage_int(
|
||||
usage,
|
||||
"cacheReadInputTokens",
|
||||
"cache_read_input_tokens",
|
||||
)
|
||||
tokens_in += _usage_int(
|
||||
usage,
|
||||
"cacheCreationInputTokens",
|
||||
"cache_creation_input_tokens",
|
||||
)
|
||||
tokens_out += _usage_int(usage, "outputTokens", "output_tokens")
|
||||
return tokens_in, tokens_out
|
||||
|
||||
|
||||
class EngineSession:
|
||||
"""claude -p 상주 프로세스 1개 = 상담 회기 1개."""
|
||||
|
||||
|
|
@ -103,6 +175,55 @@ class EngineSession:
|
|||
self.cost_usd = 0.0
|
||||
self.turns = 0
|
||||
self.last_used_at = time.monotonic()
|
||||
# stderr는 계속 비워야 한다. PIPE를 열고 아무도 읽지 않으면 자식이 파이프 버퍼가
|
||||
# 찬 시점에 블록된다. 겸사겸사 마지막 몇 줄을 진단용으로 남긴다.
|
||||
self._stderr_tail: deque[str] = deque(maxlen=STDERR_TAIL_LINES)
|
||||
self._stderr_task: asyncio.Task | None = None
|
||||
|
||||
async def _drain_stderr(self) -> None:
|
||||
proc = self.proc
|
||||
if proc is None or proc.stderr is None:
|
||||
return
|
||||
try:
|
||||
while True:
|
||||
line = await proc.stderr.readline()
|
||||
if not line:
|
||||
return
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
if text:
|
||||
self._stderr_tail.append(text)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
return
|
||||
|
||||
def stderr_tail(self) -> str:
|
||||
"""자식 CLI가 마지막으로 남긴 stderr 요약(진단 표면용)."""
|
||||
joined = " | ".join(self._stderr_tail)
|
||||
if len(joined) > STDERR_TAIL_CHARS:
|
||||
return "…" + joined[-STDERR_TAIL_CHARS:]
|
||||
return joined
|
||||
|
||||
async def failure_detail(self, fallback: str) -> str:
|
||||
"""텍스트를 못 받았을 때 원인을 최대한 좁힌 detail을 만든다."""
|
||||
task = self._stderr_task
|
||||
if task is not None and not task.done():
|
||||
# 자식이 즉시 죽은 경우 stdout EOF가 stderr 드레인보다 먼저 도착할 수 있다.
|
||||
# shield로 감싸 세션이 계속 살아있는 경우의 드레인을 취소하지 않는다.
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=0.5)
|
||||
except Exception:
|
||||
pass
|
||||
parts: list[str] = []
|
||||
code = self.proc.returncode if self.proc is not None else None
|
||||
if code is not None:
|
||||
parts.append(f"exit={code}")
|
||||
tail = self.stderr_tail()
|
||||
if tail:
|
||||
parts.append(tail)
|
||||
if not parts:
|
||||
return fallback
|
||||
return f"{fallback} ({' · '.join(parts)})"
|
||||
|
||||
async def start(self) -> None:
|
||||
args = [CLAUDE_BIN, *BASE_ARGS, "--max-budget-usd", str(self.budget)]
|
||||
|
|
@ -123,6 +244,7 @@ class EngineSession:
|
|||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self._stderr_task = asyncio.create_task(self._drain_stderr())
|
||||
|
||||
async def turn(self, content: str, timeout: float = 120.0) -> dict:
|
||||
if self.proc is None or self.proc.returncode is not None:
|
||||
|
|
@ -158,17 +280,28 @@ class EngineSession:
|
|||
result = await asyncio.wait_for(_read_until_result(), timeout=timeout)
|
||||
self.last_used_at = time.monotonic()
|
||||
self.cost_usd = result.get("total_cost_usd", self.cost_usd)
|
||||
tokens_in, tokens_out = _claude_result_tokens(result)
|
||||
self.turns += 1
|
||||
assistant_text = "".join(text_parts)
|
||||
terminal_result = result.get("result")
|
||||
terminal_text = terminal_result if isinstance(terminal_result, str) else ""
|
||||
# Claude CLI stream-json can emit the complete successful answer only
|
||||
# on the terminal result event. Prefer assistant events when present
|
||||
# (they preserve streaming semantics), but never discard a terminal-only
|
||||
# structured response.
|
||||
response_text = assistant_text or terminal_text
|
||||
error_detail = (
|
||||
result.get("error")
|
||||
or result.get("result")
|
||||
or (result.get("errors") or [None])[0]
|
||||
or "".join(text_parts)
|
||||
or response_text
|
||||
or None
|
||||
)
|
||||
return {
|
||||
"text": "".join(text_parts),
|
||||
"text": response_text,
|
||||
"cost_usd": self.cost_usd,
|
||||
"tokens_in": tokens_in,
|
||||
"tokens_out": tokens_out,
|
||||
"turns": self.turns,
|
||||
"is_error": result.get("is_error", False),
|
||||
"error": error_detail,
|
||||
|
|
@ -237,7 +370,15 @@ class EngineSession:
|
|||
elif t == "result":
|
||||
self.last_used_at = time.monotonic()
|
||||
self.cost_usd = obj.get("total_cost_usd", self.cost_usd)
|
||||
tokens_in, tokens_out = _claude_result_tokens(obj)
|
||||
self.turns += 1
|
||||
terminal_result = obj.get("result")
|
||||
terminal_text = (
|
||||
terminal_result if isinstance(terminal_result, str) else ""
|
||||
)
|
||||
if not emitted and terminal_text:
|
||||
emitted = terminal_text
|
||||
yield {"type": "delta", "text": terminal_text}
|
||||
error_detail = (
|
||||
obj.get("error")
|
||||
or obj.get("result")
|
||||
|
|
@ -249,6 +390,8 @@ class EngineSession:
|
|||
"type": "done",
|
||||
"text": emitted,
|
||||
"cost_usd": self.cost_usd,
|
||||
"tokens_in": tokens_in,
|
||||
"tokens_out": tokens_out,
|
||||
"turns": self.turns,
|
||||
"is_error": obj.get("is_error", False),
|
||||
"error": error_detail,
|
||||
|
|
@ -265,6 +408,14 @@ class EngineSession:
|
|||
await asyncio.wait_for(self.proc.wait(), timeout=5)
|
||||
except Exception:
|
||||
self.proc.kill()
|
||||
task = self._stderr_task
|
||||
self._stderr_task = None
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
|
||||
SESSIONS: dict[str, EngineSession] = {}
|
||||
|
|
@ -272,6 +423,21 @@ _SESSION_RESOLVE_LOCK = asyncio.Lock()
|
|||
_READY_CACHE: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
_READY_LOCK = asyncio.Lock()
|
||||
app = FastAPI(title="Vignette Engine Gateway")
|
||||
_AUTH_EXEMPT_PATHS = frozenset({"/health"})
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def require_gateway_shared_secret(request: Request, call_next):
|
||||
"""설정된 인스턴스는 순수 liveness 외 모든 HTTP 경로를 fail-closed로 막는다."""
|
||||
expected = ENGINE_GATEWAY_SHARED_SECRET
|
||||
if expected and request.url.path not in _AUTH_EXEMPT_PATHS:
|
||||
provided = request.headers.get(ENGINE_TOKEN_HEADER, "")
|
||||
if not secrets.compare_digest(provided, expected):
|
||||
return JSONResponse(
|
||||
{"detail": "invalid engine gateway credentials"},
|
||||
status_code=401,
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
class CreateReq(BaseModel):
|
||||
|
|
@ -379,13 +545,19 @@ async def ready(
|
|||
"Reply with exactly OK.", timeout=READY_TIMEOUT_SECONDS
|
||||
)
|
||||
if result.get("is_error"):
|
||||
detail = str(result.get("error") or "engine returned an error")
|
||||
detail = await probe.failure_detail(
|
||||
str(result.get("error") or "engine returned an error")
|
||||
)
|
||||
else:
|
||||
text = str(result.get("text") or "").strip()
|
||||
ok = bool(text)
|
||||
detail = text or "empty engine response"
|
||||
detail = text or await probe.failure_detail(
|
||||
"empty engine response"
|
||||
)
|
||||
except Exception as exc:
|
||||
detail = str(exc)
|
||||
# TimeoutError 등 str()이 비는 예외가 있어 타입명을 최소 보장한다.
|
||||
reason = str(exc).strip() or type(exc).__name__
|
||||
detail = await probe.failure_detail(reason)
|
||||
finally:
|
||||
await probe.close()
|
||||
else:
|
||||
|
|
@ -667,8 +839,8 @@ async def v1_generate(req: GwGenerateReq):
|
|||
text=text,
|
||||
model=_response_model_name(s),
|
||||
provider=GATEWAY_PROVIDER,
|
||||
tokens_in=0,
|
||||
tokens_out=0,
|
||||
tokens_in=result.get("tokens_in", 0),
|
||||
tokens_out=result.get("tokens_out", 0),
|
||||
cost_usd=result.get("cost_usd", 0.0),
|
||||
inference_geo="us",
|
||||
structured=structured,
|
||||
|
|
@ -747,8 +919,8 @@ async def v1_stream(req: GwGenerateReq):
|
|||
StreamDoneEvent(
|
||||
provider=GATEWAY_PROVIDER,
|
||||
model=_response_model_name(s),
|
||||
tokens_in=0,
|
||||
tokens_out=0,
|
||||
tokens_in=evt.get("tokens_in", 0),
|
||||
tokens_out=evt.get("tokens_out", 0),
|
||||
cost_usd=evt.get("cost_usd", 0.0),
|
||||
turns=evt.get("turns", 0),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ from typing import Any, AsyncIterator, Iterable, Literal, cast
|
|||
import httpx
|
||||
|
||||
from app.contracts.engine_gateway import (
|
||||
ENGINE_GATEWAY_DEFAULT_MODEL_SENTINEL,
|
||||
ENGINE_PROVIDER_DEFAULTS,
|
||||
ENGINE_REASONING_EFFORTS,
|
||||
EngineCapabilitiesResponse,
|
||||
|
|
@ -29,6 +28,7 @@ from app.contracts.engine_gateway import (
|
|||
ReasoningEffort,
|
||||
normalize_engine_gateway_model,
|
||||
)
|
||||
from app.services.llm_pricing import estimate_reference_cost
|
||||
|
||||
CODEX_DEFAULT_MODEL, CODEX_DEFAULT_EFFORT = ENGINE_PROVIDER_DEFAULTS["codex_cli"]
|
||||
AGY_DEFAULT_MODEL, AGY_DEFAULT_EFFORT = ENGINE_PROVIDER_DEFAULTS["agy_cli"]
|
||||
|
|
@ -580,6 +580,7 @@ async def _generate_codex(
|
|||
text = ""
|
||||
tokens_in = 0
|
||||
tokens_out = 0
|
||||
cached_input_tokens = 0
|
||||
for line in stdout.splitlines():
|
||||
try:
|
||||
event = json.loads(line)
|
||||
|
|
@ -593,16 +594,25 @@ async def _generate_codex(
|
|||
usage = event.get("usage") or {}
|
||||
tokens_in = int(usage.get("input_tokens") or 0)
|
||||
tokens_out = int(usage.get("output_tokens") or 0)
|
||||
cached_input_tokens = int(usage.get("cached_input_tokens") or 0)
|
||||
elif event.get("type") in {"turn.failed", "error"}:
|
||||
raise ProviderError(str(event.get("message") or event))
|
||||
if not text.strip():
|
||||
raise ProviderError("Codex CLI가 최종 응답을 반환하지 않았습니다.")
|
||||
estimate = estimate_reference_cost(
|
||||
provider="codex_cli",
|
||||
model=model,
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cached_input_tokens=cached_input_tokens,
|
||||
)
|
||||
return ProviderGenerateResult(
|
||||
text=text,
|
||||
model=model,
|
||||
provider="codex_cli",
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cost_usd=estimate.cost_usd if estimate is not None else 0.0,
|
||||
structured=_structured_or_none(text, req),
|
||||
)
|
||||
|
||||
|
|
@ -610,32 +620,12 @@ async def _generate_codex(
|
|||
async def _generate_agy(
|
||||
req: GenerateRequest, system_prompt: str, user_payload: str
|
||||
) -> ProviderGenerateResult:
|
||||
binary = _binary("AGY_BIN", "agy")
|
||||
if binary is None:
|
||||
raise ProviderError("Agy CLI를 찾을 수 없습니다.")
|
||||
model, effort = await _resolve_selection(req, "agy_cli")
|
||||
prompt = _cli_prompt(system_prompt, user_payload)
|
||||
if os.name == "nt" and len(prompt) > 24_000:
|
||||
raise ProviderError(
|
||||
"Agy CLI 프롬프트가 Windows 명령줄 안전 한도(24,000자)를 초과했습니다."
|
||||
)
|
||||
args = [binary, "--model", model, "--sandbox"]
|
||||
if effort:
|
||||
args += ["--effort", effort]
|
||||
args += ["--print-timeout", f"{int(CLI_TIMEOUT_SECONDS)}s"]
|
||||
# Agy의 --print는 바로 뒤 토큰을 프롬프트로 해석하며 stdin 입력은
|
||||
# 지원하지 않는다. 옵션을 모두 앞에 두고 프롬프트를 마지막에 둔다.
|
||||
args += ["--print", prompt]
|
||||
stdout, _ = await _run_process(args, cwd=str(_cli_runtime_cwd()))
|
||||
text = stdout.strip()
|
||||
if not text:
|
||||
raise ProviderError("Agy CLI가 최종 응답을 반환하지 않았습니다.")
|
||||
return ProviderGenerateResult(
|
||||
text=text,
|
||||
model=model,
|
||||
provider="agy_cli",
|
||||
structured=_structured_or_none(text, req),
|
||||
)
|
||||
# text 출력은 토큰 사용량을 주지 않는다. stream-json의 terminal result를
|
||||
# 동일하게 소비해 generate와 stream 모두 같은 token/cost 계약을 유지한다.
|
||||
async for event in _stream_agy(req, system_prompt, user_payload):
|
||||
if event.type == "done" and event.result is not None:
|
||||
return event.result
|
||||
raise ProviderError("Agy CLI가 최종 응답을 반환하지 않았습니다.")
|
||||
|
||||
|
||||
async def _stream_agy(
|
||||
|
|
@ -680,6 +670,7 @@ async def _stream_agy(
|
|||
final_text = ""
|
||||
tokens_in = 0
|
||||
tokens_out = 0
|
||||
cached_input_tokens = 0
|
||||
result_status = ""
|
||||
try:
|
||||
async with asyncio.timeout(CLI_TIMEOUT_SECONDS):
|
||||
|
|
@ -705,6 +696,11 @@ async def _stream_agy(
|
|||
usage = result.get("usage") or {}
|
||||
tokens_in = int(usage.get("input_tokens") or 0)
|
||||
tokens_out = int(usage.get("output_tokens") or 0)
|
||||
cached_input_tokens = int(
|
||||
usage.get("cache_read_tokens")
|
||||
or usage.get("cached_input_tokens")
|
||||
or 0
|
||||
)
|
||||
returncode = await proc.wait()
|
||||
except TimeoutError as exc:
|
||||
raise ProviderError(
|
||||
|
|
@ -731,6 +727,13 @@ async def _stream_agy(
|
|||
elif not emitted:
|
||||
emitted = resolved_text
|
||||
yield ProviderStreamEvent(type="delta", text=resolved_text)
|
||||
estimate = estimate_reference_cost(
|
||||
provider="agy_cli",
|
||||
model=model,
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cached_input_tokens=cached_input_tokens,
|
||||
)
|
||||
yield ProviderStreamEvent(
|
||||
type="done",
|
||||
result=ProviderGenerateResult(
|
||||
|
|
@ -739,6 +742,7 @@ async def _stream_agy(
|
|||
provider="agy_cli",
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cost_usd=estimate.cost_usd if estimate is not None else 0.0,
|
||||
structured=_structured_or_none(resolved_text, req),
|
||||
),
|
||||
)
|
||||
|
|
@ -789,12 +793,22 @@ async def _generate_claude_api(
|
|||
raise ProviderError("Anthropic Messages API가 텍스트 응답을 반환하지 않았습니다.")
|
||||
usage = body.get("usage") or {}
|
||||
inference_geo = body.get("inference_geo")
|
||||
tokens_in = int(usage.get("input_tokens") or 0)
|
||||
tokens_out = int(usage.get("output_tokens") or 0)
|
||||
estimate = estimate_reference_cost(
|
||||
provider="claude_api",
|
||||
model=str(body.get("model") or model),
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cached_input_tokens=int(usage.get("cache_read_input_tokens") or 0),
|
||||
)
|
||||
return ProviderGenerateResult(
|
||||
text=text,
|
||||
model=str(body.get("model") or model),
|
||||
provider="claude_api",
|
||||
tokens_in=int(usage.get("input_tokens") or 0),
|
||||
tokens_out=int(usage.get("output_tokens") or 0),
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
cost_usd=estimate.cost_usd if estimate is not None else 0.0,
|
||||
inference_geo=str(inference_geo) if inference_geo else None,
|
||||
structured=_structured_or_none(text, req),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
fastapi\nuvicorn[standard]
|
||||
fastapi==0.111.0
|
||||
uvicorn[standard]==0.30.6
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
|
|
@ -8,6 +9,7 @@ from types import SimpleNamespace
|
|||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import engine_client
|
||||
from app.contracts import engine_gateway as contract
|
||||
|
|
@ -195,6 +197,106 @@ class _FakeStreamSession:
|
|||
self.closed = True
|
||||
|
||||
|
||||
class GatewayAuthenticationTest(unittest.TestCase):
|
||||
SECRET = "engine-gateway-test-secret-" + ("x" * 32)
|
||||
|
||||
def test_unset_secret_preserves_local_gateway_compatibility(self):
|
||||
with patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", ""):
|
||||
response = TestClient(gateway.app).delete("/session/not-running")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_configured_secret_rejects_missing_and_wrong_credentials(self):
|
||||
with patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", self.SECRET):
|
||||
client = TestClient(gateway.app)
|
||||
missing = client.post("/v1/generate", json={})
|
||||
wrong = client.post(
|
||||
"/v1/generate",
|
||||
json={},
|
||||
headers={gateway.ENGINE_TOKEN_HEADER: "wrong"},
|
||||
)
|
||||
authenticated = client.post(
|
||||
"/v1/generate",
|
||||
json={},
|
||||
headers={gateway.ENGINE_TOKEN_HEADER: self.SECRET},
|
||||
)
|
||||
|
||||
self.assertEqual(missing.status_code, 401)
|
||||
self.assertEqual(wrong.status_code, 401)
|
||||
self.assertEqual(authenticated.status_code, 422)
|
||||
|
||||
def test_health_probe_remains_unauthenticated(self):
|
||||
with patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", self.SECRET):
|
||||
response = TestClient(gateway.app).get("/health")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_ready_probe_requires_credentials_because_it_runs_generation(self):
|
||||
with patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", self.SECRET):
|
||||
response = TestClient(gateway.app).get("/ready")
|
||||
|
||||
self.assertEqual(response.status_code, 401)
|
||||
|
||||
def test_gateway_rejects_weak_configured_secret_at_startup(self):
|
||||
for value in ("too-short", "example-gateway-secret-with-32-characters"):
|
||||
with (
|
||||
self.subTest(value=value),
|
||||
patch.dict(
|
||||
gateway.os.environ,
|
||||
{"ENGINE_GATEWAY_SHARED_SECRET": value},
|
||||
),
|
||||
self.assertRaises(RuntimeError),
|
||||
):
|
||||
gateway._load_gateway_shared_secret()
|
||||
|
||||
def test_invalid_token_uses_constant_time_comparison(self):
|
||||
with (
|
||||
patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", self.SECRET),
|
||||
patch.object(
|
||||
gateway.secrets,
|
||||
"compare_digest",
|
||||
wraps=secrets.compare_digest,
|
||||
) as compare_digest,
|
||||
):
|
||||
response = TestClient(gateway.app).get(
|
||||
"/v1/capabilities",
|
||||
headers={gateway.ENGINE_TOKEN_HEADER: "wrong"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 401)
|
||||
compare_digest.assert_called_once_with("wrong", self.SECRET)
|
||||
|
||||
def test_openapi_schema_does_not_expose_secret_or_auth_header(self):
|
||||
with patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", self.SECRET):
|
||||
schema = json.dumps(gateway.app.openapi())
|
||||
|
||||
self.assertNotIn(self.SECRET, schema)
|
||||
self.assertNotIn(gateway.ENGINE_TOKEN_HEADER, schema)
|
||||
|
||||
def test_engine_client_adds_gateway_token_to_default_headers(self):
|
||||
with patch.object(engine_client.httpx, "AsyncClient") as async_client_cls:
|
||||
client = engine_client.EngineClient(
|
||||
"http://127.0.0.1:9099",
|
||||
shared_secret=self.SECRET,
|
||||
)
|
||||
client._new_client()
|
||||
|
||||
self.assertEqual(
|
||||
async_client_cls.call_args.kwargs["headers"],
|
||||
{gateway.ENGINE_TOKEN_HEADER: self.SECRET},
|
||||
)
|
||||
|
||||
def test_engine_client_omits_gateway_token_when_unset(self):
|
||||
with patch.object(engine_client.httpx, "AsyncClient") as async_client_cls:
|
||||
client = engine_client.EngineClient(
|
||||
"http://127.0.0.1:9099",
|
||||
shared_secret="",
|
||||
)
|
||||
client._new_client()
|
||||
|
||||
self.assertEqual(async_client_cls.call_args.kwargs["headers"], {})
|
||||
|
||||
|
||||
class GatewayModelTest(unittest.TestCase):
|
||||
def test_contract_owns_gateway_default_model_sentinel(self):
|
||||
self.assertEqual(contract.ENGINE_GATEWAY_DEFAULT_MODEL_SENTINEL, "gateway-default")
|
||||
|
|
@ -456,6 +558,35 @@ class GatewayModelTest(unittest.TestCase):
|
|||
{"reply": "embedded"},
|
||||
)
|
||||
|
||||
def test_generate_response_structured_payload_repairs_only_trailing_commas(self):
|
||||
response = contract.GenerateResponse(
|
||||
text=(
|
||||
'```json\n'
|
||||
'{"reply":"keep literal , } and escaped \\\" text",'
|
||||
'"items":[{"value":1,},],}\n'
|
||||
'```'
|
||||
),
|
||||
provider="test",
|
||||
model="test-model",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
contract.structured_payload_from_response(response),
|
||||
{
|
||||
"reply": 'keep literal , } and escaped " text',
|
||||
"items": [{"value": 1}],
|
||||
},
|
||||
)
|
||||
|
||||
def test_generate_response_structured_payload_does_not_repair_other_corruption(self):
|
||||
response = contract.GenerateResponse(
|
||||
text='{"reply":"missing separator" "items":[]}',
|
||||
provider="test",
|
||||
model="test-model",
|
||||
)
|
||||
|
||||
self.assertIsNone(contract.structured_payload_from_response(response))
|
||||
|
||||
def test_generate_response_structured_payload_rejects_non_object_json(self):
|
||||
response = contract.GenerateResponse(
|
||||
text='["not", "object"]',
|
||||
|
|
@ -610,7 +741,25 @@ class GatewayModelTest(unittest.TestCase):
|
|||
"type": "assistant",
|
||||
"message": {"content": [{"type": "text", "text": "안녕!"}]},
|
||||
},
|
||||
{"type": "result", "is_error": False, "total_cost_usd": 0.01},
|
||||
{
|
||||
"type": "result",
|
||||
"is_error": False,
|
||||
"total_cost_usd": 0.01,
|
||||
"modelUsage": {
|
||||
"claude-opus-4-8": {
|
||||
"inputTokens": 12,
|
||||
"outputTokens": 7,
|
||||
"cacheReadInputTokens": 101,
|
||||
"cacheCreationInputTokens": 23,
|
||||
},
|
||||
"claude-haiku-4-5": {
|
||||
"inputTokens": 3,
|
||||
"outputTokens": 2,
|
||||
"cacheReadInputTokens": 9,
|
||||
"cacheCreationInputTokens": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
session = gateway.EngineSession()
|
||||
|
|
@ -630,6 +779,8 @@ class GatewayModelTest(unittest.TestCase):
|
|||
"type": "done",
|
||||
"text": "안녕!",
|
||||
"cost_usd": 0.01,
|
||||
"tokens_in": 148,
|
||||
"tokens_out": 9,
|
||||
"turns": 1,
|
||||
"is_error": False,
|
||||
"error": "안녕!",
|
||||
|
|
@ -637,6 +788,66 @@ class GatewayModelTest(unittest.TestCase):
|
|||
],
|
||||
)
|
||||
|
||||
def test_engine_session_uses_terminal_result_when_assistant_event_is_absent(self):
|
||||
process = _FakeProcess()
|
||||
process.stdin = _StreamStdin()
|
||||
process.stdout = _StreamStdout(
|
||||
[
|
||||
{
|
||||
"type": "result",
|
||||
"is_error": False,
|
||||
"result": '{"goal":{"score":0.2}}',
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
]
|
||||
)
|
||||
session = gateway.EngineSession()
|
||||
session.proc = process
|
||||
|
||||
result = asyncio.run(session.turn("평가"))
|
||||
|
||||
self.assertEqual(result["text"], '{"goal":{"score":0.2}}')
|
||||
self.assertFalse(result["is_error"])
|
||||
|
||||
def test_engine_session_streams_terminal_result_when_assistant_event_is_absent(self):
|
||||
process = _FakeProcess()
|
||||
process.stdin = _StreamStdin()
|
||||
process.stdout = _StreamStdout(
|
||||
[
|
||||
{
|
||||
"type": "result",
|
||||
"is_error": False,
|
||||
"result": "terminal-only",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 2},
|
||||
}
|
||||
]
|
||||
)
|
||||
session = gateway.EngineSession()
|
||||
session.proc = process
|
||||
|
||||
async def collect():
|
||||
return [event async for event in session.turn_stream("질문")]
|
||||
|
||||
events = asyncio.run(collect())
|
||||
|
||||
self.assertEqual(events[0], {"type": "delta", "text": "terminal-only"})
|
||||
self.assertEqual(events[-1]["text"], "terminal-only")
|
||||
|
||||
def test_claude_result_tokens_falls_back_to_top_level_usage(self):
|
||||
self.assertEqual(
|
||||
gateway._claude_result_tokens(
|
||||
{
|
||||
"usage": {
|
||||
"input_tokens": 17,
|
||||
"output_tokens": 5,
|
||||
"cache_read_input_tokens": 200,
|
||||
"cache_creation_input_tokens": 30,
|
||||
}
|
||||
}
|
||||
),
|
||||
(247, 5),
|
||||
)
|
||||
|
||||
def test_v1_generate_routes_non_claude_provider_through_registry(self):
|
||||
result = SimpleNamespace(
|
||||
text="registry response",
|
||||
|
|
@ -770,7 +981,13 @@ class GatewayModelTest(unittest.TestCase):
|
|||
|
||||
async def fake_turn(content, timeout=120.0):
|
||||
calls.append((content, timeout))
|
||||
return {"text": "reused response", "cost_usd": 0.01, "is_error": False}
|
||||
return {
|
||||
"text": "reused response",
|
||||
"cost_usd": 0.01,
|
||||
"tokens_in": 321,
|
||||
"tokens_out": 12,
|
||||
"is_error": False,
|
||||
}
|
||||
|
||||
async def fake_close():
|
||||
closes.append(True)
|
||||
|
|
@ -784,6 +1001,8 @@ class GatewayModelTest(unittest.TestCase):
|
|||
self.assertEqual(validated.text, "reused response")
|
||||
self.assertEqual(validated.provider, "claude_cli")
|
||||
self.assertEqual(validated.cost_usd, 0.01)
|
||||
self.assertEqual(validated.tokens_in, 321)
|
||||
self.assertEqual(validated.tokens_out, 12)
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[("[이번 상담자 발화]\nhello", gateway.GENERATE_TURN_TIMEOUT_SECONDS)],
|
||||
|
|
@ -801,7 +1020,13 @@ class GatewayModelTest(unittest.TestCase):
|
|||
|
||||
async def fake_turn(self, content, timeout=120.0):
|
||||
turned.append((self, content, timeout))
|
||||
return {"text": "fresh response", "cost_usd": 0.02, "is_error": False}
|
||||
return {
|
||||
"text": "fresh response",
|
||||
"cost_usd": 0.02,
|
||||
"tokens_in": 654,
|
||||
"tokens_out": 21,
|
||||
"is_error": False,
|
||||
}
|
||||
|
||||
async def fake_close(self):
|
||||
closed.append(self)
|
||||
|
|
@ -817,6 +1042,8 @@ class GatewayModelTest(unittest.TestCase):
|
|||
self.assertEqual(validated.text, "fresh response")
|
||||
self.assertEqual(validated.provider, "claude_cli")
|
||||
self.assertEqual(validated.cost_usd, 0.02)
|
||||
self.assertEqual(validated.tokens_in, 654)
|
||||
self.assertEqual(validated.tokens_out, 21)
|
||||
self.assertEqual(len(started), 1)
|
||||
self.assertEqual(
|
||||
turned,
|
||||
|
|
@ -850,7 +1077,13 @@ class GatewayModelTest(unittest.TestCase):
|
|||
session = _FakeStreamSession(
|
||||
[
|
||||
{"type": "delta", "text": "안녕"},
|
||||
{"type": "done", "cost_usd": 0.03, "turns": 2},
|
||||
{
|
||||
"type": "done",
|
||||
"cost_usd": 0.03,
|
||||
"tokens_in": 456,
|
||||
"tokens_out": 18,
|
||||
"turns": 2,
|
||||
},
|
||||
],
|
||||
model="stream-model",
|
||||
)
|
||||
|
|
@ -868,6 +1101,8 @@ class GatewayModelTest(unittest.TestCase):
|
|||
self.assertIn('"provider": "claude_cli"', body)
|
||||
self.assertIn('"model": "stream-model"', body)
|
||||
self.assertIn('"cost_usd": 0.03', body)
|
||||
self.assertIn('"tokens_in": 456', body)
|
||||
self.assertIn('"tokens_out": 18', body)
|
||||
self.assertEqual(session.content, "[이번 상담자 발화]\nhello")
|
||||
self.assertEqual(session.timeout, 600.0)
|
||||
self.assertTrue(session.closed)
|
||||
|
|
|
|||
|
|
@ -166,7 +166,11 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
json.dumps(
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 12, "output_tokens": 3},
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"cached_input_tokens": 2,
|
||||
"output_tokens": 3,
|
||||
},
|
||||
}
|
||||
),
|
||||
]
|
||||
|
|
@ -196,13 +200,14 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertEqual(result.text, "OK")
|
||||
self.assertEqual(result.tokens_in, 12)
|
||||
self.assertEqual(result.tokens_out, 3)
|
||||
self.assertEqual(result.cost_usd, 0.0000705)
|
||||
args = runner.await_args.args[0]
|
||||
self.assertIn("gpt-5.6-terra", args)
|
||||
self.assertIn('model_reasoning_effort="medium"', args)
|
||||
self.assertEqual(args[-1], "-")
|
||||
self.assertIn("[시스템 지침]", runner.await_args.kwargs["input_text"])
|
||||
|
||||
async def test_agy_generation_passes_prompt_immediately_after_print_flag(self):
|
||||
async def test_agy_generation_uses_stream_json_usage_and_reference_cost(self):
|
||||
capabilities = provider_registry.EngineCapabilitiesResponse(
|
||||
provider="agy_cli",
|
||||
available=True,
|
||||
|
|
@ -225,7 +230,35 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
reasoning_effort="high",
|
||||
messages=[EngineMessage(role="user", content="hello")],
|
||||
)
|
||||
runner = AsyncMock(return_value=("OK\n", ""))
|
||||
process = _FakeAgyProcess(
|
||||
[
|
||||
{
|
||||
"event": "step_update",
|
||||
"step_update": {
|
||||
"step_type": "agent_response",
|
||||
"state": "DONE",
|
||||
"text_delta": "OK",
|
||||
},
|
||||
},
|
||||
{
|
||||
"event": "result",
|
||||
"result": {
|
||||
"status": "SUCCESS",
|
||||
"response": "OK",
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"cache_read_tokens": 2,
|
||||
"output_tokens": 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
captured: list[tuple] = []
|
||||
|
||||
async def fake_create_subprocess_exec(*args, **kwargs):
|
||||
captured.append(args)
|
||||
return process
|
||||
|
||||
with (
|
||||
patch.object(provider_registry, "_binary", return_value="agy"),
|
||||
|
|
@ -234,7 +267,11 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
"discover_capabilities",
|
||||
AsyncMock(return_value=capabilities),
|
||||
),
|
||||
patch.object(provider_registry, "_run_process", runner),
|
||||
patch.object(
|
||||
provider_registry.asyncio,
|
||||
"create_subprocess_exec",
|
||||
fake_create_subprocess_exec,
|
||||
),
|
||||
):
|
||||
result = await provider_registry.generate_with_provider(
|
||||
request,
|
||||
|
|
@ -243,11 +280,14 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
)
|
||||
|
||||
self.assertEqual(result.text, "OK")
|
||||
args = runner.await_args.args[0]
|
||||
self.assertEqual(result.tokens_in, 12)
|
||||
self.assertEqual(result.tokens_out, 2)
|
||||
self.assertEqual(result.cost_usd, 0.0000303)
|
||||
args = captured[0]
|
||||
print_index = args.index("--print")
|
||||
self.assertEqual(print_index, len(args) - 2)
|
||||
self.assertIn("[시스템 지침]", args[-1])
|
||||
self.assertNotIn("input_text", runner.await_args.kwargs)
|
||||
self.assertEqual(args[args.index("--output-format") + 1], "stream-json")
|
||||
|
||||
async def test_agy_stream_forwards_live_deltas_without_repeating_final_response(self):
|
||||
capabilities = provider_registry.EngineCapabilitiesResponse(
|
||||
|
|
@ -296,7 +336,11 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
"result": {
|
||||
"status": "SUCCESS",
|
||||
"response": "안녕",
|
||||
"usage": {"input_tokens": 12, "output_tokens": 2},
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"cache_read_tokens": 2,
|
||||
"output_tokens": 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -334,6 +378,7 @@ class ProviderRegistryTest(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertEqual(events[-1].result.text, "안녕")
|
||||
self.assertEqual(events[-1].result.tokens_in, 12)
|
||||
self.assertEqual(events[-1].result.tokens_out, 2)
|
||||
self.assertEqual(events[-1].result.cost_usd, 0.0000303)
|
||||
args = captured[0]
|
||||
self.assertIn("--output-format", args)
|
||||
self.assertEqual(args[args.index("--output-format") + 1], "stream-json")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue