fix(llm-proxy): reserve quota before the provider call and stop cutting off long streams
This commit is contained in:
Yun Chan
parent
2f94d24c99
commit
b306034bfc
7 changed files with 1489 additions and 307 deletions
|
|
@ -1,293 +1,50 @@
|
|||
// server/supabase/functions/llm-proxy/index.ts
|
||||
// Anthropic Claude Messages API 프록시.
|
||||
// Anthropic Claude Messages API 프록시 — composition root.
|
||||
// Phase 3.2: 모델별 쿼터 (Haiku/Sonnet/Opus × Free/Pro/Pro+)
|
||||
// 요청: application/json { messages, system?, max_tokens?, model? }
|
||||
// 응답: JSON (non-stream) 또는 SSE (stream=true)
|
||||
// 유스케이스와 쿼터/기한 규칙은 handler.ts, 공급자 기한은 provider-deadline.ts 에 있다.
|
||||
|
||||
import { corsHeaders, handleCorsPreflightRequest } from '../_shared/cors.ts'
|
||||
import { requireUser, authErrorResponse, type AuthError } from '../_shared/auth.ts'
|
||||
import { requireUser } from '../_shared/auth.ts'
|
||||
import {
|
||||
checkQuota,
|
||||
consumeQuota,
|
||||
createServiceRoleClient,
|
||||
modelToQuotaKey,
|
||||
getQuotaPolicy,
|
||||
type Tier,
|
||||
finalizeLlmQuota,
|
||||
readSubscriptionTier,
|
||||
reserveLlmQuota,
|
||||
} from '../_shared/quota.ts'
|
||||
import {
|
||||
hasAssistantText,
|
||||
LlmRequestError,
|
||||
parseLlmRequest,
|
||||
} from '../_shared/llm-contract.ts'
|
||||
import {
|
||||
GENERATION_ID_HEADER,
|
||||
GENERATION_PURPOSE_HEADER,
|
||||
GenerationReceiptError,
|
||||
parseGenerationPurpose,
|
||||
parseGenerationReceiptId,
|
||||
type GenerationPurpose,
|
||||
} from '../_shared/generation-receipt.ts'
|
||||
import { buildAnthropicSystemBlocks } from '../_shared/generative-ai-safety.ts'
|
||||
import { GenerationReceiptError, parseGenerationReceiptId } from '../_shared/generation-receipt.ts'
|
||||
import { readProviderKey } from '../_shared/provider-key.ts'
|
||||
import { createLlmProxyHandler, type LlmQuotaStore } from './handler.ts'
|
||||
|
||||
/** 티어별 허용 모델 — free는 Haiku만, pro는 +Sonnet, pro_plus는 +Opus, team/enterprise는 전 모델 */
|
||||
const TIER_MODELS: Record<Tier, string[]> = {
|
||||
free: ['claude-haiku-4-5-20251001'],
|
||||
pro: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6', 'claude-opus-4-6'],
|
||||
pro_plus: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6', 'claude-opus-4-6'],
|
||||
team: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6', 'claude-opus-4-6'],
|
||||
enterprise: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6', 'claude-opus-4-6'],
|
||||
let serviceClient: ReturnType<typeof createServiceRoleClient> | null = null
|
||||
function service(): ReturnType<typeof createServiceRoleClient> {
|
||||
serviceClient ??= createServiceRoleClient()
|
||||
return serviceClient
|
||||
}
|
||||
|
||||
const DEFAULT_MODEL: Record<Tier, string> = {
|
||||
free: 'claude-haiku-4-5-20251001',
|
||||
pro: 'claude-sonnet-4-6',
|
||||
pro_plus: 'claude-sonnet-4-6',
|
||||
team: 'claude-sonnet-4-6',
|
||||
enterprise: 'claude-sonnet-4-6',
|
||||
}
|
||||
|
||||
async function issueGenerationReceipt(
|
||||
serviceClient: ReturnType<typeof createServiceRoleClient>,
|
||||
userId: string,
|
||||
purpose: GenerationPurpose | null,
|
||||
model: string,
|
||||
): Promise<string | null> {
|
||||
if (purpose === null) return null
|
||||
const { data, error } = await serviceClient.rpc('issue_content_generation_receipt_v1', {
|
||||
p_actor_id: userId,
|
||||
p_purpose: purpose,
|
||||
p_model: model,
|
||||
})
|
||||
if (error) {
|
||||
console.error('Generation receipt issuance failed', { code: error.code ?? 'unknown' })
|
||||
throw new GenerationReceiptError('generation_receipt_unavailable')
|
||||
}
|
||||
return parseGenerationReceiptId(data)
|
||||
}
|
||||
|
||||
/** 쓰지 않을 공급자 응답 본문을 닫아 연결과 생성을 정리한다. */
|
||||
async function discardBody(resp: Response): Promise<void> {
|
||||
if (!resp.body || resp.bodyUsed) return
|
||||
try {
|
||||
await resp.body.cancel()
|
||||
} catch {
|
||||
// 이미 닫힌 스트림 — 무시
|
||||
function supabaseLlmQuotaStore(client: ReturnType<typeof createServiceRoleClient>): LlmQuotaStore {
|
||||
return {
|
||||
readTier: (userId) => readSubscriptionTier(userId, client),
|
||||
reserve: (userId, reservationId, feature, baseLimit, period) =>
|
||||
reserveLlmQuota(userId, reservationId, feature, baseLimit, period, client),
|
||||
finalize: (reservationId, succeeded) => finalizeLlmQuota(reservationId, succeeded, client),
|
||||
}
|
||||
}
|
||||
|
||||
function generationHeaders(generationId: string | null): Record<string, string> {
|
||||
return generationId === null ? {} : { [GENERATION_ID_HEADER]: generationId }
|
||||
}
|
||||
|
||||
Deno.serve(async (req: Request) => {
|
||||
const preflight = handleCorsPreflightRequest(req)
|
||||
if (preflight) return preflight
|
||||
|
||||
if (req.method !== 'POST') {
|
||||
return new Response(JSON.stringify({ error: 'Method not allowed' }), {
|
||||
status: 405,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
Deno.serve(createLlmProxyHandler({
|
||||
authenticate: requireUser,
|
||||
providerKey: () => readProviderKey('ANTHROPIC_API_KEY'),
|
||||
quotaStore: () => supabaseLlmQuotaStore(service()),
|
||||
issueGenerationReceipt: async (userId, purpose, model) => {
|
||||
const { data, error } = await service().rpc('issue_content_generation_receipt_v1', {
|
||||
p_actor_id: userId,
|
||||
p_purpose: purpose,
|
||||
p_model: model,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const user = await requireUser(req)
|
||||
const serviceClient = createServiceRoleClient()
|
||||
|
||||
let rawBody: unknown
|
||||
try {
|
||||
rawBody = await req.json()
|
||||
} catch {
|
||||
throw new LlmRequestError('Invalid JSON body')
|
||||
if (error) {
|
||||
console.error('Generation receipt issuance failed', { code: error.code ?? 'unknown' })
|
||||
throw new GenerationReceiptError('generation_receipt_unavailable')
|
||||
}
|
||||
const body = parseLlmRequest(rawBody)
|
||||
const generationPurpose = parseGenerationPurpose(
|
||||
req.headers.get(GENERATION_PURPOSE_HEADER),
|
||||
)
|
||||
|
||||
// A deployment without a provider must not consume quota or fabricate an answer.
|
||||
const anthropicKey = readProviderKey('ANTHROPIC_API_KEY')
|
||||
if (!anthropicKey) {
|
||||
return new Response(JSON.stringify({ error: 'provider_unavailable' }), {
|
||||
status: 503,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
|
||||
// 1단계: 티어 조회 (어떤 모델이든 한 번만 읽으면 됨 — haiku로 대리 조회)
|
||||
const tierCheck = await checkQuota(user.id, 'llm_haiku', serviceClient)
|
||||
const tier = tierCheck.tier
|
||||
|
||||
// 모델 선택 + 티어 검증
|
||||
const requestedModel = body.model ?? DEFAULT_MODEL[tier]
|
||||
if (!TIER_MODELS[tier].includes(requestedModel)) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: 'model_not_allowed',
|
||||
tier,
|
||||
requested: requestedModel,
|
||||
allowed: TIER_MODELS[tier],
|
||||
}),
|
||||
{
|
||||
status: 403,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// 2단계: 해당 모델의 쿼터 확인 (모델별 일간/주간)
|
||||
const quotaKey = modelToQuotaKey(requestedModel)
|
||||
const modelQuota = await checkQuota(user.id, quotaKey, serviceClient)
|
||||
if (!modelQuota.allowed) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: 'quota_exceeded',
|
||||
model: requestedModel,
|
||||
current: modelQuota.current,
|
||||
limit: modelQuota.limit,
|
||||
period: modelQuota.period,
|
||||
tier,
|
||||
overage_credits: modelQuota.overageCredits,
|
||||
}),
|
||||
{
|
||||
status: 429,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// 3단계: 공급자 호출 (Prompt Caching 2024-07-31 활성화)
|
||||
// 쿼터는 공급자가 요청을 받아들인 뒤에만 소비한다. 공급자 5xx/과부하/타임아웃으로
|
||||
// 답을 주지 못한 요청이 쿼터를 깎고, 클라이언트 재시도가 한 번 더 깎는 일을 막는다.
|
||||
const systemPayload = buildAnthropicSystemBlocks(body.system)
|
||||
|
||||
const anthropicResp = await fetch('https://api.anthropic.com/v1/messages', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': anthropicKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: requestedModel,
|
||||
max_tokens: body.max_tokens,
|
||||
system: systemPayload,
|
||||
messages: body.messages,
|
||||
stream: body.stream,
|
||||
}),
|
||||
signal: AbortSignal.timeout(45_000),
|
||||
})
|
||||
|
||||
if (!anthropicResp.ok) {
|
||||
console.error('Anthropic request failed', { status: anthropicResp.status })
|
||||
await discardBody(anthropicResp)
|
||||
return new Response(JSON.stringify({ error: 'provider_request_failed' }), {
|
||||
status: 502,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
|
||||
// 비스트리밍 응답은 형식 검증까지 통과한 뒤에 소비한다.
|
||||
let nonStreamData: unknown = null
|
||||
if (!body.stream || !anthropicResp.body) {
|
||||
nonStreamData = await anthropicResp.json()
|
||||
if (!hasAssistantText(nonStreamData)) {
|
||||
console.error('Anthropic returned an invalid response shape')
|
||||
return new Response(JSON.stringify({ error: 'provider_invalid_response' }), {
|
||||
status: 502,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 4단계: 쿼터 소비 (원자적 base → overage fallback).
|
||||
// 동시 요청이 checkQuota 를 함께 통과했다면 여기서 초과분이 거절된다.
|
||||
const policy = getQuotaPolicy(tier, quotaKey)
|
||||
let consume: Awaited<ReturnType<typeof consumeQuota>>
|
||||
try {
|
||||
consume = await consumeQuota(user.id, quotaKey, serviceClient, policy.limit)
|
||||
} catch (err) {
|
||||
await discardBody(anthropicResp)
|
||||
throw err
|
||||
}
|
||||
if (!consume.allowed) {
|
||||
await discardBody(anthropicResp)
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: 'quota_exceeded',
|
||||
model: requestedModel,
|
||||
current: consume.current,
|
||||
limit: consume.limit,
|
||||
tier,
|
||||
overage_credits: consume.overageCredits,
|
||||
}),
|
||||
{
|
||||
status: 429,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
if (body.stream && anthropicResp.body) {
|
||||
const generationId = await issueGenerationReceipt(
|
||||
serviceClient,
|
||||
user.id,
|
||||
generationPurpose,
|
||||
requestedModel,
|
||||
)
|
||||
return new Response(anthropicResp.body, {
|
||||
status: 200,
|
||||
headers: {
|
||||
...corsHeaders,
|
||||
...generationHeaders(generationId),
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-store',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const generationId = await issueGenerationReceipt(
|
||||
serviceClient,
|
||||
user.id,
|
||||
generationPurpose,
|
||||
requestedModel,
|
||||
)
|
||||
return new Response(JSON.stringify(nonStreamData), {
|
||||
status: 200,
|
||||
headers: {
|
||||
...corsHeaders,
|
||||
...generationHeaders(generationId),
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof GenerationReceiptError) {
|
||||
const invalidPurpose = err.code === 'invalid_generation_purpose'
|
||||
return new Response(JSON.stringify({
|
||||
error: invalidPurpose ? 'invalid_request' : 'generation_receipt_unavailable',
|
||||
}), {
|
||||
status: invalidPurpose ? 400 : 503,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
if (err instanceof LlmRequestError) {
|
||||
return new Response(JSON.stringify({ error: 'invalid_request', message: err.message }), {
|
||||
status: err.status,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
if (err && typeof err === 'object' && 'status' in err && 'message' in err) {
|
||||
return authErrorResponse(err as AuthError, corsHeaders)
|
||||
}
|
||||
const timedOut = err instanceof DOMException && err.name === 'TimeoutError'
|
||||
console.error('llm-proxy failed', { kind: timedOut ? 'provider_timeout' : 'internal_error' })
|
||||
return new Response(JSON.stringify({ error: timedOut ? 'provider_timeout' : 'internal_error' }), {
|
||||
status: timedOut ? 504 : 500,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' },
|
||||
})
|
||||
}
|
||||
})
|
||||
return parseGenerationReceiptId(data)
|
||||
},
|
||||
}))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue