feat: ship Wave 5-6 multi-surface hygiene, optimization, and bug fixes
This commit is contained in:
parent
01ef3293cf
commit
66b6b45ba9
399 changed files with 17828 additions and 11330 deletions
|
|
@ -376,3 +376,29 @@ Deno.test('SSE completion tracker finds message_stop split across chunks', () =>
|
|||
tracker.push(encoder.encode('op\ndata: {}\n\n'))
|
||||
assert(tracker.completed, 'completed')
|
||||
})
|
||||
|
||||
Deno.test('non-stream invalid upstream JSON returns 502 and refunds quota', async () => {
|
||||
const store = new FakeLlmQuotaStore(5)
|
||||
const handler = createLlmProxyHandler(buildDeps(store, () => Promise.resolve(new Response('not-json', { status: 200 }))))
|
||||
|
||||
const resp = await handler(llmRequest(false))
|
||||
assertEquals(resp.status, 502, 'status')
|
||||
assertEquals(await resp.json(), { error: 'provider_invalid_response' }, 'body')
|
||||
assertEquals(store.finalized, [{ id: 'res-1', succeeded: false }], 'refunded')
|
||||
})
|
||||
|
||||
Deno.test('oversized JSON body returns 413 without reserving quota', async () => {
|
||||
const store = new FakeLlmQuotaStore(5)
|
||||
const handler = createLlmProxyHandler(buildDeps(store, () => Promise.resolve(okJson())))
|
||||
|
||||
const resp = await handler(new Request('http://localhost/llm-proxy', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
messages: [{ role: 'user', content: 'a'.repeat(600 * 1024) }],
|
||||
}),
|
||||
}))
|
||||
assertEquals(resp.status, 413, 'status')
|
||||
assertEquals(await resp.json(), { error: 'request_too_large' }, 'body')
|
||||
assertEquals(store.reserveCalls, 0, 'quota must not be touched')
|
||||
})
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import { corsHeaders, handleCorsPreflightRequest } from '../_shared/cors.ts'
|
||||
import { authErrorResponse, type AuthError } from '../_shared/auth.ts'
|
||||
import { createStatusJsonResponder, JsonBodyError, readJsonBody } from '../_shared/json-response.ts'
|
||||
import {
|
||||
getQuotaPolicy,
|
||||
modelToQuotaKey,
|
||||
|
|
@ -100,13 +101,12 @@ export interface LlmProxyDeps {
|
|||
timeouts?: Partial<LlmProviderTimeouts>
|
||||
}
|
||||
|
||||
const NO_STORE_JSON = { ...corsHeaders, 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }
|
||||
const MAX_REQUEST_BYTES = 512 * 1024
|
||||
const noStoreJson = createStatusJsonResponder({ noStore: true })
|
||||
const corsJson = createStatusJsonResponder({ noStore: false })
|
||||
|
||||
function json(status: number, body: Record<string, unknown>, noStore = true): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: noStore ? NO_STORE_JSON : { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
})
|
||||
return (noStore ? noStoreJson : corsJson)(status, body)
|
||||
}
|
||||
|
||||
/** 쓰지 않을 공급자 응답 본문을 닫아 연결과 생성을 정리한다. */
|
||||
|
|
@ -163,8 +163,11 @@ export function createLlmProxyHandler(deps: LlmProxyDeps): (req: Request) => Pro
|
|||
|
||||
let rawBody: unknown
|
||||
try {
|
||||
rawBody = await req.json()
|
||||
} catch {
|
||||
rawBody = await readJsonBody(req, { maxBytes: MAX_REQUEST_BYTES })
|
||||
} catch (err) {
|
||||
if (err instanceof JsonBodyError && err.status === 413) {
|
||||
return json(413, { error: 'request_too_large' })
|
||||
}
|
||||
throw new LlmRequestError('Invalid JSON body')
|
||||
}
|
||||
const body = parseLlmRequest(rawBody)
|
||||
|
|
@ -291,6 +294,9 @@ export function createLlmProxyHandler(deps: LlmProxyDeps): (req: Request) => Pro
|
|||
let nonStreamData: unknown
|
||||
try {
|
||||
nonStreamData = await anthropicResp.json()
|
||||
} catch {
|
||||
await settle(false)
|
||||
return json(502, { error: 'provider_invalid_response' })
|
||||
} finally {
|
||||
deadline.dispose()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue