fix(edge): meter and size-cap knowledge embeddings in embed-chunks and search-knowledge
This commit is contained in:
parent
b35676c75c
commit
1afaea7214
12 changed files with 1111 additions and 270 deletions
166
server/supabase/functions/embed-chunks/handler.test.ts
Normal file
166
server/supabase/functions/embed-chunks/handler.test.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
// Regression tests for the embed-chunks cost guard (redteam r1 #18): before
|
||||
// the guard any signed-in account could have an unbounded number of chunks of
|
||||
// any size embedded by the paid provider, with no tier budget.
|
||||
|
||||
import { createEmbedChunksHandler, type KnowledgeChunkRow, type KnowledgeIndexStore } from './handler.ts'
|
||||
import { EMBEDDING_DIMENSIONS, type EmbeddingProvider } from '../_shared/openai-embeddings.ts'
|
||||
import { EMBEDDING_LIMITS, EMBEDDING_QUOTA } from '../_shared/embedding-quota.ts'
|
||||
import { MemoryUsageStore } from '../_shared/embedding-quota.fake.ts'
|
||||
import type { PlanQuotaTier } from '../_shared/core-contract.generated.ts'
|
||||
|
||||
function assert(condition: boolean, message: string): asserts condition {
|
||||
if (!condition) throw new Error(message)
|
||||
}
|
||||
|
||||
const USER = 'user-1'
|
||||
const DOC = '0b6c1f3e-4a5d-4e7f-8a9b-1c2d3e4f5a6b'
|
||||
const NOW = new Date('2026-09-28T12:00:00Z')
|
||||
const TODAY = '2026-09-28'
|
||||
|
||||
class FakeKnowledgeStore implements KnowledgeIndexStore {
|
||||
embedded = new Set<string>()
|
||||
indexed = false
|
||||
listLimit = -1
|
||||
constructor(public chunks: KnowledgeChunkRow[], public owner: string | null = USER) {}
|
||||
documentOwner(): Promise<string | null> {
|
||||
return Promise.resolve(this.owner)
|
||||
}
|
||||
countChunks(_doc: string, pendingOnly: boolean): Promise<number> {
|
||||
return Promise.resolve(pendingOnly ? this.pending().length : this.chunks.length)
|
||||
}
|
||||
listPendingChunks(_doc: string, limit: number): Promise<KnowledgeChunkRow[]> {
|
||||
this.listLimit = limit
|
||||
return Promise.resolve(this.pending().slice(0, limit))
|
||||
}
|
||||
saveEmbedding(_doc: string, chunkId: string): Promise<boolean> {
|
||||
this.embedded.add(chunkId)
|
||||
return Promise.resolve(true)
|
||||
}
|
||||
clearIndexed(): Promise<boolean> {
|
||||
this.indexed = false
|
||||
return Promise.resolve(true)
|
||||
}
|
||||
markIndexed(): Promise<boolean> {
|
||||
this.indexed = true
|
||||
return Promise.resolve(true)
|
||||
}
|
||||
private pending(): KnowledgeChunkRow[] {
|
||||
return this.chunks.filter((chunk) => !this.embedded.has(chunk.id))
|
||||
}
|
||||
}
|
||||
|
||||
class FakeProvider implements EmbeddingProvider {
|
||||
calls = 0
|
||||
inputs = 0
|
||||
constructor(private fail = false) {}
|
||||
embed(input: string | readonly string[]) {
|
||||
this.calls += 1
|
||||
const list = typeof input === 'string' ? [input] : input
|
||||
this.inputs += list.length
|
||||
if (this.fail) return Promise.resolve({ ok: false as const, reason: 'upstream' as const })
|
||||
return Promise.resolve({
|
||||
ok: true as const,
|
||||
data: list.map((_, index) => ({ index, embedding: new Array(EMBEDDING_DIMENSIONS).fill(0.1) })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function chunks(count: number, size = 800): KnowledgeChunkRow[] {
|
||||
return Array.from({ length: count }, (_, i) => ({ id: `c${i}`, content: 'x'.repeat(size) }))
|
||||
}
|
||||
|
||||
function setup(options: {
|
||||
tier?: PlanQuotaTier
|
||||
chunks: KnowledgeChunkRow[]
|
||||
providerFails?: boolean
|
||||
usageFails?: boolean
|
||||
}) {
|
||||
const store = new FakeKnowledgeStore(options.chunks)
|
||||
const provider = new FakeProvider(options.providerFails)
|
||||
const usage = new MemoryUsageStore(options.tier ?? 'free', TODAY)
|
||||
usage.failIncrement = options.usageFails ?? false
|
||||
const handler = createEmbedChunksHandler({
|
||||
authenticate: () => Promise.resolve({ id: USER }),
|
||||
embeddingProvider: () => provider,
|
||||
knowledgeStore: () => store,
|
||||
usageStore: () => usage,
|
||||
now: () => NOW,
|
||||
})
|
||||
const call = () => handler(new Request('https://edge/embed-chunks', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ document_id: DOC }),
|
||||
}))
|
||||
return { store, provider, usage, call }
|
||||
}
|
||||
|
||||
Deno.test('a free account over its weekly embedding budget is refused before any provider call', async () => {
|
||||
const { provider, usage, call } = setup({ chunks: chunks(10) })
|
||||
usage.rows.set('2026-09-26', EMBEDDING_QUOTA.free.limit - 5)
|
||||
const response = await call()
|
||||
assert(response.status === 429, `expected 429, got ${response.status}`)
|
||||
const body = await response.json()
|
||||
assert(body.error === 'quota_exceeded' && body.tier === 'free', 'quota body')
|
||||
assert(provider.calls === 0, 'provider was called despite the exhausted budget')
|
||||
assert((usage.rows.get(TODAY) ?? 0) === 0, 'refused request left units counted')
|
||||
})
|
||||
|
||||
Deno.test('a document over the chunk cap is refused without spending', async () => {
|
||||
const { provider, usage, call } = setup({ tier: 'enterprise', chunks: chunks(EMBEDDING_LIMITS.maxChunksPerDocument + 1, 10) })
|
||||
const response = await call()
|
||||
assert(response.status === 413, `expected 413, got ${response.status}`)
|
||||
assert((await response.json()).error === 'knowledge_document_too_large', 'error code')
|
||||
assert(provider.calls === 0 && !usage.rows.has(TODAY), 'spent on an oversized document')
|
||||
})
|
||||
|
||||
Deno.test('a chunk over the size cap is refused without spending', async () => {
|
||||
const list = chunks(3)
|
||||
list[1] = { id: 'huge', content: 'x'.repeat(EMBEDDING_LIMITS.maxChunkChars + 1) }
|
||||
const { provider, call } = setup({ tier: 'enterprise', chunks: list })
|
||||
const response = await call()
|
||||
assert(response.status === 413, `expected 413, got ${response.status}`)
|
||||
const body = await response.json()
|
||||
assert(body.error === 'knowledge_chunk_too_large' && body.chunk_id === 'huge', 'error body')
|
||||
assert(provider.calls === 0, 'provider was called for an oversized chunk')
|
||||
})
|
||||
|
||||
Deno.test('pending chunks are loaded with a bound', async () => {
|
||||
const { store, call } = setup({ tier: 'pro', chunks: chunks(3) })
|
||||
await call()
|
||||
assert(store.listLimit === EMBEDDING_LIMITS.maxChunksPerDocument, `unbounded pending select (${store.listLimit})`)
|
||||
})
|
||||
|
||||
Deno.test('a successful run indexes the document and counts the embedded volume', async () => {
|
||||
const { store, provider, usage, call } = setup({ tier: 'pro', chunks: chunks(150) })
|
||||
const response = await call()
|
||||
assert(response.status === 200, `expected 200, got ${response.status}`)
|
||||
const body = await response.json()
|
||||
assert(body.embedded === 150 && body.total === 150 && body.indexed === true, 'response body')
|
||||
assert(store.indexed, 'document not marked indexed')
|
||||
assert(provider.calls === 2 && provider.inputs === 150, 'batching changed')
|
||||
assert(usage.rows.get(TODAY) === 150, `usage ${usage.rows.get(TODAY)}`)
|
||||
})
|
||||
|
||||
Deno.test('units of batches the provider rejected are refunded', async () => {
|
||||
const { store, usage, call } = setup({ tier: 'pro', chunks: chunks(5), providerFails: true })
|
||||
const response = await call()
|
||||
assert(response.status === 502, `expected 502, got ${response.status}`)
|
||||
const body = await response.json()
|
||||
assert(body.error === 'embedding_failed' && body.remaining === 5, 'error body')
|
||||
assert(!store.indexed, 'failed document marked indexed')
|
||||
assert(usage.rows.get(TODAY) === 0, 'failed batch was charged')
|
||||
})
|
||||
|
||||
Deno.test('the quota store being down fails closed', async () => {
|
||||
const { provider, call } = setup({ tier: 'pro', chunks: chunks(2), usageFails: true })
|
||||
const response = await call()
|
||||
assert(response.status === 503, `expected 503, got ${response.status}`)
|
||||
assert((await response.json()).error === 'quota_unavailable', 'error code')
|
||||
assert(provider.calls === 0, 'provider called without a reservation')
|
||||
})
|
||||
|
||||
Deno.test('another user\'s document stays hidden', async () => {
|
||||
const { store, provider, call } = setup({ tier: 'pro', chunks: chunks(2) })
|
||||
store.owner = 'someone-else'
|
||||
const response = await call()
|
||||
assert(response.status === 404 && provider.calls === 0, 'foreign document was embedded')
|
||||
})
|
||||
178
server/supabase/functions/embed-chunks/handler.ts
Normal file
178
server/supabase/functions/embed-chunks/handler.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
// server/supabase/functions/embed-chunks/handler.ts
|
||||
// Use case: embed a document's pending knowledge chunks.
|
||||
// IO is injected through ports (auth, knowledge store, embedding provider,
|
||||
// usage store) so the size and quota guards are testable without Supabase or
|
||||
// OpenAI. index.ts wires the real adapters.
|
||||
|
||||
import { handleCorsPreflightRequest } from '../_shared/cors.ts'
|
||||
import { errorResponse, jsonResponse as json } from '../_shared/json-response.ts'
|
||||
import { type EmbeddingProvider, isEmbedding } from '../_shared/openai-embeddings.ts'
|
||||
import {
|
||||
EMBEDDING_LIMITS,
|
||||
type EmbeddingReservation,
|
||||
type EmbeddingUsageStore,
|
||||
embeddingUnits,
|
||||
firstOversizedChunk,
|
||||
planEmbeddingBatches,
|
||||
quotaExceededBody,
|
||||
refundEmbeddingUnits,
|
||||
reserveEmbeddingUnits,
|
||||
} from '../_shared/embedding-quota.ts'
|
||||
|
||||
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
||||
|
||||
export interface KnowledgeChunkRow {
|
||||
id: string
|
||||
content: string
|
||||
}
|
||||
|
||||
/** Thrown by a KnowledgeIndexStore when the database call fails. */
|
||||
export class KnowledgeStorageError extends Error {
|
||||
constructor() {
|
||||
super('knowledge_storage_failed')
|
||||
}
|
||||
}
|
||||
|
||||
/** Port over knowledge_documents / knowledge_chunks. Read methods throw KnowledgeStorageError. */
|
||||
export interface KnowledgeIndexStore {
|
||||
/** Owner of the document, or null when it does not exist. */
|
||||
documentOwner(documentId: string): Promise<string | null>
|
||||
/** Exact row count, or null when the database returned no count. */
|
||||
countChunks(documentId: string, pendingOnly: boolean): Promise<number | null>
|
||||
listPendingChunks(documentId: string, limit: number): Promise<KnowledgeChunkRow[]>
|
||||
/** Returns false when the write failed. */
|
||||
saveEmbedding(documentId: string, chunkId: string, embedding: number[]): Promise<boolean>
|
||||
/** Clear the indexed flag. With a userId the update is scoped to that owner. Returns false on failure. */
|
||||
clearIndexed(documentId: string, userId: string | null): Promise<boolean>
|
||||
/** Set the indexed flag. Returns false when the write failed or matched no row. */
|
||||
markIndexed(documentId: string, userId: string): Promise<boolean>
|
||||
}
|
||||
|
||||
export interface EmbedChunksDeps {
|
||||
authenticate(req: Request): Promise<{ id: string }>
|
||||
/** null when the provider key is not configured. */
|
||||
embeddingProvider(): EmbeddingProvider | null
|
||||
knowledgeStore(): KnowledgeIndexStore
|
||||
usageStore(): EmbeddingUsageStore
|
||||
now?(): Date
|
||||
}
|
||||
|
||||
async function embedBatch(
|
||||
provider: EmbeddingProvider,
|
||||
store: KnowledgeIndexStore,
|
||||
documentId: string,
|
||||
batch: KnowledgeChunkRow[],
|
||||
): Promise<number> {
|
||||
const outcome = await provider.embed(batch.map((chunk) => chunk.content))
|
||||
if (!outcome.ok || outcome.data.length !== batch.length) return 0
|
||||
|
||||
const received = new Set<number>()
|
||||
let saved = 0
|
||||
for (const item of outcome.data) {
|
||||
if (
|
||||
typeof item.index !== 'number'
|
||||
|| !Number.isInteger(item.index)
|
||||
|| item.index < 0
|
||||
|| item.index >= batch.length
|
||||
|| received.has(item.index)
|
||||
|| !isEmbedding(item.embedding)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
received.add(item.index)
|
||||
if (await store.saveEmbedding(documentId, batch[item.index].id, item.embedding)) saved += 1
|
||||
}
|
||||
return saved
|
||||
}
|
||||
|
||||
export function createEmbedChunksHandler(deps: EmbedChunksDeps): (req: Request) => Promise<Response> {
|
||||
const now = () => deps.now?.() ?? new Date()
|
||||
|
||||
return async (req: Request): Promise<Response> => {
|
||||
const preflight = handleCorsPreflightRequest(req)
|
||||
if (preflight) return preflight
|
||||
if (req.method !== 'POST') return json(405, { error: 'method_not_allowed' })
|
||||
|
||||
try {
|
||||
const user = await deps.authenticate(req)
|
||||
const body = await req.json().catch(() => null) as { document_id?: unknown } | null
|
||||
if (!body || typeof body.document_id !== 'string' || !UUID_PATTERN.test(body.document_id)) {
|
||||
return json(400, { error: 'invalid_document_id' })
|
||||
}
|
||||
const documentId = body.document_id
|
||||
|
||||
const provider = deps.embeddingProvider()
|
||||
if (!provider) return json(503, { error: 'embedding_provider_unavailable' })
|
||||
|
||||
const store = deps.knowledgeStore()
|
||||
const owner = await store.documentOwner(documentId)
|
||||
if (owner !== user.id) return json(404, { error: 'knowledge_document_not_found' })
|
||||
|
||||
const total = await store.countChunks(documentId, false) ?? 0
|
||||
if (total === 0) {
|
||||
await store.clearIndexed(documentId, null)
|
||||
return json(409, { error: 'knowledge_document_empty' })
|
||||
}
|
||||
if (total > EMBEDDING_LIMITS.maxChunksPerDocument) {
|
||||
return json(413, {
|
||||
error: 'knowledge_document_too_large',
|
||||
total,
|
||||
max_chunks: EMBEDDING_LIMITS.maxChunksPerDocument,
|
||||
})
|
||||
}
|
||||
|
||||
const chunks = await store.listPendingChunks(documentId, EMBEDDING_LIMITS.maxChunksPerDocument)
|
||||
const oversized = firstOversizedChunk(chunks)
|
||||
if (oversized !== -1) {
|
||||
return json(413, {
|
||||
error: 'knowledge_chunk_too_large',
|
||||
chunk_id: chunks[oversized].id,
|
||||
max_chars: EMBEDDING_LIMITS.maxChunkChars,
|
||||
})
|
||||
}
|
||||
|
||||
// Reserve the whole run before the first paid call; unspent units are refunded.
|
||||
const usage = deps.usageStore()
|
||||
const units = embeddingUnits(chunks.map((chunk) => chunk.content))
|
||||
let reservation: EmbeddingReservation
|
||||
try {
|
||||
reservation = await reserveEmbeddingUnits(usage, user.id, units, now())
|
||||
} catch {
|
||||
return json(503, { error: 'quota_unavailable' })
|
||||
}
|
||||
if (!reservation.allowed) return json(429, quotaExceededBody(reservation, units))
|
||||
|
||||
let embedded = 0
|
||||
let failed = 0
|
||||
let unspentUnits = 0
|
||||
for (const batch of planEmbeddingBatches(chunks)) {
|
||||
let saved = 0
|
||||
try {
|
||||
saved = await embedBatch(provider, store, documentId, batch)
|
||||
} catch {
|
||||
saved = 0
|
||||
}
|
||||
embedded += saved
|
||||
failed += batch.length - saved
|
||||
if (saved === 0) unspentUnits += embeddingUnits(batch.map((chunk) => chunk.content))
|
||||
}
|
||||
await refundEmbeddingUnits(usage, user.id, reservation, unspentUnits, now())
|
||||
|
||||
const remaining = await store.countChunks(documentId, true) ?? total
|
||||
if (failed > 0 || remaining > 0) {
|
||||
if (!await store.clearIndexed(documentId, user.id)) {
|
||||
return json(500, { error: 'knowledge_storage_failed' })
|
||||
}
|
||||
return json(502, { error: 'embedding_failed', embedded, remaining })
|
||||
}
|
||||
|
||||
if (!await store.markIndexed(documentId, user.id)) {
|
||||
return json(500, { error: 'knowledge_storage_failed' })
|
||||
}
|
||||
return json(200, { embedded, total, indexed: true })
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeStorageError) return json(500, { error: 'knowledge_storage_failed' })
|
||||
return errorResponse(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,181 +1,26 @@
|
|||
import { corsHeaders, handleCorsPreflightRequest } from '../_shared/cors.ts'
|
||||
import { requireUser, authErrorResponse, type AuthError } from '../_shared/auth.ts'
|
||||
// server/supabase/functions/embed-chunks/index.ts
|
||||
// Composition root: wires the real adapters into the embed-chunks use case.
|
||||
|
||||
import { requireUser } from '../_shared/auth.ts'
|
||||
import { createServiceRoleClient } from '../_shared/quota.ts'
|
||||
import { readProviderKey } from '../_shared/provider-key.ts'
|
||||
import { createOpenAIEmbeddingProvider } from '../_shared/openai-embeddings.ts'
|
||||
import { createSupabaseEmbeddingUsageStore } from '../_shared/embedding-quota.ts'
|
||||
import { createEmbedChunksHandler } from './handler.ts'
|
||||
import { createSupabaseKnowledgeStore } from './supabase-knowledge-store.ts'
|
||||
|
||||
const EMBEDDING_DIMENSIONS = 1536
|
||||
const PROVIDER_TIMEOUT_MS = 45_000
|
||||
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
||||
|
||||
interface OpenAIEmbeddingResponse {
|
||||
data?: Array<{ embedding?: unknown; index?: unknown }>
|
||||
let serviceClient: ReturnType<typeof createServiceRoleClient> | null = null
|
||||
function service(): ReturnType<typeof createServiceRoleClient> {
|
||||
serviceClient ??= createServiceRoleClient()
|
||||
return serviceClient
|
||||
}
|
||||
|
||||
function json(status: number, body: Record<string, unknown>): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
function validEmbedding(value: unknown): value is number[] {
|
||||
return Array.isArray(value)
|
||||
&& value.length === EMBEDDING_DIMENSIONS
|
||||
&& value.every((entry) => typeof entry === 'number' && Number.isFinite(entry))
|
||||
}
|
||||
|
||||
Deno.serve(async (req: Request) => {
|
||||
const preflight = handleCorsPreflightRequest(req)
|
||||
if (preflight) return preflight
|
||||
if (req.method !== 'POST') return json(405, { error: 'method_not_allowed' })
|
||||
|
||||
try {
|
||||
const user = await requireUser(req)
|
||||
const body = await req.json().catch(() => null) as { document_id?: unknown } | null
|
||||
if (!body || typeof body.document_id !== 'string' || !UUID_PATTERN.test(body.document_id)) {
|
||||
return json(400, { error: 'invalid_document_id' })
|
||||
}
|
||||
|
||||
const openaiKey = readProviderKey('OPENAI_API_KEY')
|
||||
if (!openaiKey) return json(503, { error: 'embedding_provider_unavailable' })
|
||||
|
||||
const serviceClient = createServiceRoleClient()
|
||||
const documentResult = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.select('id,user_id')
|
||||
.eq('id', body.document_id)
|
||||
.maybeSingle()
|
||||
if (documentResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
if (!documentResult.data || documentResult.data.user_id !== user.id) {
|
||||
return json(404, { error: 'knowledge_document_not_found' })
|
||||
}
|
||||
|
||||
const totalResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', body.document_id)
|
||||
if (totalResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
const total = totalResult.count ?? 0
|
||||
if (total === 0) {
|
||||
await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', body.document_id)
|
||||
return json(409, { error: 'knowledge_document_empty' })
|
||||
}
|
||||
|
||||
const pendingResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id,content')
|
||||
.eq('document_id', body.document_id)
|
||||
.is('embedding', null)
|
||||
.order('chunk_index', { ascending: true })
|
||||
if (pendingResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
const chunks = (pendingResult.data ?? []) as Array<{ id: string; content: string }>
|
||||
let embedded = 0
|
||||
let failed = 0
|
||||
|
||||
for (let offset = 0; offset < chunks.length; offset += 100) {
|
||||
const batch = chunks.slice(offset, offset + 100)
|
||||
try {
|
||||
const response = await fetch('https://api.openai.com/v1/embeddings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${openaiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'text-embedding-3-small',
|
||||
input: batch.map((chunk) => chunk.content),
|
||||
dimensions: EMBEDDING_DIMENSIONS,
|
||||
}),
|
||||
signal: AbortSignal.timeout(PROVIDER_TIMEOUT_MS),
|
||||
})
|
||||
if (!response.ok) {
|
||||
failed += batch.length
|
||||
continue
|
||||
}
|
||||
|
||||
const payload = await response.json().catch(() => null) as OpenAIEmbeddingResponse | null
|
||||
if (!payload || !Array.isArray(payload.data) || payload.data.length !== batch.length) {
|
||||
failed += batch.length
|
||||
continue
|
||||
}
|
||||
|
||||
const received = new Set<number>()
|
||||
let batchSuccess = 0
|
||||
for (const item of payload.data) {
|
||||
if (
|
||||
typeof item.index !== 'number'
|
||||
|| !Number.isInteger(item.index)
|
||||
|| item.index < 0
|
||||
|| item.index >= batch.length
|
||||
|| received.has(item.index)
|
||||
|| !validEmbedding(item.embedding)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
received.add(item.index)
|
||||
const update = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.update({ embedding: item.embedding })
|
||||
.eq('id', batch[item.index].id)
|
||||
.eq('document_id', body.document_id)
|
||||
if (!update.error) {
|
||||
embedded += 1
|
||||
batchSuccess += 1
|
||||
}
|
||||
}
|
||||
failed += batch.length - batchSuccess
|
||||
} catch {
|
||||
failed += batch.length
|
||||
}
|
||||
}
|
||||
|
||||
const remainingResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', body.document_id)
|
||||
.is('embedding', null)
|
||||
if (remainingResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
const remaining = remainingResult.count ?? total
|
||||
if (failed > 0 || remaining > 0) {
|
||||
const rollback = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', body.document_id)
|
||||
.eq('user_id', user.id)
|
||||
if (rollback.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
return json(502, {
|
||||
error: 'embedding_failed',
|
||||
embedded,
|
||||
remaining,
|
||||
})
|
||||
}
|
||||
|
||||
const indexed = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: true, indexed_at: new Date().toISOString() })
|
||||
.eq('id', body.document_id)
|
||||
.eq('user_id', user.id)
|
||||
.select('id')
|
||||
.maybeSingle()
|
||||
if (indexed.error || !indexed.data) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
return json(200, { embedded, total, indexed: true })
|
||||
} catch (error) {
|
||||
if (
|
||||
error
|
||||
&& typeof error === 'object'
|
||||
&& 'status' in error
|
||||
&& (error.status === 401 || error.status === 403)
|
||||
&& 'message' in error
|
||||
&& typeof error.message === 'string'
|
||||
) {
|
||||
return authErrorResponse(error as AuthError, corsHeaders)
|
||||
}
|
||||
return json(500, { error: 'internal_error' })
|
||||
}
|
||||
})
|
||||
Deno.serve(createEmbedChunksHandler({
|
||||
authenticate: requireUser,
|
||||
embeddingProvider: () => {
|
||||
const key = readProviderKey('OPENAI_API_KEY')
|
||||
return key ? createOpenAIEmbeddingProvider(key) : null
|
||||
},
|
||||
knowledgeStore: () => createSupabaseKnowledgeStore(service()),
|
||||
usageStore: () => createSupabaseEmbeddingUsageStore(service()),
|
||||
}))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
// server/supabase/functions/embed-chunks/supabase-knowledge-store.ts
|
||||
// KnowledgeIndexStore adapter over the service-role Supabase client.
|
||||
|
||||
import type { createClient } from '@supabase/supabase-js'
|
||||
import { type KnowledgeChunkRow, type KnowledgeIndexStore, KnowledgeStorageError } from './handler.ts'
|
||||
|
||||
type SupabaseClient = ReturnType<typeof createClient>
|
||||
|
||||
export function createSupabaseKnowledgeStore(client: SupabaseClient): KnowledgeIndexStore {
|
||||
return {
|
||||
async documentOwner(documentId) {
|
||||
const { data, error } = await client
|
||||
.from('knowledge_documents')
|
||||
.select('id,user_id')
|
||||
.eq('id', documentId)
|
||||
.maybeSingle()
|
||||
if (error) throw new KnowledgeStorageError()
|
||||
const row = data as { user_id?: unknown } | null
|
||||
return row && typeof row.user_id === 'string' ? row.user_id : null
|
||||
},
|
||||
async countChunks(documentId, pendingOnly) {
|
||||
let query = client
|
||||
.from('knowledge_chunks')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', documentId)
|
||||
if (pendingOnly) query = query.is('embedding', null)
|
||||
const { count, error } = await query
|
||||
if (error) throw new KnowledgeStorageError()
|
||||
return count
|
||||
},
|
||||
async listPendingChunks(documentId, limit) {
|
||||
const { data, error } = await client
|
||||
.from('knowledge_chunks')
|
||||
.select('id,content')
|
||||
.eq('document_id', documentId)
|
||||
.is('embedding', null)
|
||||
.order('chunk_index', { ascending: true })
|
||||
.limit(limit)
|
||||
if (error) throw new KnowledgeStorageError()
|
||||
return (data ?? []) as KnowledgeChunkRow[]
|
||||
},
|
||||
async saveEmbedding(documentId, chunkId, embedding) {
|
||||
const { error } = await client
|
||||
.from('knowledge_chunks')
|
||||
.update({ embedding })
|
||||
.eq('id', chunkId)
|
||||
.eq('document_id', documentId)
|
||||
return !error
|
||||
},
|
||||
async clearIndexed(documentId, userId) {
|
||||
let query = client
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', documentId)
|
||||
if (userId !== null) query = query.eq('user_id', userId)
|
||||
const { error } = await query
|
||||
return !error
|
||||
},
|
||||
async markIndexed(documentId, userId) {
|
||||
const { data, error } = await client
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: true, indexed_at: new Date().toISOString() })
|
||||
.eq('id', documentId)
|
||||
.eq('user_id', userId)
|
||||
.select('id')
|
||||
.maybeSingle()
|
||||
return !error && Boolean(data)
|
||||
},
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue