feat(release): prepare 1.1.0 candidate
This commit is contained in:
Yun Chan
parent
5a34f66981
commit
5205dcdfa9
736 changed files with 115667 additions and 12203 deletions
|
|
@ -1,159 +1,180 @@
|
|||
// server/supabase/functions/embed-chunks/index.ts
|
||||
// 특정 document의 청크들을 OpenAI 임베딩으로 변환하여 knowledge_chunks.embedding 컬럼 업데이트.
|
||||
//
|
||||
// 요청:
|
||||
// POST { document_id: "uuid" }
|
||||
// 응답:
|
||||
// { embedded: number, errors: string[] }
|
||||
//
|
||||
// 환경변수:
|
||||
// OPENAI_API_KEY (text-embedding-3-small 사용)
|
||||
|
||||
import { corsHeaders, handleCorsPreflightRequest } from '../_shared/cors.ts'
|
||||
import { requireUser, authErrorResponse, type AuthError } from '../_shared/auth.ts'
|
||||
import { createServiceRoleClient } from '../_shared/quota.ts'
|
||||
|
||||
interface EmbedRequest {
|
||||
document_id: string
|
||||
}
|
||||
const EMBEDDING_DIMENSIONS = 1536
|
||||
const PROVIDER_TIMEOUT_MS = 45_000
|
||||
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
||||
|
||||
interface OpenAIEmbeddingResponse {
|
||||
data: Array<{ embedding: number[]; index: number }>
|
||||
model: string
|
||||
usage: { prompt_tokens: number; total_tokens: number }
|
||||
data?: Array<{ embedding?: unknown; index?: unknown }>
|
||||
}
|
||||
|
||||
function json(status: number, body: Record<string, unknown>): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
function validEmbedding(value: unknown): value is number[] {
|
||||
return Array.isArray(value)
|
||||
&& value.length === EMBEDDING_DIMENSIONS
|
||||
&& value.every((entry) => typeof entry === 'number' && Number.isFinite(entry))
|
||||
}
|
||||
|
||||
// @ts-expect-error — Deno 런타임 전역
|
||||
Deno.serve(async (req: Request) => {
|
||||
const preflight = handleCorsPreflightRequest(req)
|
||||
if (preflight) return preflight
|
||||
|
||||
if (req.method !== 'POST') {
|
||||
return new Response(JSON.stringify({ error: 'Method not allowed' }), {
|
||||
status: 405,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' }
|
||||
})
|
||||
}
|
||||
if (req.method !== 'POST') return json(405, { error: 'method_not_allowed' })
|
||||
|
||||
try {
|
||||
const user = await requireUser(req)
|
||||
const body = (await req.json()) as EmbedRequest
|
||||
|
||||
if (!body.document_id) {
|
||||
return new Response(JSON.stringify({ error: 'document_id 필요' }), {
|
||||
status: 400,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' }
|
||||
})
|
||||
const body = await req.json().catch(() => null) as { document_id?: unknown } | null
|
||||
if (!body || typeof body.document_id !== 'string' || !UUID_PATTERN.test(body.document_id)) {
|
||||
return json(400, { error: 'invalid_document_id' })
|
||||
}
|
||||
|
||||
// @ts-expect-error — Deno.env
|
||||
const openaiKey = Deno.env.get('OPENAI_API_KEY') ?? ''
|
||||
if (!openaiKey) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'openai_not_configured', message: 'OPENAI_API_KEY 미설정' }),
|
||||
{ status: 503, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
||||
)
|
||||
}
|
||||
if (!openaiKey) return json(503, { error: 'embedding_provider_unavailable' })
|
||||
|
||||
const serviceClient = createServiceRoleClient()
|
||||
|
||||
// 문서 소유권 확인
|
||||
const { data: doc } = await serviceClient
|
||||
const documentResult = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.select('id, user_id')
|
||||
.select('id,user_id')
|
||||
.eq('id', body.document_id)
|
||||
.maybeSingle()
|
||||
|
||||
if (!doc || (doc.user_id as string) !== user.id) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'forbidden' }),
|
||||
{ status: 403, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
||||
)
|
||||
if (documentResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
if (!documentResult.data || documentResult.data.user_id !== user.id) {
|
||||
return json(404, { error: 'knowledge_document_not_found' })
|
||||
}
|
||||
|
||||
// 임베딩 대상 청크 조회 (embedding IS NULL인 것만)
|
||||
const { data: chunks, error: chunksErr } = await serviceClient
|
||||
const totalResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id, content')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', body.document_id)
|
||||
if (totalResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
const total = totalResult.count ?? 0
|
||||
if (total === 0) {
|
||||
await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', body.document_id)
|
||||
return json(409, { error: 'knowledge_document_empty' })
|
||||
}
|
||||
|
||||
const pendingResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id,content')
|
||||
.eq('document_id', body.document_id)
|
||||
.is('embedding', null)
|
||||
.order('chunk_index', { ascending: true })
|
||||
if (pendingResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
if (chunksErr) {
|
||||
throw new Error(`청크 조회 실패: ${chunksErr.message}`)
|
||||
}
|
||||
if (!chunks || chunks.length === 0) {
|
||||
return new Response(JSON.stringify({ embedded: 0, errors: [] }), {
|
||||
status: 200,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
// OpenAI 임베딩 API 배치 호출 (한 번에 최대 100개)
|
||||
const errors: string[] = []
|
||||
const chunks = (pendingResult.data ?? []) as Array<{ id: string; content: string }>
|
||||
let embedded = 0
|
||||
const batchSize = 100
|
||||
let failed = 0
|
||||
|
||||
for (let i = 0; i < chunks.length; i += batchSize) {
|
||||
const batch = chunks.slice(i, i + batchSize) as Array<{ id: string; content: string }>
|
||||
for (let offset = 0; offset < chunks.length; offset += 100) {
|
||||
const batch = chunks.slice(offset, offset + 100)
|
||||
try {
|
||||
const resp = await fetch('https://api.openai.com/v1/embeddings', {
|
||||
const response = await fetch('https://api.openai.com/v1/embeddings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${openaiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'text-embedding-3-small',
|
||||
input: batch.map((c) => c.content),
|
||||
dimensions: 1536
|
||||
})
|
||||
input: batch.map((chunk) => chunk.content),
|
||||
dimensions: EMBEDDING_DIMENSIONS,
|
||||
}),
|
||||
signal: AbortSignal.timeout(PROVIDER_TIMEOUT_MS),
|
||||
})
|
||||
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text()
|
||||
errors.push(`OpenAI ${resp.status}: ${errText.slice(0, 200)}`)
|
||||
if (!response.ok) {
|
||||
failed += batch.length
|
||||
continue
|
||||
}
|
||||
|
||||
const data = (await resp.json()) as OpenAIEmbeddingResponse
|
||||
const payload = await response.json().catch(() => null) as OpenAIEmbeddingResponse | null
|
||||
if (!payload || !Array.isArray(payload.data) || payload.data.length !== batch.length) {
|
||||
failed += batch.length
|
||||
continue
|
||||
}
|
||||
|
||||
// 각 청크에 embedding 업데이트
|
||||
for (const item of data.data) {
|
||||
const chunk = batch[item.index]
|
||||
const { error: updateErr } = await serviceClient
|
||||
const received = new Set<number>()
|
||||
let batchSuccess = 0
|
||||
for (const item of payload.data) {
|
||||
if (
|
||||
typeof item.index !== 'number'
|
||||
|| !Number.isInteger(item.index)
|
||||
|| item.index < 0
|
||||
|| item.index >= batch.length
|
||||
|| received.has(item.index)
|
||||
|| !validEmbedding(item.embedding)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
received.add(item.index)
|
||||
const update = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.update({ embedding: item.embedding })
|
||||
.eq('id', chunk.id)
|
||||
|
||||
if (updateErr) {
|
||||
errors.push(`chunk ${chunk.id}: ${updateErr.message}`)
|
||||
} else {
|
||||
embedded++
|
||||
.eq('id', batch[item.index].id)
|
||||
.eq('document_id', body.document_id)
|
||||
if (!update.error) {
|
||||
embedded += 1
|
||||
batchSuccess += 1
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
errors.push(e instanceof Error ? e.message : String(e))
|
||||
failed += batch.length - batchSuccess
|
||||
} catch {
|
||||
failed += batch.length
|
||||
}
|
||||
}
|
||||
|
||||
// 문서 indexed 플래그 업데이트
|
||||
await serviceClient
|
||||
const remainingResult = await serviceClient
|
||||
.from('knowledge_chunks')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', body.document_id)
|
||||
.is('embedding', null)
|
||||
if (remainingResult.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
const remaining = remainingResult.count ?? total
|
||||
if (failed > 0 || remaining > 0) {
|
||||
const rollback = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', body.document_id)
|
||||
.eq('user_id', user.id)
|
||||
if (rollback.error) return json(500, { error: 'knowledge_storage_failed' })
|
||||
return json(502, {
|
||||
error: 'embedding_failed',
|
||||
embedded,
|
||||
remaining,
|
||||
})
|
||||
}
|
||||
|
||||
const indexed = await serviceClient
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: true, indexed_at: new Date().toISOString() })
|
||||
.eq('id', body.document_id)
|
||||
.eq('user_id', user.id)
|
||||
.select('id')
|
||||
.maybeSingle()
|
||||
if (indexed.error || !indexed.data) return json(500, { error: 'knowledge_storage_failed' })
|
||||
|
||||
return new Response(JSON.stringify({ embedded, errors }), {
|
||||
status: 200,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' }
|
||||
})
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'status' in err && 'message' in err) {
|
||||
return authErrorResponse(err as AuthError, corsHeaders)
|
||||
return json(200, { embedded, total, indexed: true })
|
||||
} catch (error) {
|
||||
if (
|
||||
error
|
||||
&& typeof error === 'object'
|
||||
&& 'status' in error
|
||||
&& (error.status === 401 || error.status === 403)
|
||||
&& 'message' in error
|
||||
&& typeof error.message === 'string'
|
||||
) {
|
||||
return authErrorResponse(error as AuthError, corsHeaders)
|
||||
}
|
||||
const message = err instanceof Error ? err.message : 'Unknown error'
|
||||
return new Response(JSON.stringify({ error: message }), {
|
||||
status: 500,
|
||||
headers: { ...corsHeaders, 'Content-Type': 'application/json' }
|
||||
})
|
||||
return json(500, { error: 'internal_error' })
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue