feat(release): prepare 1.1.0 candidate
This commit is contained in:
parent
5a34f66981
commit
5205dcdfa9
736 changed files with 115667 additions and 12203 deletions
183
apps/mobile-rn/src/features/knowledge/knowledge-file-picker.ts
Normal file
183
apps/mobile-rn/src/features/knowledge/knowledge-file-picker.ts
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
import {
|
||||
errorCodes,
|
||||
isErrorWithCode,
|
||||
keepLocalCopy,
|
||||
pick,
|
||||
types,
|
||||
} from '@react-native-documents/picker'
|
||||
import { Dirs, FileSystem } from 'react-native-file-access'
|
||||
import {
|
||||
KnowledgeServiceError,
|
||||
KNOWLEDGE_MAX_CONTENT_CHARS,
|
||||
normalizeKnowledgeContent,
|
||||
type MobileKnowledgeFileType,
|
||||
} from './knowledge-service'
|
||||
|
||||
export const KNOWLEDGE_MAX_FILE_BYTES = 1_048_576
|
||||
|
||||
export interface PickedKnowledgeFile {
|
||||
fileName: string
|
||||
fileType: MobileKnowledgeFileType
|
||||
title: string
|
||||
content: string
|
||||
dispose: () => Promise<void>
|
||||
}
|
||||
|
||||
const TEXT_MIME_TYPES = new Set([
|
||||
'text/plain',
|
||||
'text/markdown',
|
||||
'text/x-markdown',
|
||||
'application/octet-stream',
|
||||
])
|
||||
|
||||
function localPathFromUri(uri: string): string {
|
||||
if (!uri.startsWith('file://')) return uri
|
||||
const withoutScheme = uri.replace(/^file:\/\//, '')
|
||||
return decodeURIComponent(withoutScheme.startsWith('/') ? withoutScheme : `/${withoutScheme}`)
|
||||
}
|
||||
|
||||
function isOwnedCachePath(path: string): boolean {
|
||||
const normalizedRoot = Dirs.CacheDir.replace(/\\/g, '/').replace(/\/$/, '')
|
||||
const normalizedPath = path.replace(/\\/g, '/')
|
||||
return normalizedPath.startsWith(`${normalizedRoot}/`)
|
||||
&& !normalizedPath.slice(normalizedRoot.length + 1).split('/').includes('..')
|
||||
}
|
||||
|
||||
async function removeOwnedCopy(path: string): Promise<void> {
|
||||
if (!isOwnedCachePath(path)) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
'Refusing to remove a file outside the app import cache',
|
||||
)
|
||||
}
|
||||
if (await FileSystem.exists(path)) await FileSystem.unlink(path)
|
||||
}
|
||||
|
||||
export function classifyKnowledgeFile(
|
||||
fileNameValue: string | null,
|
||||
mimeTypeValue: string | null,
|
||||
): { fileName: string; fileType: MobileKnowledgeFileType; title: string } {
|
||||
const fileName = (fileNameValue ?? '').trim()
|
||||
const mimeType = (mimeTypeValue ?? '').trim().toLowerCase()
|
||||
const lowerName = fileName.toLowerCase()
|
||||
let fileType: MobileKnowledgeFileType | null = null
|
||||
|
||||
if (lowerName.endsWith('.txt')) fileType = 'txt'
|
||||
if (lowerName.endsWith('.md') || lowerName.endsWith('.markdown')) fileType = 'md'
|
||||
if (fileType === null && fileName.length === 0 && mimeType === 'text/plain') fileType = 'txt'
|
||||
if (fileType === null || (mimeType.length > 0 && !TEXT_MIME_TYPES.has(mimeType))) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
'Only UTF-8 .txt and .md files are supported',
|
||||
)
|
||||
}
|
||||
|
||||
const resolvedName = fileName.length > 0 ? fileName : 'knowledge.txt'
|
||||
if (resolvedName.length > 255) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Knowledge file name is too long')
|
||||
}
|
||||
const title = resolvedName.replace(/\.(?:txt|md|markdown)$/i, '').trim()
|
||||
if (title.length === 0) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Knowledge file name has no title')
|
||||
}
|
||||
return { fileName: resolvedName, fileType, title }
|
||||
}
|
||||
|
||||
export function validatePickedKnowledgeContent(value: string): string {
|
||||
const content = normalizeKnowledgeContent(value)
|
||||
const replacementCount = [...content].filter((character) => character === '\ufffd').length
|
||||
if (replacementCount > Math.max(3, Math.floor(content.length * 0.01))) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
'The selected file is not valid UTF-8 text',
|
||||
)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
export async function pickKnowledgeTextFile(): Promise<PickedKnowledgeFile> {
|
||||
try {
|
||||
const [selected] = await pick({
|
||||
type: [types.plainText, 'text/markdown', 'text/x-markdown'],
|
||||
allowMultiSelection: false,
|
||||
allowVirtualFiles: false,
|
||||
mode: 'import',
|
||||
presentationStyle: 'fullScreen',
|
||||
})
|
||||
if (selected.error !== null) {
|
||||
throw new KnowledgeServiceError('validation', true, null, 'The file provider returned invalid metadata')
|
||||
}
|
||||
if (selected.isVirtual === true) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Virtual documents are not supported')
|
||||
}
|
||||
if (
|
||||
selected.size !== null
|
||||
&& (!Number.isSafeInteger(selected.size) || selected.size <= 0 || selected.size > KNOWLEDGE_MAX_FILE_BYTES)
|
||||
) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
`Knowledge files must be between 1 and ${KNOWLEDGE_MAX_FILE_BYTES} bytes`,
|
||||
)
|
||||
}
|
||||
const metadata = classifyKnowledgeFile(selected.name, selected.type)
|
||||
const [copy] = await keepLocalCopy({
|
||||
files: [{ uri: selected.uri, fileName: metadata.fileName }],
|
||||
destination: 'cachesDirectory',
|
||||
})
|
||||
if (copy.status !== 'success') {
|
||||
throw new KnowledgeServiceError('validation', true, null, 'The selected file could not be copied')
|
||||
}
|
||||
const path = localPathFromUri(copy.localUri)
|
||||
const dispose = async (): Promise<void> => removeOwnedCopy(path)
|
||||
|
||||
try {
|
||||
const stat = await FileSystem.stat(path)
|
||||
if (
|
||||
!Number.isSafeInteger(stat.size)
|
||||
|| stat.size <= 0
|
||||
|| stat.size > KNOWLEDGE_MAX_FILE_BYTES
|
||||
) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
`Knowledge files must be between 1 and ${KNOWLEDGE_MAX_FILE_BYTES} bytes`,
|
||||
)
|
||||
}
|
||||
const raw = await FileSystem.readFile(path, 'utf8')
|
||||
if (raw.length > KNOWLEDGE_MAX_CONTENT_CHARS + 1) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Knowledge text is too large')
|
||||
}
|
||||
return {
|
||||
...metadata,
|
||||
content: validatePickedKnowledgeContent(raw),
|
||||
dispose,
|
||||
}
|
||||
} catch (error) {
|
||||
await dispose().catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeServiceError) throw error
|
||||
if (isErrorWithCode(error)) {
|
||||
if (error.code === errorCodes.OPERATION_CANCELED) {
|
||||
throw new KnowledgeServiceError('cancelled', false, null, 'File selection was cancelled')
|
||||
}
|
||||
if (error.code === errorCodes.IN_PROGRESS) {
|
||||
throw new KnowledgeServiceError('validation', true, null, 'A file picker is already open')
|
||||
}
|
||||
if (error.code === errorCodes.UNABLE_TO_OPEN_FILE_TYPE) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'This provider cannot export a UTF-8 text file')
|
||||
}
|
||||
}
|
||||
throw new KnowledgeServiceError('validation', true, null, 'The knowledge file picker could not be opened')
|
||||
}
|
||||
}
|
||||
638
apps/mobile-rn/src/features/knowledge/knowledge-service.ts
Normal file
638
apps/mobile-rn/src/features/knowledge/knowledge-service.ts
Normal file
|
|
@ -0,0 +1,638 @@
|
|||
import type { RealtimeChannel } from '@supabase/supabase-js'
|
||||
import type {
|
||||
D3roSupabaseClient,
|
||||
KnowledgeDocument,
|
||||
KnowledgeFileType,
|
||||
} from '@d3ro/api-client'
|
||||
import { SUPABASE_URL } from '@d3ro/core/supabase-config'
|
||||
import { supabase } from '../../lib/supabase'
|
||||
import { createUuidV4 } from '../../lib/random-id'
|
||||
|
||||
export type MobileKnowledgeFileType = Extract<KnowledgeFileType, 'txt' | 'md'>
|
||||
|
||||
export type KnowledgeServiceErrorCode =
|
||||
| 'auth'
|
||||
| 'cancelled'
|
||||
| 'conflict'
|
||||
| 'forbidden'
|
||||
| 'index-failed'
|
||||
| 'index-unavailable'
|
||||
| 'invalid-response'
|
||||
| 'network'
|
||||
| 'not-found'
|
||||
| 'server'
|
||||
| 'timeout'
|
||||
| 'validation'
|
||||
|
||||
export class KnowledgeServiceError extends Error {
|
||||
constructor(
|
||||
public readonly code: KnowledgeServiceErrorCode,
|
||||
public readonly retryable: boolean,
|
||||
public readonly status: number | null = null,
|
||||
message: string = code,
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'KnowledgeServiceError'
|
||||
}
|
||||
}
|
||||
|
||||
export interface KnowledgeListOptions {
|
||||
userId: string
|
||||
search?: string
|
||||
page?: number
|
||||
pageSize?: number
|
||||
}
|
||||
|
||||
export interface KnowledgeListResult {
|
||||
documents: KnowledgeDocument[]
|
||||
total: number
|
||||
page: number
|
||||
pageSize: number
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
export interface CreateKnowledgeDocumentInput {
|
||||
userId: string
|
||||
title: string
|
||||
fileName: string
|
||||
fileType: MobileKnowledgeFileType
|
||||
content: string
|
||||
}
|
||||
|
||||
export interface KnowledgeSearchResult {
|
||||
id: string
|
||||
documentId: string
|
||||
chunkIndex: number
|
||||
content: string
|
||||
similarity: number
|
||||
}
|
||||
|
||||
export interface KnowledgeDocumentSubscription {
|
||||
unsubscribe: () => Promise<void>
|
||||
}
|
||||
|
||||
interface KnowledgeSearchOptions {
|
||||
accessToken: string
|
||||
query: string
|
||||
count?: number
|
||||
signal?: AbortSignal
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
interface KnowledgeIndexOptions {
|
||||
accessToken: string
|
||||
userId: string
|
||||
document: KnowledgeDocument
|
||||
signal?: AbortSignal
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
||||
const DEFAULT_PAGE_SIZE = 30
|
||||
const MAX_PAGE_SIZE = 50
|
||||
const MAX_TITLE_LENGTH = 160
|
||||
export const KNOWLEDGE_MAX_CONTENT_CHARS = 250_000
|
||||
export const KNOWLEDGE_CHUNK_SIZE = 800
|
||||
const MAX_SEARCH_LENGTH = 500
|
||||
const MAX_SEARCH_RESULTS = 12
|
||||
|
||||
function client(): D3roSupabaseClient {
|
||||
return supabase as unknown as D3roSupabaseClient
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function requireUuid(value: string, label: string): void {
|
||||
if (!UUID_PATTERN.test(value)) {
|
||||
throw new KnowledgeServiceError('auth', false, null, `A valid ${label} is required`)
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeTitle(value: string): string {
|
||||
const title = value.trim().replace(/\s+/g, ' ')
|
||||
if (title.length === 0 || title.length > MAX_TITLE_LENGTH) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
`Knowledge titles must contain 1-${MAX_TITLE_LENGTH} characters`,
|
||||
)
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
export function sanitizeKnowledgeSearch(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.slice(0, MAX_SEARCH_LENGTH)
|
||||
.replace(/[\\%_,()]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function normalizeKnowledgeContent(value: string): string {
|
||||
const withoutBom = value.charCodeAt(0) === 0xfeff ? value.slice(1) : value
|
||||
const content = withoutBom.trim()
|
||||
if (
|
||||
content.length === 0
|
||||
|| content.length > KNOWLEDGE_MAX_CONTENT_CHARS
|
||||
|| content.includes('\u0000')
|
||||
) {
|
||||
throw new KnowledgeServiceError(
|
||||
'validation',
|
||||
false,
|
||||
null,
|
||||
`Knowledge content must contain 1-${KNOWLEDGE_MAX_CONTENT_CHARS} text characters`,
|
||||
)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
export function chunkKnowledgeText(value: string): string[] {
|
||||
const content = normalizeKnowledgeContent(value)
|
||||
const chunks: string[] = []
|
||||
let offset = 0
|
||||
|
||||
while (offset < content.length) {
|
||||
const hardEnd = Math.min(offset + KNOWLEDGE_CHUNK_SIZE, content.length)
|
||||
let end = hardEnd
|
||||
if (hardEnd < content.length) {
|
||||
const candidate = content.lastIndexOf('\n', hardEnd)
|
||||
if (candidate > offset + Math.floor(KNOWLEDGE_CHUNK_SIZE * 0.6)) end = candidate
|
||||
}
|
||||
const chunk = content.slice(offset, end).trim()
|
||||
if (chunk.length > 0) chunks.push(chunk)
|
||||
offset = end
|
||||
while (offset < content.length && /\s/.test(content[offset])) offset += 1
|
||||
}
|
||||
|
||||
if (chunks.length === 0) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Knowledge content is empty')
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
function assertDocument(row: KnowledgeDocument | null, expectedId?: string): KnowledgeDocument {
|
||||
if (row === null) {
|
||||
throw new KnowledgeServiceError('not-found', false, 404, 'Knowledge document was not found')
|
||||
}
|
||||
requireUuid(row.id, 'knowledge document id')
|
||||
requireUuid(row.user_id, 'knowledge owner id')
|
||||
if (expectedId !== undefined && row.id !== expectedId) {
|
||||
throw new KnowledgeServiceError('invalid-response', true, null, 'Server returned a different document')
|
||||
}
|
||||
if (
|
||||
typeof row.title !== 'string'
|
||||
|| row.title.trim().length === 0
|
||||
|| row.title.length > 500
|
||||
|| (row.file_name !== null && (typeof row.file_name !== 'string' || row.file_name.length > 255))
|
||||
|| typeof row.indexed !== 'boolean'
|
||||
|| !Number.isSafeInteger(row.chunk_count)
|
||||
|| row.chunk_count < 0
|
||||
|| !Number.isFinite(Date.parse(row.created_at))
|
||||
|| !Number.isFinite(Date.parse(row.updated_at))
|
||||
) {
|
||||
throw new KnowledgeServiceError('invalid-response', true, null, 'Server returned an invalid document')
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
function toKnowledgeError(error: unknown): KnowledgeServiceError {
|
||||
if (error instanceof KnowledgeServiceError) return error
|
||||
|
||||
const candidate = error as { code?: unknown; message?: unknown }
|
||||
const code = typeof candidate?.code === 'string' ? candidate.code : ''
|
||||
const message = typeof candidate?.message === 'string'
|
||||
? candidate.message
|
||||
: 'Knowledge request failed'
|
||||
const lower = message.toLowerCase()
|
||||
|
||||
if (
|
||||
error instanceof TypeError
|
||||
|| lower.includes('network request failed')
|
||||
|| lower.includes('failed to fetch')
|
||||
|| lower.includes('networkerror')
|
||||
) {
|
||||
return new KnowledgeServiceError('network', true, null, message)
|
||||
}
|
||||
if (code === '42501' || lower.includes('permission denied') || lower.includes('row-level security')) {
|
||||
return new KnowledgeServiceError('forbidden', false, 403, message)
|
||||
}
|
||||
if (code === 'PGRST301' || lower.includes('jwt')) {
|
||||
return new KnowledgeServiceError('auth', false, 401, message)
|
||||
}
|
||||
if (code.startsWith('22')) {
|
||||
return new KnowledgeServiceError('validation', false, 400, message)
|
||||
}
|
||||
return new KnowledgeServiceError('server', true, null, message)
|
||||
}
|
||||
|
||||
function createRequestDeadline(
|
||||
externalSignal: AbortSignal | undefined,
|
||||
timeoutMs: number,
|
||||
): { signal: AbortSignal; dispose: () => void; didTimeout: () => boolean } {
|
||||
const controller = new AbortController()
|
||||
let timedOut = false
|
||||
const abortFromCaller = (): void => controller.abort()
|
||||
if (externalSignal?.aborted === true) controller.abort()
|
||||
else externalSignal?.addEventListener('abort', abortFromCaller, { once: true })
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true
|
||||
controller.abort()
|
||||
}, timeoutMs)
|
||||
return {
|
||||
signal: controller.signal,
|
||||
didTimeout: () => timedOut,
|
||||
dispose: () => {
|
||||
clearTimeout(timer)
|
||||
externalSignal?.removeEventListener('abort', abortFromCaller)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response: Response): Promise<unknown> {
|
||||
return response.json().catch(() => null)
|
||||
}
|
||||
|
||||
function edgeErrorCode(status: number, body: unknown): KnowledgeServiceErrorCode {
|
||||
const serverCode = isRecord(body) && typeof body.error === 'string' ? body.error : ''
|
||||
if (status === 401) return 'auth'
|
||||
if (status === 400) return 'validation'
|
||||
if (status === 403) return 'forbidden'
|
||||
if (status === 404) return 'not-found'
|
||||
if (
|
||||
status === 503
|
||||
&& (serverCode === 'embedding_provider_unavailable'
|
||||
|| serverCode === 'knowledge_storage_unavailable')
|
||||
) return 'index-unavailable'
|
||||
if (
|
||||
status === 502
|
||||
&& (serverCode === 'embedding_failed'
|
||||
|| serverCode === 'embedding_upstream_failed'
|
||||
|| serverCode === 'embedding_response_invalid')
|
||||
) return 'index-failed'
|
||||
if (status === 504) return 'timeout'
|
||||
return 'server'
|
||||
}
|
||||
|
||||
export async function listKnowledgeDocuments(
|
||||
options: KnowledgeListOptions,
|
||||
): Promise<KnowledgeListResult> {
|
||||
requireUuid(options.userId, 'authenticated user')
|
||||
const page = Math.max(0, Math.floor(options.page ?? 0))
|
||||
const pageSize = Math.max(1, Math.min(Math.floor(options.pageSize ?? DEFAULT_PAGE_SIZE), MAX_PAGE_SIZE))
|
||||
const search = sanitizeKnowledgeSearch(options.search ?? '')
|
||||
|
||||
try {
|
||||
let query = client()
|
||||
.from('knowledge_documents')
|
||||
.select('*', { count: 'exact' })
|
||||
.order('created_at', { ascending: false })
|
||||
.order('id', { ascending: false })
|
||||
.range(page * pageSize, (page + 1) * pageSize - 1)
|
||||
|
||||
if (search.length > 0) query = query.ilike('title', `%${search}%`)
|
||||
const { data, error, count } = await query
|
||||
if (error !== null) throw error
|
||||
const documents = (data ?? []).map((row) => assertDocument(row))
|
||||
const total = count ?? documents.length
|
||||
return {
|
||||
documents,
|
||||
total,
|
||||
page,
|
||||
pageSize,
|
||||
hasMore: (page + 1) * pageSize < total,
|
||||
}
|
||||
} catch (error) {
|
||||
throw toKnowledgeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
export function mergeKnowledgeDocuments(
|
||||
current: KnowledgeDocument[],
|
||||
incoming: KnowledgeDocument[],
|
||||
): KnowledgeDocument[] {
|
||||
const merged = new Map(current.map((document) => [document.id, document]))
|
||||
for (const document of incoming) {
|
||||
const existing = merged.get(document.id)
|
||||
if (existing === undefined || document.updated_at >= existing.updated_at) {
|
||||
merged.set(document.id, document)
|
||||
}
|
||||
}
|
||||
return [...merged.values()].sort((left, right) => {
|
||||
const byCreated = right.created_at.localeCompare(left.created_at)
|
||||
return byCreated !== 0 ? byCreated : right.id.localeCompare(left.id)
|
||||
})
|
||||
}
|
||||
|
||||
export async function createKnowledgeDocument(
|
||||
input: CreateKnowledgeDocumentInput,
|
||||
): Promise<KnowledgeDocument> {
|
||||
requireUuid(input.userId, 'authenticated user')
|
||||
const title = normalizeKnowledgeTitle(input.title)
|
||||
const fileName = input.fileName.trim()
|
||||
if (fileName.length === 0 || fileName.length > 255 || !['txt', 'md'].includes(input.fileType)) {
|
||||
throw new KnowledgeServiceError('validation', false, null, 'Knowledge file metadata is invalid')
|
||||
}
|
||||
const chunks = chunkKnowledgeText(input.content)
|
||||
let created: KnowledgeDocument | null = null
|
||||
|
||||
try {
|
||||
const documentResult = await client()
|
||||
.from('knowledge_documents')
|
||||
.insert({
|
||||
user_id: input.userId,
|
||||
title,
|
||||
file_name: fileName,
|
||||
file_type: input.fileType,
|
||||
storage_key: null,
|
||||
chunk_count: chunks.length,
|
||||
indexed: false,
|
||||
indexed_at: null,
|
||||
})
|
||||
.select('*')
|
||||
.single()
|
||||
if (documentResult.error !== null) throw documentResult.error
|
||||
created = assertDocument(documentResult.data)
|
||||
|
||||
const chunkResult = await client()
|
||||
.from('knowledge_chunks')
|
||||
.insert(chunks.map((content, chunkIndex) => ({
|
||||
document_id: created?.id ?? '',
|
||||
chunk_index: chunkIndex,
|
||||
content,
|
||||
})))
|
||||
.select('id')
|
||||
if (chunkResult.error !== null) throw chunkResult.error
|
||||
if ((chunkResult.data ?? []).length !== chunks.length) {
|
||||
throw new KnowledgeServiceError(
|
||||
'invalid-response',
|
||||
true,
|
||||
null,
|
||||
'Server did not confirm every knowledge chunk',
|
||||
)
|
||||
}
|
||||
return created
|
||||
} catch (error) {
|
||||
if (created !== null) {
|
||||
const cleanup = await client()
|
||||
.from('knowledge_documents')
|
||||
.delete()
|
||||
.eq('id', created.id)
|
||||
.eq('user_id', input.userId)
|
||||
if (cleanup.error !== null) {
|
||||
throw new KnowledgeServiceError(
|
||||
'server',
|
||||
true,
|
||||
null,
|
||||
'Knowledge import failed and its incomplete document could not be removed',
|
||||
)
|
||||
}
|
||||
}
|
||||
throw toKnowledgeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async function forcePendingIndexState(
|
||||
userId: string,
|
||||
documentId: string,
|
||||
): Promise<void> {
|
||||
await client()
|
||||
.from('knowledge_documents')
|
||||
.update({ indexed: false, indexed_at: null })
|
||||
.eq('id', documentId)
|
||||
.eq('user_id', userId)
|
||||
}
|
||||
|
||||
export async function indexKnowledgeDocument(
|
||||
options: KnowledgeIndexOptions,
|
||||
): Promise<KnowledgeDocument> {
|
||||
requireUuid(options.userId, 'authenticated user')
|
||||
const document = assertDocument(options.document)
|
||||
if (document.user_id !== options.userId) {
|
||||
throw new KnowledgeServiceError('forbidden', false, 403, 'Only the document owner can index it')
|
||||
}
|
||||
const token = options.accessToken.trim()
|
||||
if (token.length === 0) throw new KnowledgeServiceError('auth', false, 401)
|
||||
const deadline = createRequestDeadline(options.signal, options.timeoutMs ?? 90_000)
|
||||
|
||||
try {
|
||||
const response = await fetch(`${SUPABASE_URL}/functions/v1/embed-chunks`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ document_id: document.id }),
|
||||
signal: deadline.signal,
|
||||
})
|
||||
const body = await responseJson(response)
|
||||
if (!response.ok) {
|
||||
const code = edgeErrorCode(response.status, body)
|
||||
throw new KnowledgeServiceError(
|
||||
code,
|
||||
code === 'index-unavailable' || code === 'timeout' || code === 'server',
|
||||
response.status,
|
||||
)
|
||||
}
|
||||
if (
|
||||
!isRecord(body)
|
||||
|| !Number.isSafeInteger(body.embedded)
|
||||
|| Number(body.embedded) < 0
|
||||
|| !Number.isSafeInteger(body.total)
|
||||
|| Number(body.total) !== document.chunk_count
|
||||
|| body.indexed !== true
|
||||
) {
|
||||
throw new KnowledgeServiceError(
|
||||
'invalid-response',
|
||||
true,
|
||||
response.status,
|
||||
'Embedding service did not confirm a complete index',
|
||||
)
|
||||
}
|
||||
|
||||
const [documentResult, embeddedCountResult] = await Promise.all([
|
||||
client()
|
||||
.from('knowledge_documents')
|
||||
.select('*')
|
||||
.eq('id', document.id)
|
||||
.maybeSingle(),
|
||||
client()
|
||||
.from('knowledge_chunks')
|
||||
.select('id', { count: 'exact', head: true })
|
||||
.eq('document_id', document.id)
|
||||
.not('embedding', 'is', null),
|
||||
])
|
||||
if (documentResult.error !== null) throw documentResult.error
|
||||
if (embeddedCountResult.error !== null) throw embeddedCountResult.error
|
||||
const updated = assertDocument(documentResult.data, document.id)
|
||||
if (updated.indexed !== true || embeddedCountResult.count !== updated.chunk_count) {
|
||||
throw new KnowledgeServiceError(
|
||||
'invalid-response',
|
||||
true,
|
||||
response.status,
|
||||
'Server marked an incomplete knowledge index as ready',
|
||||
)
|
||||
}
|
||||
return updated
|
||||
} catch (error) {
|
||||
await forcePendingIndexState(options.userId, document.id).catch(() => undefined)
|
||||
if (error instanceof KnowledgeServiceError) throw error
|
||||
if (deadline.signal.aborted) {
|
||||
const callerCancelled = options.signal?.aborted === true && !deadline.didTimeout()
|
||||
throw new KnowledgeServiceError(
|
||||
callerCancelled ? 'cancelled' : 'timeout',
|
||||
!callerCancelled,
|
||||
callerCancelled ? null : 504,
|
||||
)
|
||||
}
|
||||
throw toKnowledgeError(error)
|
||||
} finally {
|
||||
deadline.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteKnowledgeDocument(
|
||||
userId: string,
|
||||
document: KnowledgeDocument,
|
||||
): Promise<void> {
|
||||
requireUuid(userId, 'authenticated user')
|
||||
assertDocument(document)
|
||||
if (document.user_id !== userId) {
|
||||
throw new KnowledgeServiceError('forbidden', false, 403, 'Only the owner can delete this document')
|
||||
}
|
||||
try {
|
||||
const { data, error } = await client()
|
||||
.from('knowledge_documents')
|
||||
.delete()
|
||||
.eq('id', document.id)
|
||||
.eq('user_id', userId)
|
||||
.eq('updated_at', document.updated_at)
|
||||
.select('id')
|
||||
.maybeSingle()
|
||||
if (error !== null) throw error
|
||||
if (data === null) {
|
||||
throw new KnowledgeServiceError(
|
||||
'conflict',
|
||||
true,
|
||||
409,
|
||||
'Knowledge document changed or was removed on another device',
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
throw toKnowledgeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
function parseSearchResult(value: unknown): KnowledgeSearchResult {
|
||||
if (!isRecord(value)) throw new KnowledgeServiceError('invalid-response', true)
|
||||
const id = typeof value.id === 'string' ? value.id : ''
|
||||
const documentId = typeof value.document_id === 'string' ? value.document_id : ''
|
||||
const content = typeof value.content === 'string' ? value.content.trim() : ''
|
||||
const chunkIndex = value.chunk_index
|
||||
const similarity = value.similarity
|
||||
if (
|
||||
!UUID_PATTERN.test(id)
|
||||
|| !UUID_PATTERN.test(documentId)
|
||||
|| content.length === 0
|
||||
|| content.length > 8_000
|
||||
|| !Number.isSafeInteger(chunkIndex)
|
||||
|| Number(chunkIndex) < 0
|
||||
|| typeof similarity !== 'number'
|
||||
|| !Number.isFinite(similarity)
|
||||
|| similarity < 0
|
||||
|| similarity > 1
|
||||
) {
|
||||
throw new KnowledgeServiceError('invalid-response', true)
|
||||
}
|
||||
return {
|
||||
id,
|
||||
documentId,
|
||||
chunkIndex: Number(chunkIndex),
|
||||
content,
|
||||
similarity,
|
||||
}
|
||||
}
|
||||
|
||||
export async function searchKnowledge(
|
||||
options: KnowledgeSearchOptions,
|
||||
): Promise<KnowledgeSearchResult[]> {
|
||||
const token = options.accessToken.trim()
|
||||
if (token.length === 0) throw new KnowledgeServiceError('auth', false, 401)
|
||||
const query = options.query.trim()
|
||||
if (query.length === 0 || query.length > MAX_SEARCH_LENGTH) {
|
||||
throw new KnowledgeServiceError('validation', false, 400)
|
||||
}
|
||||
const count = Math.max(1, Math.min(Math.floor(options.count ?? 8), MAX_SEARCH_RESULTS))
|
||||
const deadline = createRequestDeadline(options.signal, options.timeoutMs ?? 45_000)
|
||||
|
||||
try {
|
||||
const response = await fetch(`${SUPABASE_URL}/functions/v1/search-knowledge`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ query, count }),
|
||||
signal: deadline.signal,
|
||||
})
|
||||
const body = await responseJson(response)
|
||||
if (!response.ok) {
|
||||
const code = edgeErrorCode(response.status, body)
|
||||
throw new KnowledgeServiceError(
|
||||
code,
|
||||
code === 'index-unavailable' || code === 'timeout' || code === 'server',
|
||||
response.status,
|
||||
)
|
||||
}
|
||||
if (!isRecord(body) || !Array.isArray(body.results) || body.results.length > count) {
|
||||
throw new KnowledgeServiceError('invalid-response', true, response.status)
|
||||
}
|
||||
return body.results.map(parseSearchResult)
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeServiceError) throw error
|
||||
if (deadline.signal.aborted) {
|
||||
const callerCancelled = options.signal?.aborted === true && !deadline.didTimeout()
|
||||
throw new KnowledgeServiceError(
|
||||
callerCancelled ? 'cancelled' : 'timeout',
|
||||
!callerCancelled,
|
||||
callerCancelled ? null : 504,
|
||||
)
|
||||
}
|
||||
throw toKnowledgeError(error)
|
||||
} finally {
|
||||
deadline.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export function subscribeToKnowledgeDocuments(
|
||||
onChanged: () => void,
|
||||
onStatus: (connected: boolean) => void,
|
||||
): KnowledgeDocumentSubscription {
|
||||
const channel: RealtimeChannel = supabase
|
||||
.channel(`mobile-knowledge-${createUuidV4()}`)
|
||||
.on('postgres_changes', {
|
||||
event: 'INSERT',
|
||||
schema: 'public',
|
||||
table: 'knowledge_documents',
|
||||
}, onChanged)
|
||||
.on('postgres_changes', {
|
||||
event: 'UPDATE',
|
||||
schema: 'public',
|
||||
table: 'knowledge_documents',
|
||||
}, onChanged)
|
||||
.on('postgres_changes', {
|
||||
event: 'DELETE',
|
||||
schema: 'public',
|
||||
table: 'knowledge_documents',
|
||||
}, onChanged)
|
||||
.subscribe((status) => onStatus(status === 'SUBSCRIBED'))
|
||||
|
||||
return {
|
||||
unsubscribe: async () => {
|
||||
await supabase.removeChannel(channel)
|
||||
},
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue