227 lines
7.7 KiB
TypeScript
227 lines
7.7 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
import { eq } from 'drizzle-orm'
|
|
import { bindTestDatabase, unbindTestDatabase } from '../../../src/main/db'
|
|
import { ragChunks } from '../../../src/main/db/schema'
|
|
import {
|
|
decodeFloat32Buffer,
|
|
invalidateEmbeddedChunksCache,
|
|
SqliteChunkStore,
|
|
} from '../../../src/main/services/rag/chunk-store'
|
|
import { rankChunks } from '../../../src/main/services/rag/retrieval'
|
|
import { resetRAGServiceForTests } from '../../../src/main/services/RAGService'
|
|
import { adapterFor } from '../../../src/main/services/sync/sync-adapters'
|
|
import { createTestDb } from '../../helpers/createTestDb'
|
|
|
|
describe('SqliteChunkStore cache optimization, invalidation & hardening', () => {
|
|
let closeDb: (() => void) | null = null
|
|
|
|
function setupDb() {
|
|
const testDb = createTestDb()
|
|
closeDb = testDb.close
|
|
bindTestDatabase(testDb.db)
|
|
return testDb
|
|
}
|
|
|
|
beforeEach(() => {
|
|
unbindTestDatabase()
|
|
resetRAGServiceForTests()
|
|
})
|
|
|
|
afterEach(() => {
|
|
resetRAGServiceForTests()
|
|
unbindTestDatabase()
|
|
if (closeDb) {
|
|
closeDb()
|
|
closeDb = null
|
|
}
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
it('listEmbeddedChunks_decodesFloat32Vector_stripsRedundantJsonStringFromCache', () => {
|
|
setupDb()
|
|
const store = new SqliteChunkStore()
|
|
store.insertDocument({
|
|
id: 'doc-1',
|
|
fileName: 'guide.md',
|
|
filePath: 'C:/docs/guide.md',
|
|
fileType: 'md',
|
|
chunkCount: 2,
|
|
indexed: true,
|
|
indexedAt: 1_000,
|
|
addedAt: 1_000,
|
|
})
|
|
store.replaceChunks('doc-1', ['alpha content', 'beta content'])
|
|
const chunks = store.listChunks('doc-1')
|
|
store.setEmbedding(chunks[0].id, [1, 0, 0])
|
|
store.setEmbedding(chunks[1].id, [0, 1, 0])
|
|
|
|
const parseSpy = vi.spyOn(JSON, 'parse')
|
|
const cached = store.listEmbeddedChunks()
|
|
|
|
expect(cached).toHaveLength(2)
|
|
expect(cached.map((c) => c.embedding)).toEqual(['', ''])
|
|
expect(Array.from(cached[0].vector ?? [])).toEqual([1, 0, 0])
|
|
expect(Array.from(cached[1].vector ?? [])).toEqual([0, 1, 0])
|
|
|
|
const ranked = rankChunks([0, 0.9, 0.1], cached, new Map([['doc-1', 'guide.md']]), 1)
|
|
expect(ranked).toHaveLength(1)
|
|
expect(ranked[0].content).toBe('beta content')
|
|
expect(parseSpy).not.toHaveBeenCalled()
|
|
})
|
|
|
|
it('invalidateEmbeddedChunksCache_afterDirectDbDelete_clearsStaleCachedChunks', () => {
|
|
const { db } = setupDb()
|
|
const store = new SqliteChunkStore()
|
|
store.insertDocument({
|
|
id: 'doc-stale',
|
|
fileName: 'stale.txt',
|
|
filePath: 'C:/docs/stale.txt',
|
|
fileType: 'txt',
|
|
chunkCount: 1,
|
|
indexed: true,
|
|
indexedAt: 2_000,
|
|
addedAt: 2_000,
|
|
})
|
|
store.replaceChunks('doc-stale', ['to be deleted'])
|
|
const chunks = store.listChunks('doc-stale')
|
|
store.setEmbedding(chunks[0].id, [0.5, 0.5])
|
|
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
|
|
// Simulate out-of-band delete bypassing SqliteChunkStore methods
|
|
db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run()
|
|
// Before invalidation cache still serves stale entry
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
|
|
invalidateEmbeddedChunksCache(db)
|
|
expect(store.listEmbeddedChunks()).toEqual([])
|
|
|
|
// Also verify zero-arg invalidation clears currently open DB
|
|
store.replaceChunks('doc-stale', ['re-added'])
|
|
const nextChunks = store.listChunks('doc-stale')
|
|
store.setEmbedding(nextChunks[0].id, [1, 0])
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
|
|
db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run()
|
|
invalidateEmbeddedChunksCache()
|
|
expect(store.listEmbeddedChunks()).toEqual([])
|
|
})
|
|
|
|
it('knowledgeAdapter_deleteLocalAndApplyRemote_invalidatesEmbeddedChunksCache', async () => {
|
|
const { db } = setupDb()
|
|
const store = new SqliteChunkStore()
|
|
const docId = '11111111-1111-4111-8111-111111111111'
|
|
store.insertDocument({
|
|
id: docId,
|
|
fileName: 'sync.md',
|
|
filePath: 'remote://sync.md',
|
|
fileType: 'md',
|
|
chunkCount: 1,
|
|
indexed: true,
|
|
indexedAt: 3_000,
|
|
addedAt: 3_000,
|
|
})
|
|
store.replaceChunks(docId, ['synced chunk'])
|
|
const [chunk] = store.listChunks(docId)
|
|
store.setEmbedding(chunk.id, [1, 0])
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
|
|
// Even if RAGService uses an external custom store that mutates SQL directly,
|
|
// knowledgeAdapter.deleteLocal / applyRemote must invalidate SqliteChunkStore cache.
|
|
const spyStore: SqliteChunkStore = Object.assign(Object.create(SqliteChunkStore.prototype), {
|
|
removeDocument(id: string) {
|
|
db.delete(ragChunks).where(eq(ragChunks.documentId, id)).run()
|
|
return true
|
|
},
|
|
getDocument: () => null,
|
|
insertDocument: () => {},
|
|
replaceChunks: () => {},
|
|
listChunks: () => [],
|
|
updateDocument: () => {},
|
|
})
|
|
resetRAGServiceForTests({ store: spyStore })
|
|
|
|
const adapter = adapterFor('knowledge_documents')
|
|
expect(adapter === null).toBe(false)
|
|
expect(adapter!.deleteLocal(docId)).toBe(true)
|
|
expect(store.listEmbeddedChunks()).toEqual([])
|
|
|
|
// Populate again and verify applyRemote also invalidates cache
|
|
store.replaceChunks(docId, ['before remote apply'])
|
|
const [reChunk] = store.listChunks(docId)
|
|
store.setEmbedding(reChunk.id, [0, 1])
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
|
|
await adapter!.applyRemote(
|
|
{
|
|
id: docId,
|
|
file_name: 'sync.md',
|
|
file_type: 'md',
|
|
chunk_count: 1,
|
|
created_at: '2026-01-01T00:00:00.000Z',
|
|
updated_at: '2026-01-02T00:00:00.000Z',
|
|
},
|
|
{
|
|
userId: 'user-1',
|
|
isCurrent: () => true,
|
|
remote: {
|
|
selectChildren: vi.fn(async () => [{ chunk_index: 0, content: 'updated chunk' }]),
|
|
} as never,
|
|
},
|
|
)
|
|
// Cache was cleared so listEmbeddedChunks re-queries DB (still 1 here until re-indexed, selectSpy verifies DB query)
|
|
const selectSpy = vi.spyOn(db, 'select')
|
|
expect(store.listEmbeddedChunks()).toHaveLength(1)
|
|
expect(selectSpy).toHaveBeenCalledTimes(1)
|
|
})
|
|
|
|
it.each([
|
|
{ label: 'malformed json', json: '{bad-json' },
|
|
{ label: 'non-array primitive', json: '42' },
|
|
{ label: 'empty array', json: '[]' },
|
|
{ label: 'non-finite element', json: '[1, null, 3]' },
|
|
])('decodeFloat32Buffer_$label_returnsUndefinedWithoutThrowing', ({ json }) => {
|
|
expect(decodeFloat32Buffer(null, json)).toBe(undefined)
|
|
expect(decodeFloat32Buffer(Buffer.alloc(3), json)).toBe(undefined)
|
|
})
|
|
|
|
it('listEmbeddedChunksAndRankChunks_oneCorruptedRow_skipsBrokenRowAndRetrievesHealthyRows', () => {
|
|
const { db } = setupDb()
|
|
const store = new SqliteChunkStore()
|
|
store.insertDocument({
|
|
id: 'doc-mixed',
|
|
fileName: 'mixed.txt',
|
|
filePath: 'C:/docs/mixed.txt',
|
|
fileType: 'txt',
|
|
chunkCount: 2,
|
|
indexed: true,
|
|
indexedAt: 4_000,
|
|
addedAt: 4_000,
|
|
})
|
|
store.replaceChunks('doc-mixed', ['corrupt chunk', 'healthy chunk'])
|
|
const chunks = store.listChunks('doc-mixed')
|
|
|
|
// Inject corrupted JSON fallback without binary blob into chunk 0
|
|
db.update(ragChunks)
|
|
.set({ embedding: '{corrupt', embeddingBlob: null })
|
|
.where(eq(ragChunks.id, chunks[0].id))
|
|
.run()
|
|
store.setEmbedding(chunks[1].id, [0, 1, 0])
|
|
|
|
const loaded = store.listEmbeddedChunks()
|
|
expect(loaded).toHaveLength(1)
|
|
expect(loaded[0].content).toBe('healthy chunk')
|
|
|
|
const ranked = rankChunks(
|
|
[0, 1, 0],
|
|
[
|
|
...loaded,
|
|
{ documentId: 'doc-mixed', content: 'broken memory chunk', embedding: '{bad' },
|
|
],
|
|
new Map([['doc-mixed', 'mixed.txt']]),
|
|
5,
|
|
)
|
|
expect(ranked).toHaveLength(1)
|
|
expect(ranked[0].content).toBe('healthy chunk')
|
|
})
|
|
})
|