d3ro-voice/apps/desktop/tests/main/services/rag-chunk-store-cache.test.ts

227 lines
7.7 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { eq } from 'drizzle-orm'
import { bindTestDatabase, unbindTestDatabase } from '../../../src/main/db'
import { ragChunks } from '../../../src/main/db/schema'
import {
decodeFloat32Buffer,
invalidateEmbeddedChunksCache,
SqliteChunkStore,
} from '../../../src/main/services/rag/chunk-store'
import { rankChunks } from '../../../src/main/services/rag/retrieval'
import { resetRAGServiceForTests } from '../../../src/main/services/RAGService'
import { adapterFor } from '../../../src/main/services/sync/sync-adapters'
import { createTestDb } from '../../helpers/createTestDb'
describe('SqliteChunkStore cache optimization, invalidation & hardening', () => {
let closeDb: (() => void) | null = null
function setupDb() {
const testDb = createTestDb()
closeDb = testDb.close
bindTestDatabase(testDb.db)
return testDb
}
beforeEach(() => {
unbindTestDatabase()
resetRAGServiceForTests()
})
afterEach(() => {
resetRAGServiceForTests()
unbindTestDatabase()
if (closeDb) {
closeDb()
closeDb = null
}
vi.restoreAllMocks()
})
it('listEmbeddedChunks_decodesFloat32Vector_stripsRedundantJsonStringFromCache', () => {
setupDb()
const store = new SqliteChunkStore()
store.insertDocument({
id: 'doc-1',
fileName: 'guide.md',
filePath: 'C:/docs/guide.md',
fileType: 'md',
chunkCount: 2,
indexed: true,
indexedAt: 1_000,
addedAt: 1_000,
})
store.replaceChunks('doc-1', ['alpha content', 'beta content'])
const chunks = store.listChunks('doc-1')
store.setEmbedding(chunks[0].id, [1, 0, 0])
store.setEmbedding(chunks[1].id, [0, 1, 0])
const parseSpy = vi.spyOn(JSON, 'parse')
const cached = store.listEmbeddedChunks()
expect(cached).toHaveLength(2)
expect(cached.map((c) => c.embedding)).toEqual(['', ''])
expect(Array.from(cached[0].vector ?? [])).toEqual([1, 0, 0])
expect(Array.from(cached[1].vector ?? [])).toEqual([0, 1, 0])
const ranked = rankChunks([0, 0.9, 0.1], cached, new Map([['doc-1', 'guide.md']]), 1)
expect(ranked).toHaveLength(1)
expect(ranked[0].content).toBe('beta content')
expect(parseSpy).not.toHaveBeenCalled()
})
it('invalidateEmbeddedChunksCache_afterDirectDbDelete_clearsStaleCachedChunks', () => {
const { db } = setupDb()
const store = new SqliteChunkStore()
store.insertDocument({
id: 'doc-stale',
fileName: 'stale.txt',
filePath: 'C:/docs/stale.txt',
fileType: 'txt',
chunkCount: 1,
indexed: true,
indexedAt: 2_000,
addedAt: 2_000,
})
store.replaceChunks('doc-stale', ['to be deleted'])
const chunks = store.listChunks('doc-stale')
store.setEmbedding(chunks[0].id, [0.5, 0.5])
expect(store.listEmbeddedChunks()).toHaveLength(1)
// Simulate out-of-band delete bypassing SqliteChunkStore methods
db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run()
// Before invalidation cache still serves stale entry
expect(store.listEmbeddedChunks()).toHaveLength(1)
invalidateEmbeddedChunksCache(db)
expect(store.listEmbeddedChunks()).toEqual([])
// Also verify zero-arg invalidation clears currently open DB
store.replaceChunks('doc-stale', ['re-added'])
const nextChunks = store.listChunks('doc-stale')
store.setEmbedding(nextChunks[0].id, [1, 0])
expect(store.listEmbeddedChunks()).toHaveLength(1)
db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run()
invalidateEmbeddedChunksCache()
expect(store.listEmbeddedChunks()).toEqual([])
})
it('knowledgeAdapter_deleteLocalAndApplyRemote_invalidatesEmbeddedChunksCache', async () => {
const { db } = setupDb()
const store = new SqliteChunkStore()
const docId = '11111111-1111-4111-8111-111111111111'
store.insertDocument({
id: docId,
fileName: 'sync.md',
filePath: 'remote://sync.md',
fileType: 'md',
chunkCount: 1,
indexed: true,
indexedAt: 3_000,
addedAt: 3_000,
})
store.replaceChunks(docId, ['synced chunk'])
const [chunk] = store.listChunks(docId)
store.setEmbedding(chunk.id, [1, 0])
expect(store.listEmbeddedChunks()).toHaveLength(1)
// Even if RAGService uses an external custom store that mutates SQL directly,
// knowledgeAdapter.deleteLocal / applyRemote must invalidate SqliteChunkStore cache.
const spyStore: SqliteChunkStore = Object.assign(Object.create(SqliteChunkStore.prototype), {
removeDocument(id: string) {
db.delete(ragChunks).where(eq(ragChunks.documentId, id)).run()
return true
},
getDocument: () => null,
insertDocument: () => {},
replaceChunks: () => {},
listChunks: () => [],
updateDocument: () => {},
})
resetRAGServiceForTests({ store: spyStore })
const adapter = adapterFor('knowledge_documents')
expect(adapter === null).toBe(false)
expect(adapter!.deleteLocal(docId)).toBe(true)
expect(store.listEmbeddedChunks()).toEqual([])
// Populate again and verify applyRemote also invalidates cache
store.replaceChunks(docId, ['before remote apply'])
const [reChunk] = store.listChunks(docId)
store.setEmbedding(reChunk.id, [0, 1])
expect(store.listEmbeddedChunks()).toHaveLength(1)
await adapter!.applyRemote(
{
id: docId,
file_name: 'sync.md',
file_type: 'md',
chunk_count: 1,
created_at: '2026-01-01T00:00:00.000Z',
updated_at: '2026-01-02T00:00:00.000Z',
},
{
userId: 'user-1',
isCurrent: () => true,
remote: {
selectChildren: vi.fn(async () => [{ chunk_index: 0, content: 'updated chunk' }]),
} as never,
},
)
// Cache was cleared so listEmbeddedChunks re-queries DB (still 1 here until re-indexed, selectSpy verifies DB query)
const selectSpy = vi.spyOn(db, 'select')
expect(store.listEmbeddedChunks()).toHaveLength(1)
expect(selectSpy).toHaveBeenCalledTimes(1)
})
it.each([
{ label: 'malformed json', json: '{bad-json' },
{ label: 'non-array primitive', json: '42' },
{ label: 'empty array', json: '[]' },
{ label: 'non-finite element', json: '[1, null, 3]' },
])('decodeFloat32Buffer_$label_returnsUndefinedWithoutThrowing', ({ json }) => {
expect(decodeFloat32Buffer(null, json)).toBe(undefined)
expect(decodeFloat32Buffer(Buffer.alloc(3), json)).toBe(undefined)
})
it('listEmbeddedChunksAndRankChunks_oneCorruptedRow_skipsBrokenRowAndRetrievesHealthyRows', () => {
const { db } = setupDb()
const store = new SqliteChunkStore()
store.insertDocument({
id: 'doc-mixed',
fileName: 'mixed.txt',
filePath: 'C:/docs/mixed.txt',
fileType: 'txt',
chunkCount: 2,
indexed: true,
indexedAt: 4_000,
addedAt: 4_000,
})
store.replaceChunks('doc-mixed', ['corrupt chunk', 'healthy chunk'])
const chunks = store.listChunks('doc-mixed')
// Inject corrupted JSON fallback without binary blob into chunk 0
db.update(ragChunks)
.set({ embedding: '{corrupt', embeddingBlob: null })
.where(eq(ragChunks.id, chunks[0].id))
.run()
store.setEmbedding(chunks[1].id, [0, 1, 0])
const loaded = store.listEmbeddedChunks()
expect(loaded).toHaveLength(1)
expect(loaded[0].content).toBe('healthy chunk')
const ranked = rankChunks(
[0, 1, 0],
[
...loaded,
{ documentId: 'doc-mixed', content: 'broken memory chunk', embedding: '{bad' },
],
new Map([['doc-mixed', 'mixed.txt']]),
5,
)
expect(ranked).toHaveLength(1)
expect(ranked[0].content).toBe('healthy chunk')
})
})