import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { eq } from 'drizzle-orm' import { bindTestDatabase, unbindTestDatabase } from '../../../src/main/db' import { ragChunks } from '../../../src/main/db/schema' import { decodeFloat32Buffer, invalidateEmbeddedChunksCache, SqliteChunkStore, } from '../../../src/main/services/rag/chunk-store' import { rankChunks } from '../../../src/main/services/rag/retrieval' import { resetRAGServiceForTests } from '../../../src/main/services/RAGService' import { adapterFor } from '../../../src/main/services/sync/sync-adapters' import { createTestDb } from '../../helpers/createTestDb' describe('SqliteChunkStore cache optimization, invalidation & hardening', () => { let closeDb: (() => void) | null = null function setupDb() { const testDb = createTestDb() closeDb = testDb.close bindTestDatabase(testDb.db) return testDb } beforeEach(() => { unbindTestDatabase() resetRAGServiceForTests() }) afterEach(() => { resetRAGServiceForTests() unbindTestDatabase() if (closeDb) { closeDb() closeDb = null } vi.restoreAllMocks() }) it('listEmbeddedChunks_decodesFloat32Vector_stripsRedundantJsonStringFromCache', () => { setupDb() const store = new SqliteChunkStore() store.insertDocument({ id: 'doc-1', fileName: 'guide.md', filePath: 'C:/docs/guide.md', fileType: 'md', chunkCount: 2, indexed: true, indexedAt: 1_000, addedAt: 1_000, }) store.replaceChunks('doc-1', ['alpha content', 'beta content']) const chunks = store.listChunks('doc-1') store.setEmbedding(chunks[0].id, [1, 0, 0]) store.setEmbedding(chunks[1].id, [0, 1, 0]) const parseSpy = vi.spyOn(JSON, 'parse') const cached = store.listEmbeddedChunks() expect(cached).toHaveLength(2) expect(cached.map((c) => c.embedding)).toEqual(['', '']) expect(Array.from(cached[0].vector ?? [])).toEqual([1, 0, 0]) expect(Array.from(cached[1].vector ?? [])).toEqual([0, 1, 0]) const ranked = rankChunks([0, 0.9, 0.1], cached, new Map([['doc-1', 'guide.md']]), 1) expect(ranked).toHaveLength(1) expect(ranked[0].content).toBe('beta content') expect(parseSpy).not.toHaveBeenCalled() }) it('invalidateEmbeddedChunksCache_afterDirectDbDelete_clearsStaleCachedChunks', () => { const { db } = setupDb() const store = new SqliteChunkStore() store.insertDocument({ id: 'doc-stale', fileName: 'stale.txt', filePath: 'C:/docs/stale.txt', fileType: 'txt', chunkCount: 1, indexed: true, indexedAt: 2_000, addedAt: 2_000, }) store.replaceChunks('doc-stale', ['to be deleted']) const chunks = store.listChunks('doc-stale') store.setEmbedding(chunks[0].id, [0.5, 0.5]) expect(store.listEmbeddedChunks()).toHaveLength(1) // Simulate out-of-band delete bypassing SqliteChunkStore methods db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run() // Before invalidation cache still serves stale entry expect(store.listEmbeddedChunks()).toHaveLength(1) invalidateEmbeddedChunksCache(db) expect(store.listEmbeddedChunks()).toEqual([]) // Also verify zero-arg invalidation clears currently open DB store.replaceChunks('doc-stale', ['re-added']) const nextChunks = store.listChunks('doc-stale') store.setEmbedding(nextChunks[0].id, [1, 0]) expect(store.listEmbeddedChunks()).toHaveLength(1) db.delete(ragChunks).where(eq(ragChunks.documentId, 'doc-stale')).run() invalidateEmbeddedChunksCache() expect(store.listEmbeddedChunks()).toEqual([]) }) it('knowledgeAdapter_deleteLocalAndApplyRemote_invalidatesEmbeddedChunksCache', async () => { const { db } = setupDb() const store = new SqliteChunkStore() const docId = '11111111-1111-4111-8111-111111111111' store.insertDocument({ id: docId, fileName: 'sync.md', filePath: 'remote://sync.md', fileType: 'md', chunkCount: 1, indexed: true, indexedAt: 3_000, addedAt: 3_000, }) store.replaceChunks(docId, ['synced chunk']) const [chunk] = store.listChunks(docId) store.setEmbedding(chunk.id, [1, 0]) expect(store.listEmbeddedChunks()).toHaveLength(1) // Even if RAGService uses an external custom store that mutates SQL directly, // knowledgeAdapter.deleteLocal / applyRemote must invalidate SqliteChunkStore cache. const spyStore: SqliteChunkStore = Object.assign(Object.create(SqliteChunkStore.prototype), { removeDocument(id: string) { db.delete(ragChunks).where(eq(ragChunks.documentId, id)).run() return true }, getDocument: () => null, insertDocument: () => {}, replaceChunks: () => {}, listChunks: () => [], updateDocument: () => {}, }) resetRAGServiceForTests({ store: spyStore }) const adapter = adapterFor('knowledge_documents') expect(adapter === null).toBe(false) expect(adapter!.deleteLocal(docId)).toBe(true) expect(store.listEmbeddedChunks()).toEqual([]) // Populate again and verify applyRemote also invalidates cache store.replaceChunks(docId, ['before remote apply']) const [reChunk] = store.listChunks(docId) store.setEmbedding(reChunk.id, [0, 1]) expect(store.listEmbeddedChunks()).toHaveLength(1) await adapter!.applyRemote( { id: docId, file_name: 'sync.md', file_type: 'md', chunk_count: 1, created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-02T00:00:00.000Z', }, { userId: 'user-1', isCurrent: () => true, remote: { selectChildren: vi.fn(async () => [{ chunk_index: 0, content: 'updated chunk' }]), } as never, }, ) // Cache was cleared so listEmbeddedChunks re-queries DB (still 1 here until re-indexed, selectSpy verifies DB query) const selectSpy = vi.spyOn(db, 'select') expect(store.listEmbeddedChunks()).toHaveLength(1) expect(selectSpy).toHaveBeenCalledTimes(1) }) it.each([ { label: 'malformed json', json: '{bad-json' }, { label: 'non-array primitive', json: '42' }, { label: 'empty array', json: '[]' }, { label: 'non-finite element', json: '[1, null, 3]' }, ])('decodeFloat32Buffer_$label_returnsUndefinedWithoutThrowing', ({ json }) => { expect(decodeFloat32Buffer(null, json)).toBe(undefined) expect(decodeFloat32Buffer(Buffer.alloc(3), json)).toBe(undefined) }) it('listEmbeddedChunksAndRankChunks_oneCorruptedRow_skipsBrokenRowAndRetrievesHealthyRows', () => { const { db } = setupDb() const store = new SqliteChunkStore() store.insertDocument({ id: 'doc-mixed', fileName: 'mixed.txt', filePath: 'C:/docs/mixed.txt', fileType: 'txt', chunkCount: 2, indexed: true, indexedAt: 4_000, addedAt: 4_000, }) store.replaceChunks('doc-mixed', ['corrupt chunk', 'healthy chunk']) const chunks = store.listChunks('doc-mixed') // Inject corrupted JSON fallback without binary blob into chunk 0 db.update(ragChunks) .set({ embedding: '{corrupt', embeddingBlob: null }) .where(eq(ragChunks.id, chunks[0].id)) .run() store.setEmbedding(chunks[1].id, [0, 1, 0]) const loaded = store.listEmbeddedChunks() expect(loaded).toHaveLength(1) expect(loaded[0].content).toBe('healthy chunk') const ranked = rankChunks( [0, 1, 0], [ ...loaded, { documentId: 'doc-mixed', content: 'broken memory chunk', embedding: '{bad' }, ], new Map([['doc-mixed', 'mixed.txt']]), 5, ) expect(ranked).toHaveLength(1) expect(ranked[0].content).toBe('healthy chunk') }) })