d3ro-voice/apps/desktop/tests/main/services/rag-redteam-r2-0.test.ts
Yun Chan 6fef05a39f fix(rag): don't carry a query, upload or indexing run across an account switch
The chunk store reads whatever database is open at call time, so a query that
was awaiting the model returned an answer built from the previous account's
documents, and a large file being read was inserted into the new account's
knowledge base and synced there. Each operation now remembers the account
scope it started in and stops if it changed.
2026-10-21 11:23:21 +09:00

511 lines
19 KiB
TypeScript

// 레드팀 r2-0: 로컬 RAG 오케스트레이터를 가짜 포트로 검증한다(실제 fetch·SQLite 없음).
// - 임베딩 모델이 없거나 전부 실패하면 색인 실패를 알리고 진행 표시를 닫는다.
// - 색인 중에 질의가 끝나도 상태가 idle로 덮이지 않는다.
// - 추출·청킹·순위·Ollama 어댑터는 순수 모듈로 따로 검증한다.
import fs from 'fs'
import os from 'os'
import path from 'path'
import { deflateRawSync, deflateSync } from 'zlib'
import { afterEach, describe, expect, it } from 'vitest'
import { IPC_CHANNELS } from '@d3ro/core/ipc-channels'
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
import type { RAGDocument } from '@d3ro/core/types'
import {
RAGService,
defaultRAGServiceDeps,
resetRAGServiceForTests,
type RAGIndexFailure,
type RAGServiceDeps,
} from '../../../src/main/services/RAGService'
import type { ChunkStore, StoredChunk } from '../../../src/main/services/rag/chunk-store'
import type { EmbeddingPort } from '../../../src/main/services/rag/embedding-port'
import { OllamaEmbeddingAdapter, isSameOllamaModel } from '../../../src/main/services/rag/embedding-port'
import {
CHUNK_SIZE,
chunkText,
extractBinaryDocumentText,
fileTypeForExtension,
} from '../../../src/main/services/rag/document-text'
import { extractPdfText } from '../../../src/main/services/rag/pdf-text'
import { buildAnswerSystemPrompt, cosineSimilarity, rankChunks } from '../../../src/main/services/rag/retrieval'
class MemoryChunkStore implements ChunkStore {
docs = new Map<string, RAGDocument>()
chunks: StoredChunk[] = []
listDocuments(): RAGDocument[] {
return [...this.docs.values()]
}
getDocument(id: string): RAGDocument | null {
return this.docs.get(id) ?? null
}
hasDocument(id: string): boolean {
return this.docs.has(id)
}
insertDocument(doc: RAGDocument): void {
this.docs.set(doc.id, { ...doc })
}
updateDocument(id: string, patch: Partial<Pick<RAGDocument, 'chunkCount' | 'indexed' | 'indexedAt'>>): void {
const doc = this.docs.get(id)
if (doc) Object.assign(doc, patch)
}
removeDocument(id: string): boolean {
this.chunks = this.chunks.filter((c) => c.documentId !== id)
return this.docs.delete(id)
}
replaceChunks(documentId: string, chunks: readonly string[]): void {
this.chunks = this.chunks.filter((c) => c.documentId !== documentId)
chunks.forEach((content, chunkIndex) =>
this.chunks.push({ id: crypto.randomUUID(), documentId, content, embedding: '', chunkIndex })
)
}
listChunks(documentId: string): StoredChunk[] {
return this.chunks.filter((c) => c.documentId === documentId).sort((a, b) => a.chunkIndex - b.chunkIndex)
}
setEmbedding(chunkId: string, embedding: readonly number[]): void {
const chunk = this.chunks.find((c) => c.id === chunkId)
if (chunk) chunk.embedding = JSON.stringify(embedding)
}
listEmbeddedChunks(): StoredChunk[] {
return this.chunks.filter((c) => c.embedding.length > 0)
}
counts(): { documentCount: number; totalChunks: number } {
return { documentCount: this.docs.size, totalChunks: this.chunks.length }
}
}
class FakeEmbedder implements EmbeddingPort {
readonly model = 'fake-embed'
modelInstalled = true
failWhen: (text: string) => boolean = () => false
embedCalls = 0
gate: Promise<void> | null = null
async ensureModel(): Promise<void> {
if (!this.modelInstalled) {
throw new D3ROError(ErrorCode.RAGEmbeddingFailed, 'Embedding model "fake-embed" is not installed', {
reason: 'model_missing',
})
}
}
async embed(text: string): Promise<number[]> {
this.embedCalls++
if (this.gate) await this.gate
if (this.failWhen(text)) throw new D3ROError(ErrorCode.RAGEmbeddingFailed, 'Embed API error: 500')
return [text.length, 1]
}
}
interface Harness {
service: RAGService
store: MemoryChunkStore
embedder: FakeEmbedder
events: Array<{ channel: string; data: unknown }>
failures: RAGIndexFailure[]
pushed: string[]
}
function harness(overrides: Partial<RAGServiceDeps> = {}): Harness {
const store = new MemoryChunkStore()
const embedder = new FakeEmbedder()
const events: Array<{ channel: string; data: unknown }> = []
const pushed: string[] = []
const service = new RAGService({
store,
embedder,
answerer: () => ({ generate: async () => ({ text: ' answer ' }) }),
sync: () => ({
pushOne: (_entity, id) => pushed.push(`up:${id}`),
pushDelete: (_entity, id) => pushed.push(`del:${id}`),
}),
notify: (channel, data) => events.push({ channel, data }),
readFile: (filePath) => fs.promises.readFile(filePath),
fileExists: (filePath) => fs.existsSync(filePath),
assertLicensed: async () => undefined,
yieldMs: 0,
...overrides,
})
const failures: RAGIndexFailure[] = []
service.on('index-failed', (f) => failures.push(f))
return { service, store, embedder, events, failures, pushed }
}
function writeTemp(name: string, body: string): string {
const p = path.join(os.tmpdir(), `d3ro-rag-r2-${crypto.randomUUID()}-${name}`)
fs.writeFileSync(p, body, 'utf-8')
return p
}
async function settle(): Promise<void> {
for (let i = 0; i < 20; i++) await new Promise((r) => setTimeout(r, 0))
}
function channels(h: Harness): string[] {
return h.events.map((e) => e.channel)
}
const LONG_TEXT = 'Knowledge base paragraph about quarterly planning and milestones. '.repeat(30)
afterEach(() => {
resetRAGServiceForTests()
})
describe('RAGService 색인 실패 알림', () => {
it('임베딩 모델이 없으면 청크마다 기다리지 않고 실패를 알리고 진행 표시를 닫는다', async () => {
const h = harness()
h.embedder.modelInstalled = false
const doc = await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
await settle()
expect(h.embedder.embedCalls).toBe(0)
expect(h.store.getDocument(doc.id)?.indexed).toBe(false)
expect(h.failures).toEqual([
expect.objectContaining({ documentId: doc.id, code: ErrorCode.RAGEmbeddingFailed }),
])
expect(channels(h)).toEqual([IPC_CHANNELS.RAG.INDEX_FAILED, IPC_CHANNELS.RAG.INDEX_COMPLETE])
// 원문은 남아 재색인·동기화가 가능하다
expect(h.store.listChunks(doc.id).length).toBeGreaterThan(0)
expect(h.pushed).toEqual([`up:${doc.id}`])
expect(h.service.state).toBe('idle')
})
it('모든 청크 임베딩이 실패하면 indexed=false 로 두고 실패 + 실행 종료를 알린다', async () => {
const h = harness()
h.embedder.failWhen = () => true
const doc = await h.service.addDocument(writeTemp('b.txt', LONG_TEXT))
await settle()
expect(h.store.getDocument(doc.id)?.indexed).toBe(false)
expect(h.failures).toHaveLength(1)
const tail = channels(h).slice(-2)
expect(tail).toEqual([IPC_CHANNELS.RAG.INDEX_FAILED, IPC_CHANNELS.RAG.INDEX_COMPLETE])
expect(channels(h)).toContain(IPC_CHANNELS.RAG.INDEX_PROGRESS)
})
it('일부만 성공해도 색인됨으로 표시하고 실패는 알리지 않는다', async () => {
const h = harness()
let n = 0
h.embedder.failWhen = () => n++ % 2 === 0
const doc = await h.service.addDocument(writeTemp('c.txt', LONG_TEXT))
await settle()
const stored = h.store.getDocument(doc.id)
expect(stored?.indexed).toBe(true)
expect(stored?.chunkCount).toBe(h.store.listChunks(doc.id).length)
expect(h.failures).toEqual([])
expect(channels(h).at(-1)).toBe(IPC_CHANNELS.RAG.INDEX_COMPLETE)
expect(channels(h)).not.toContain(IPC_CHANNELS.RAG.INDEX_FAILED)
})
it('동기화로 받은 문서도 모델이 없으면 실패를 알린다', async () => {
const h = harness()
h.embedder.modelInstalled = false
const applied = h.service.applyRemoteDocument({
id: crypto.randomUUID(),
fileName: 'phone.txt',
fileType: 'txt',
chunks: ['first', 'second'],
addedAt: 1,
})
await settle()
expect(applied).toBe(true)
expect(h.failures).toHaveLength(1)
})
it('재색인이 실패하면 호출자에게 RAGEmbeddingFailed 로 돌려준다', async () => {
const h = harness()
const id = crypto.randomUUID()
h.service.applyRemoteDocument({ id, fileName: 'x.txt', fileType: 'txt', chunks: ['alpha', 'beta'], addedAt: 1 })
await settle()
expect(h.store.getDocument(id)?.indexed).toBe(true)
h.embedder.modelInstalled = false
await expect(h.service.reindex(id)).rejects.toMatchObject({ code: ErrorCode.RAGEmbeddingFailed })
expect(h.store.getDocument(id)?.indexed).toBe(false)
})
})
describe('RAGService 상태', () => {
it('색인 중에 질의가 끝나도 상태는 indexing 으로 남는다', async () => {
const h = harness()
const seeded = crypto.randomUUID()
h.service.applyRemoteDocument({ id: seeded, fileName: 's.txt', fileType: 'txt', chunks: ['seed chunk'], addedAt: 1 })
await settle()
let release: () => void = () => undefined
h.embedder.gate = new Promise<void>((r) => {
release = r
})
h.service.applyRemoteDocument({ id: crypto.randomUUID(), fileName: 'slow.txt', fileType: 'txt', chunks: ['slow'], addedAt: 2 })
await settle()
expect(h.service.state).toBe('indexing')
const query = h.service.query('seed?')
await settle()
expect(h.service.state).toBe('indexing')
release()
h.embedder.gate = null
const result = await query
expect(result.answer).toBe('answer')
await settle()
expect(h.service.state).toBe('idle')
})
it('색인된 청크가 없으면 질의는 RAGQueryFailed 다', async () => {
const h = harness()
await expect(h.service.query('anything')).rejects.toMatchObject({ code: ErrorCode.RAGQueryFailed })
expect(h.service.state).toBe('idle')
})
it('resetRAGServiceForTests 로 포트를 주입할 수 있다', () => {
const store = new MemoryChunkStore()
resetRAGServiceForTests({ store })
expect(defaultRAGServiceDeps().yieldMs).toBeGreaterThanOrEqual(0)
})
})
describe('문서 텍스트 (순수 함수)', () => {
it('확장자 → 종류', () => {
expect(fileTypeForExtension('.pdf')).toBe('pdf')
expect(fileTypeForExtension('.bin')).toBeNull()
})
it('청킹은 CHUNK_SIZE 창으로 겹쳐 자르고 짧은 조각은 버린다', () => {
const chunks = chunkText('a'.repeat(CHUNK_SIZE * 2))
expect(chunks.length).toBe(3)
expect(chunks.every((c) => c.length <= CHUNK_SIZE)).toBe(true)
expect(chunkText('short')).toEqual([])
})
it('압축된 PDF 스트림에서 Tj/TJ 텍스트를 뽑는다', () => {
const content = 'BT (Hello RAG world) Tj [(second) 120 (part)] TJ ET'
const stream = deflateSync(Buffer.from(content, 'binary'))
const pdf = Buffer.concat([
Buffer.from('%PDF-1.4\n1 0 obj << /Filter /FlateDecode >>\nstream\n', 'binary'),
stream,
Buffer.from('\nendstream\nendobj\n', 'binary'),
])
expect(extractPdfText(pdf)).toBe('Hello RAG world second part')
expect(extractBinaryDocumentText(pdf, 'pdf')).toBe('Hello RAG world second part')
expect(() => extractBinaryDocumentText(Buffer.from('%PDF-1.4 nothing'), 'pdf')).toThrow()
})
it('DOCX 는 ZIP 안의 document.xml 에서 문단을 뽑는다', () => {
const xml = Buffer.from('<w:document><w:body><w:p><w:r><w:t>Docx body text</w:t></w:r></w:p></w:body></w:document>')
const name = Buffer.from('word/document.xml')
const data = deflateRawSync(xml)
const local = Buffer.alloc(30)
local.writeUInt32LE(0x04034b50, 0)
local.writeUInt16LE(8, 8)
local.writeUInt32LE(data.length, 18)
local.writeUInt32LE(xml.length, 22)
local.writeUInt16LE(name.length, 26)
const central = Buffer.alloc(46)
central.writeUInt32LE(0x02014b50, 0)
central.writeUInt16LE(8, 10)
central.writeUInt32LE(data.length, 20)
central.writeUInt32LE(xml.length, 24)
central.writeUInt16LE(name.length, 28)
central.writeUInt32LE(0, 42)
const centralOffset = local.length + name.length + data.length
const eocd = Buffer.alloc(22)
eocd.writeUInt32LE(0x06054b50, 0)
eocd.writeUInt16LE(1, 8)
eocd.writeUInt16LE(1, 10)
eocd.writeUInt32LE(central.length + name.length, 12)
eocd.writeUInt32LE(centralOffset, 16)
const zip = Buffer.concat([local, name, data, central, name, eocd])
expect(extractBinaryDocumentText(zip, 'docx')).toBe('Docx body text')
})
})
describe('검색 (순수 함수)', () => {
it('코사인 유사도 순으로 topK 를 고른다', () => {
expect(cosineSimilarity([1, 0], [1, 0])).toBeCloseTo(1)
expect(cosineSimilarity([1, 0], [1, 0, 0])).toBe(0)
const ranked = rankChunks(
[1, 0],
[
{ documentId: 'd1', content: 'far', embedding: JSON.stringify([0, 1]) },
{ documentId: 'd2', content: 'near', embedding: JSON.stringify([1, 0.1]) },
],
new Map([['d2', 'near.txt']]),
1
)
expect(ranked).toEqual([expect.objectContaining({ content: 'near', fileName: 'near.txt' })])
expect(buildAnswerSystemPrompt(ranked)).toContain('[1] (near.txt)\nnear')
})
})
describe('OllamaEmbeddingAdapter', () => {
function jsonResponse(body: unknown, status = 200): Response {
return new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } })
}
it('설치된 모델이면 통과하고, 없으면 model_missing 으로 실패한다', async () => {
const calls: string[] = []
const installed = new OllamaEmbeddingAdapter({
serverUrl: () => 'http://127.0.0.1:11434',
fetchFn: async (url) => {
calls.push(url)
return jsonResponse({ models: [{ name: 'gemma4:e4b' }, { name: 'nomic-embed-text:latest' }] })
},
})
await expect(installed.ensureModel()).resolves.toBeUndefined()
expect(calls).toEqual(['http://127.0.0.1:11434/api/tags'])
const missing = new OllamaEmbeddingAdapter({
serverUrl: () => 'http://127.0.0.1:11434',
fetchFn: async () => jsonResponse({ models: [{ name: 'gemma4:e4b' }] }),
})
await expect(missing.ensureModel()).rejects.toMatchObject({
code: ErrorCode.RAGEmbeddingFailed,
details: { reason: 'model_missing' },
})
})
it('서버에 닿지 않으면 server_unreachable 로 실패한다', async () => {
const adapter = new OllamaEmbeddingAdapter({
serverUrl: () => 'http://127.0.0.1:1',
fetchFn: async () => {
throw new TypeError('fetch failed')
},
})
await expect(adapter.ensureModel()).rejects.toMatchObject({ details: { reason: 'server_unreachable' } })
})
it('embed 는 첫 벡터를 돌려주고 HTTP 오류는 RAGEmbeddingFailed 다', async () => {
const ok = new OllamaEmbeddingAdapter({
serverUrl: () => 'http://x',
fetchFn: async () => jsonResponse({ embeddings: [[0.1, 0.2]] }),
})
expect(await ok.embed('hi')).toEqual([0.1, 0.2])
const notFound = new OllamaEmbeddingAdapter({
serverUrl: () => 'http://x',
fetchFn: async () => jsonResponse({ error: 'model not found' }, 404),
})
await expect(notFound.embed('hi')).rejects.toMatchObject({ code: ErrorCode.RAGEmbeddingFailed })
})
it('모델 이름의 :latest 태그를 같은 모델로 본다', () => {
expect(isSameOllamaModel('nomic-embed-text:latest', 'nomic-embed-text')).toBe(true)
expect(isSameOllamaModel('nomic-embed-text:v1.5', 'nomic-embed-text')).toBe(false)
})
})
describe('RAGService 색인 실행 겹침', () => {
function openGate(h: Harness): () => void {
let release: () => void = () => undefined
h.embedder.gate = new Promise<void>((r) => {
release = r
})
return () => {
h.embedder.gate = null
release()
}
}
it('색인 중에 재색인하면 앞선 실행은 멈추고, 끝 알림은 새 실행이 한 번만 보낸다', async () => {
const h = harness()
const release = openGate(h)
const doc = await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
await settle()
const reindexing = h.service.reindex(doc.id)
await settle()
release()
await reindexing
await settle()
const completes = h.events.filter((e) => e.channel === IPC_CHANNELS.RAG.INDEX_COMPLETE)
expect(completes).toHaveLength(1)
expect(h.store.listChunks(doc.id).every((c) => c.embedding.length > 0)).toBe(true)
expect(h.store.getDocument(doc.id)?.indexed).toBe(true)
})
it('색인 중에 문서를 지우면 남은 청크를 임베딩하지 않고 진행 표시를 닫는다', async () => {
const h = harness()
const release = openGate(h)
const doc = await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
await settle()
h.service.removeDocument(doc.id)
release()
await settle()
expect(h.embedder.embedCalls).toBe(1)
expect(h.events.at(-1)).toEqual({
channel: IPC_CHANNELS.RAG.INDEX_COMPLETE,
data: { documentId: doc.id, fileName: doc.fileName },
})
})
it('원본이 비어 있으면 재색인은 거부하고 저장된 청크를 지우지 않는다', async () => {
const h = harness()
const file = writeTemp('a.txt', LONG_TEXT)
const doc = await h.service.addDocument(file)
await settle()
const before = h.store.listChunks(doc.id).length
fs.writeFileSync(file, ' \n', 'utf-8')
await expect(h.service.reindex(doc.id)).rejects.toMatchObject({ code: ErrorCode.RAGIndexingFailed })
expect(h.store.listChunks(doc.id)).toHaveLength(before)
expect(h.store.getDocument(doc.id)?.indexed).toBe(true)
})
})
describe('RAGService 계정 전환', () => {
it('질의 중에 계정이 바뀌면 이전 계정 문서로 만든 답을 돌려주지 않는다', async () => {
let scope = 'user-a'
let generated = 0
const h = harness({
accountScope: () => scope,
answerer: () => ({
generate: async () => {
generated++
scope = 'user-b'
return { text: 'answer from user-a documents' }
},
}),
})
await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
await settle()
await expect(h.service.query('planning')).rejects.toMatchObject({ code: ErrorCode.RAGQueryFailed })
expect(generated).toBe(1)
})
it('파일을 읽는 사이 계정이 바뀌면 새 계정 지식베이스에 문서를 넣지 않는다', async () => {
let scope = 'user-a'
const h = harness({
accountScope: () => scope,
readFile: async (filePath) => {
scope = 'user-b'
return fs.promises.readFile(filePath)
},
})
await expect(h.service.addDocument(writeTemp('a.txt', LONG_TEXT))).rejects.toMatchObject({
code: ErrorCode.RAGIndexingFailed,
})
expect(h.store.listDocuments()).toEqual([])
expect(h.pushed).toEqual([])
})
it('색인 중에 계정이 바뀌면 남은 청크를 임베딩하지 않는다', async () => {
let scope = 'user-a'
const h = harness({ accountScope: () => scope })
let release: () => void = () => undefined
h.embedder.gate = new Promise<void>((r) => {
release = r
})
await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
await settle()
scope = 'user-b'
h.embedder.gate = null
release()
await settle()
expect(h.embedder.embedCalls).toBe(1)
})
})