fix(sync,rag): keep restored rows, reconcile after tombstone pruning, reject partial knowledge docs, surface RAG indexing failures
This commit is contained in:
Yun Chan
parent
bbaf1e0a99
commit
3a46437f28
13 changed files with 1630 additions and 448 deletions
396
apps/desktop/tests/main/services/rag-redteam-r2-0.test.ts
Normal file
396
apps/desktop/tests/main/services/rag-redteam-r2-0.test.ts
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
// 레드팀 r2-0: 로컬 RAG 오케스트레이터를 가짜 포트로 검증한다(실제 fetch·SQLite 없음).
|
||||
// - 임베딩 모델이 없거나 전부 실패하면 색인 실패를 알리고 진행 표시를 닫는다.
|
||||
// - 색인 중에 질의가 끝나도 상태가 idle로 덮이지 않는다.
|
||||
// - 추출·청킹·순위·Ollama 어댑터는 순수 모듈로 따로 검증한다.
|
||||
|
||||
import fs from 'fs'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { deflateRawSync, deflateSync } from 'zlib'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { IPC_CHANNELS } from '@d3ro/core/ipc-channels'
|
||||
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
||||
import type { RAGDocument } from '@d3ro/core/types'
|
||||
import {
|
||||
RAGService,
|
||||
defaultRAGServiceDeps,
|
||||
resetRAGServiceForTests,
|
||||
type RAGIndexFailure,
|
||||
type RAGServiceDeps,
|
||||
} from '../../../src/main/services/RAGService'
|
||||
import type { ChunkStore, StoredChunk } from '../../../src/main/services/rag/chunk-store'
|
||||
import type { EmbeddingPort } from '../../../src/main/services/rag/embedding-port'
|
||||
import { OllamaEmbeddingAdapter, isSameOllamaModel } from '../../../src/main/services/rag/embedding-port'
|
||||
import {
|
||||
CHUNK_SIZE,
|
||||
chunkText,
|
||||
extractBinaryDocumentText,
|
||||
fileTypeForExtension,
|
||||
} from '../../../src/main/services/rag/document-text'
|
||||
import { extractPdfText } from '../../../src/main/services/rag/pdf-text'
|
||||
import { buildAnswerSystemPrompt, cosineSimilarity, rankChunks } from '../../../src/main/services/rag/retrieval'
|
||||
|
||||
class MemoryChunkStore implements ChunkStore {
|
||||
docs = new Map<string, RAGDocument>()
|
||||
chunks: StoredChunk[] = []
|
||||
|
||||
listDocuments(): RAGDocument[] {
|
||||
return [...this.docs.values()]
|
||||
}
|
||||
getDocument(id: string): RAGDocument | null {
|
||||
return this.docs.get(id) ?? null
|
||||
}
|
||||
hasDocument(id: string): boolean {
|
||||
return this.docs.has(id)
|
||||
}
|
||||
insertDocument(doc: RAGDocument): void {
|
||||
this.docs.set(doc.id, { ...doc })
|
||||
}
|
||||
updateDocument(id: string, patch: Partial<Pick<RAGDocument, 'chunkCount' | 'indexed' | 'indexedAt'>>): void {
|
||||
const doc = this.docs.get(id)
|
||||
if (doc) Object.assign(doc, patch)
|
||||
}
|
||||
removeDocument(id: string): boolean {
|
||||
this.chunks = this.chunks.filter((c) => c.documentId !== id)
|
||||
return this.docs.delete(id)
|
||||
}
|
||||
replaceChunks(documentId: string, chunks: readonly string[]): void {
|
||||
this.chunks = this.chunks.filter((c) => c.documentId !== documentId)
|
||||
chunks.forEach((content, chunkIndex) =>
|
||||
this.chunks.push({ id: crypto.randomUUID(), documentId, content, embedding: '', chunkIndex })
|
||||
)
|
||||
}
|
||||
listChunks(documentId: string): StoredChunk[] {
|
||||
return this.chunks.filter((c) => c.documentId === documentId).sort((a, b) => a.chunkIndex - b.chunkIndex)
|
||||
}
|
||||
setEmbedding(chunkId: string, embedding: readonly number[]): void {
|
||||
const chunk = this.chunks.find((c) => c.id === chunkId)
|
||||
if (chunk) chunk.embedding = JSON.stringify(embedding)
|
||||
}
|
||||
listEmbeddedChunks(): StoredChunk[] {
|
||||
return this.chunks.filter((c) => c.embedding.length > 0)
|
||||
}
|
||||
counts(): { documentCount: number; totalChunks: number } {
|
||||
return { documentCount: this.docs.size, totalChunks: this.chunks.length }
|
||||
}
|
||||
}
|
||||
|
||||
class FakeEmbedder implements EmbeddingPort {
|
||||
readonly model = 'fake-embed'
|
||||
modelInstalled = true
|
||||
failWhen: (text: string) => boolean = () => false
|
||||
embedCalls = 0
|
||||
gate: Promise<void> | null = null
|
||||
|
||||
async ensureModel(): Promise<void> {
|
||||
if (!this.modelInstalled) {
|
||||
throw new D3ROError(ErrorCode.RAGEmbeddingFailed, 'Embedding model "fake-embed" is not installed', {
|
||||
reason: 'model_missing',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async embed(text: string): Promise<number[]> {
|
||||
this.embedCalls++
|
||||
if (this.gate) await this.gate
|
||||
if (this.failWhen(text)) throw new D3ROError(ErrorCode.RAGEmbeddingFailed, 'Embed API error: 500')
|
||||
return [text.length, 1]
|
||||
}
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
service: RAGService
|
||||
store: MemoryChunkStore
|
||||
embedder: FakeEmbedder
|
||||
events: Array<{ channel: string; data: unknown }>
|
||||
failures: RAGIndexFailure[]
|
||||
pushed: string[]
|
||||
}
|
||||
|
||||
function harness(overrides: Partial<RAGServiceDeps> = {}): Harness {
|
||||
const store = new MemoryChunkStore()
|
||||
const embedder = new FakeEmbedder()
|
||||
const events: Array<{ channel: string; data: unknown }> = []
|
||||
const pushed: string[] = []
|
||||
const service = new RAGService({
|
||||
store,
|
||||
embedder,
|
||||
answerer: () => ({ generate: async () => ({ text: ' answer ' }) }),
|
||||
sync: () => ({
|
||||
pushOne: (_entity, id) => pushed.push(`up:${id}`),
|
||||
pushDelete: (_entity, id) => pushed.push(`del:${id}`),
|
||||
}),
|
||||
notify: (channel, data) => events.push({ channel, data }),
|
||||
readFile: (filePath) => fs.promises.readFile(filePath),
|
||||
fileExists: (filePath) => fs.existsSync(filePath),
|
||||
assertLicensed: async () => undefined,
|
||||
yieldMs: 0,
|
||||
...overrides,
|
||||
})
|
||||
const failures: RAGIndexFailure[] = []
|
||||
service.on('index-failed', (f) => failures.push(f))
|
||||
return { service, store, embedder, events, failures, pushed }
|
||||
}
|
||||
|
||||
function writeTemp(name: string, body: string): string {
|
||||
const p = path.join(os.tmpdir(), `d3ro-rag-r2-${crypto.randomUUID()}-${name}`)
|
||||
fs.writeFileSync(p, body, 'utf-8')
|
||||
return p
|
||||
}
|
||||
|
||||
async function settle(): Promise<void> {
|
||||
for (let i = 0; i < 20; i++) await new Promise((r) => setTimeout(r, 0))
|
||||
}
|
||||
|
||||
function channels(h: Harness): string[] {
|
||||
return h.events.map((e) => e.channel)
|
||||
}
|
||||
|
||||
const LONG_TEXT = 'Knowledge base paragraph about quarterly planning and milestones. '.repeat(30)
|
||||
|
||||
afterEach(() => {
|
||||
resetRAGServiceForTests()
|
||||
})
|
||||
|
||||
describe('RAGService 색인 실패 알림', () => {
|
||||
it('임베딩 모델이 없으면 청크마다 기다리지 않고 실패를 알리고 진행 표시를 닫는다', async () => {
|
||||
const h = harness()
|
||||
h.embedder.modelInstalled = false
|
||||
const doc = await h.service.addDocument(writeTemp('a.txt', LONG_TEXT))
|
||||
await settle()
|
||||
|
||||
expect(h.embedder.embedCalls).toBe(0)
|
||||
expect(h.store.getDocument(doc.id)?.indexed).toBe(false)
|
||||
expect(h.failures).toEqual([
|
||||
expect.objectContaining({ documentId: doc.id, code: ErrorCode.RAGEmbeddingFailed }),
|
||||
])
|
||||
expect(channels(h)).toEqual([IPC_CHANNELS.RAG.INDEX_FAILED, IPC_CHANNELS.RAG.INDEX_COMPLETE])
|
||||
// 원문은 남아 재색인·동기화가 가능하다
|
||||
expect(h.store.listChunks(doc.id).length).toBeGreaterThan(0)
|
||||
expect(h.pushed).toEqual([`up:${doc.id}`])
|
||||
expect(h.service.state).toBe('idle')
|
||||
})
|
||||
|
||||
it('모든 청크 임베딩이 실패하면 indexed=false 로 두고 실패 + 실행 종료를 알린다', async () => {
|
||||
const h = harness()
|
||||
h.embedder.failWhen = () => true
|
||||
const doc = await h.service.addDocument(writeTemp('b.txt', LONG_TEXT))
|
||||
await settle()
|
||||
|
||||
expect(h.store.getDocument(doc.id)?.indexed).toBe(false)
|
||||
expect(h.failures).toHaveLength(1)
|
||||
const tail = channels(h).slice(-2)
|
||||
expect(tail).toEqual([IPC_CHANNELS.RAG.INDEX_FAILED, IPC_CHANNELS.RAG.INDEX_COMPLETE])
|
||||
expect(channels(h)).toContain(IPC_CHANNELS.RAG.INDEX_PROGRESS)
|
||||
})
|
||||
|
||||
it('일부만 성공해도 색인됨으로 표시하고 실패는 알리지 않는다', async () => {
|
||||
const h = harness()
|
||||
let n = 0
|
||||
h.embedder.failWhen = () => n++ % 2 === 0
|
||||
const doc = await h.service.addDocument(writeTemp('c.txt', LONG_TEXT))
|
||||
await settle()
|
||||
|
||||
const stored = h.store.getDocument(doc.id)
|
||||
expect(stored?.indexed).toBe(true)
|
||||
expect(stored?.chunkCount).toBe(h.store.listChunks(doc.id).length)
|
||||
expect(h.failures).toEqual([])
|
||||
expect(channels(h).at(-1)).toBe(IPC_CHANNELS.RAG.INDEX_COMPLETE)
|
||||
expect(channels(h)).not.toContain(IPC_CHANNELS.RAG.INDEX_FAILED)
|
||||
})
|
||||
|
||||
it('동기화로 받은 문서도 모델이 없으면 실패를 알린다', async () => {
|
||||
const h = harness()
|
||||
h.embedder.modelInstalled = false
|
||||
const applied = h.service.applyRemoteDocument({
|
||||
id: crypto.randomUUID(),
|
||||
fileName: 'phone.txt',
|
||||
fileType: 'txt',
|
||||
chunks: ['first', 'second'],
|
||||
addedAt: 1,
|
||||
})
|
||||
await settle()
|
||||
expect(applied).toBe(true)
|
||||
expect(h.failures).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('재색인이 실패하면 호출자에게 RAGEmbeddingFailed 로 돌려준다', async () => {
|
||||
const h = harness()
|
||||
const id = crypto.randomUUID()
|
||||
h.service.applyRemoteDocument({ id, fileName: 'x.txt', fileType: 'txt', chunks: ['alpha', 'beta'], addedAt: 1 })
|
||||
await settle()
|
||||
expect(h.store.getDocument(id)?.indexed).toBe(true)
|
||||
|
||||
h.embedder.modelInstalled = false
|
||||
await expect(h.service.reindex(id)).rejects.toMatchObject({ code: ErrorCode.RAGEmbeddingFailed })
|
||||
expect(h.store.getDocument(id)?.indexed).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('RAGService 상태', () => {
|
||||
it('색인 중에 질의가 끝나도 상태는 indexing 으로 남는다', async () => {
|
||||
const h = harness()
|
||||
const seeded = crypto.randomUUID()
|
||||
h.service.applyRemoteDocument({ id: seeded, fileName: 's.txt', fileType: 'txt', chunks: ['seed chunk'], addedAt: 1 })
|
||||
await settle()
|
||||
|
||||
let release: () => void = () => undefined
|
||||
h.embedder.gate = new Promise<void>((r) => {
|
||||
release = r
|
||||
})
|
||||
h.service.applyRemoteDocument({ id: crypto.randomUUID(), fileName: 'slow.txt', fileType: 'txt', chunks: ['slow'], addedAt: 2 })
|
||||
await settle()
|
||||
expect(h.service.state).toBe('indexing')
|
||||
|
||||
const query = h.service.query('seed?')
|
||||
await settle()
|
||||
expect(h.service.state).toBe('indexing')
|
||||
release()
|
||||
h.embedder.gate = null
|
||||
const result = await query
|
||||
expect(result.answer).toBe('answer')
|
||||
await settle()
|
||||
expect(h.service.state).toBe('idle')
|
||||
})
|
||||
|
||||
it('색인된 청크가 없으면 질의는 RAGQueryFailed 다', async () => {
|
||||
const h = harness()
|
||||
await expect(h.service.query('anything')).rejects.toMatchObject({ code: ErrorCode.RAGQueryFailed })
|
||||
expect(h.service.state).toBe('idle')
|
||||
})
|
||||
|
||||
it('resetRAGServiceForTests 로 포트를 주입할 수 있다', () => {
|
||||
const store = new MemoryChunkStore()
|
||||
resetRAGServiceForTests({ store })
|
||||
expect(defaultRAGServiceDeps().yieldMs).toBeGreaterThanOrEqual(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('문서 텍스트 (순수 함수)', () => {
|
||||
it('확장자 → 종류', () => {
|
||||
expect(fileTypeForExtension('.pdf')).toBe('pdf')
|
||||
expect(fileTypeForExtension('.bin')).toBeNull()
|
||||
})
|
||||
|
||||
it('청킹은 CHUNK_SIZE 창으로 겹쳐 자르고 짧은 조각은 버린다', () => {
|
||||
const chunks = chunkText('a'.repeat(CHUNK_SIZE * 2))
|
||||
expect(chunks.length).toBe(3)
|
||||
expect(chunks.every((c) => c.length <= CHUNK_SIZE)).toBe(true)
|
||||
expect(chunkText('short')).toEqual([])
|
||||
})
|
||||
|
||||
it('압축된 PDF 스트림에서 Tj/TJ 텍스트를 뽑는다', () => {
|
||||
const content = 'BT (Hello RAG world) Tj [(second) 120 (part)] TJ ET'
|
||||
const stream = deflateSync(Buffer.from(content, 'binary'))
|
||||
const pdf = Buffer.concat([
|
||||
Buffer.from('%PDF-1.4\n1 0 obj << /Filter /FlateDecode >>\nstream\n', 'binary'),
|
||||
stream,
|
||||
Buffer.from('\nendstream\nendobj\n', 'binary'),
|
||||
])
|
||||
expect(extractPdfText(pdf)).toBe('Hello RAG world second part')
|
||||
expect(extractBinaryDocumentText(pdf, 'pdf')).toBe('Hello RAG world second part')
|
||||
expect(() => extractBinaryDocumentText(Buffer.from('%PDF-1.4 nothing'), 'pdf')).toThrow()
|
||||
})
|
||||
|
||||
it('DOCX 는 ZIP 안의 document.xml 에서 문단을 뽑는다', () => {
|
||||
const xml = Buffer.from('<w:document><w:body><w:p><w:r><w:t>Docx body text</w:t></w:r></w:p></w:body></w:document>')
|
||||
const name = Buffer.from('word/document.xml')
|
||||
const data = deflateRawSync(xml)
|
||||
const local = Buffer.alloc(30)
|
||||
local.writeUInt32LE(0x04034b50, 0)
|
||||
local.writeUInt16LE(8, 8)
|
||||
local.writeUInt32LE(data.length, 18)
|
||||
local.writeUInt32LE(xml.length, 22)
|
||||
local.writeUInt16LE(name.length, 26)
|
||||
const central = Buffer.alloc(46)
|
||||
central.writeUInt32LE(0x02014b50, 0)
|
||||
central.writeUInt16LE(8, 10)
|
||||
central.writeUInt32LE(data.length, 20)
|
||||
central.writeUInt32LE(xml.length, 24)
|
||||
central.writeUInt16LE(name.length, 28)
|
||||
central.writeUInt32LE(0, 42)
|
||||
const centralOffset = local.length + name.length + data.length
|
||||
const eocd = Buffer.alloc(22)
|
||||
eocd.writeUInt32LE(0x06054b50, 0)
|
||||
eocd.writeUInt16LE(1, 8)
|
||||
eocd.writeUInt16LE(1, 10)
|
||||
eocd.writeUInt32LE(central.length + name.length, 12)
|
||||
eocd.writeUInt32LE(centralOffset, 16)
|
||||
const zip = Buffer.concat([local, name, data, central, name, eocd])
|
||||
expect(extractBinaryDocumentText(zip, 'docx')).toBe('Docx body text')
|
||||
})
|
||||
})
|
||||
|
||||
describe('검색 (순수 함수)', () => {
|
||||
it('코사인 유사도 순으로 topK 를 고른다', () => {
|
||||
expect(cosineSimilarity([1, 0], [1, 0])).toBeCloseTo(1)
|
||||
expect(cosineSimilarity([1, 0], [1, 0, 0])).toBe(0)
|
||||
const ranked = rankChunks(
|
||||
[1, 0],
|
||||
[
|
||||
{ documentId: 'd1', content: 'far', embedding: JSON.stringify([0, 1]) },
|
||||
{ documentId: 'd2', content: 'near', embedding: JSON.stringify([1, 0.1]) },
|
||||
],
|
||||
new Map([['d2', 'near.txt']]),
|
||||
1
|
||||
)
|
||||
expect(ranked).toEqual([expect.objectContaining({ content: 'near', fileName: 'near.txt' })])
|
||||
expect(buildAnswerSystemPrompt(ranked)).toContain('[1] (near.txt)\nnear')
|
||||
})
|
||||
})
|
||||
|
||||
describe('OllamaEmbeddingAdapter', () => {
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } })
|
||||
}
|
||||
|
||||
it('설치된 모델이면 통과하고, 없으면 model_missing 으로 실패한다', async () => {
|
||||
const calls: string[] = []
|
||||
const installed = new OllamaEmbeddingAdapter({
|
||||
serverUrl: () => 'http://127.0.0.1:11434',
|
||||
fetchFn: async (url) => {
|
||||
calls.push(url)
|
||||
return jsonResponse({ models: [{ name: 'gemma4:e4b' }, { name: 'nomic-embed-text:latest' }] })
|
||||
},
|
||||
})
|
||||
await expect(installed.ensureModel()).resolves.toBeUndefined()
|
||||
expect(calls).toEqual(['http://127.0.0.1:11434/api/tags'])
|
||||
|
||||
const missing = new OllamaEmbeddingAdapter({
|
||||
serverUrl: () => 'http://127.0.0.1:11434',
|
||||
fetchFn: async () => jsonResponse({ models: [{ name: 'gemma4:e4b' }] }),
|
||||
})
|
||||
await expect(missing.ensureModel()).rejects.toMatchObject({
|
||||
code: ErrorCode.RAGEmbeddingFailed,
|
||||
details: { reason: 'model_missing' },
|
||||
})
|
||||
})
|
||||
|
||||
it('서버에 닿지 않으면 server_unreachable 로 실패한다', async () => {
|
||||
const adapter = new OllamaEmbeddingAdapter({
|
||||
serverUrl: () => 'http://127.0.0.1:1',
|
||||
fetchFn: async () => {
|
||||
throw new TypeError('fetch failed')
|
||||
},
|
||||
})
|
||||
await expect(adapter.ensureModel()).rejects.toMatchObject({ details: { reason: 'server_unreachable' } })
|
||||
})
|
||||
|
||||
it('embed 는 첫 벡터를 돌려주고 HTTP 오류는 RAGEmbeddingFailed 다', async () => {
|
||||
const ok = new OllamaEmbeddingAdapter({
|
||||
serverUrl: () => 'http://x',
|
||||
fetchFn: async () => jsonResponse({ embeddings: [[0.1, 0.2]] }),
|
||||
})
|
||||
expect(await ok.embed('hi')).toEqual([0.1, 0.2])
|
||||
const notFound = new OllamaEmbeddingAdapter({
|
||||
serverUrl: () => 'http://x',
|
||||
fetchFn: async () => jsonResponse({ error: 'model not found' }, 404),
|
||||
})
|
||||
await expect(notFound.embed('hi')).rejects.toMatchObject({ code: ErrorCode.RAGEmbeddingFailed })
|
||||
})
|
||||
|
||||
it('모델 이름의 :latest 태그를 같은 모델로 본다', () => {
|
||||
expect(isSameOllamaModel('nomic-embed-text:latest', 'nomic-embed-text')).toBe(true)
|
||||
expect(isSameOllamaModel('nomic-embed-text:v1.5', 'nomic-embed-text')).toBe(false)
|
||||
})
|
||||
})
|
||||
281
apps/desktop/tests/main/sync/sync-redteam-r2-0.test.ts
Normal file
281
apps/desktop/tests/main/sync/sync-redteam-r2-0.test.ts
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
// 레드팀 r2-0: 원격 삭제(tombstone) 반영의 복원·보존 기간 경계, 지식 문서 청크 완전성.
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { eq } from 'drizzle-orm'
|
||||
import { createTestDb } from '../../helpers/createTestDb'
|
||||
import { FakeSyncRemote } from '../../helpers/fakeSyncRemote'
|
||||
import { bindTestDatabase, unbindTestDatabase } from '../../../src/main/db'
|
||||
import { history, ragChunks, ragDocuments } from '../../../src/main/db/schema'
|
||||
import { initInMemoryConfig, resetInMemoryConfig } from '../../../src/main/services/ConfigService'
|
||||
import {
|
||||
getCustomInstructionService,
|
||||
resetCustomInstructionServiceForTests,
|
||||
} from '../../../src/main/services/CustomInstructionService'
|
||||
import { resetDictationTemplateServiceForTests } from '../../../src/main/services/DictationTemplateService'
|
||||
import { resetMeetingDocTemplateServiceForTests } from '../../../src/main/services/MeetingDocTemplateService'
|
||||
import { resetRAGServiceForTests } from '../../../src/main/services/RAGService'
|
||||
import { SyncEngine } from '../../../src/main/services/sync/SyncEngine'
|
||||
import { completeKnowledgeChunks } from '../../../src/main/services/sync/sync-adapters'
|
||||
import { enqueueChange, pendingOps } from '../../../src/main/services/sync/sync-outbox'
|
||||
import {
|
||||
TOMBSTONE_RETENTION_MS,
|
||||
earliestDeletedAt,
|
||||
isRestoredAfter,
|
||||
remoteTimeAfter,
|
||||
tombstoneWindowExpired,
|
||||
} from '../../../src/main/services/sync/tombstone-policy'
|
||||
import type { EmbeddingPort } from '../../../src/main/services/rag/embedding-port'
|
||||
|
||||
const USER = '11111111-1111-4111-8111-111111111111'
|
||||
const DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
let testDb: ReturnType<typeof createTestDb>
|
||||
let remote: FakeSyncRemote
|
||||
let engine: SyncEngine
|
||||
let clock: number
|
||||
|
||||
const offlineEmbedder: EmbeddingPort = {
|
||||
model: 'test-embed',
|
||||
ensureModel: () => Promise.reject(new Error('no embedding server in tests')),
|
||||
embed: () => Promise.reject(new Error('no embedding server in tests')),
|
||||
}
|
||||
|
||||
function makeEngine(): SyncEngine {
|
||||
return new SyncEngine({ remote, userId: USER, now: () => clock })
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
testDb = createTestDb()
|
||||
bindTestDatabase(testDb.db, USER)
|
||||
initInMemoryConfig()
|
||||
resetCustomInstructionServiceForTests()
|
||||
resetDictationTemplateServiceForTests()
|
||||
resetMeetingDocTemplateServiceForTests()
|
||||
resetRAGServiceForTests({ embedder: offlineEmbedder, notify: () => undefined, yieldMs: 0 })
|
||||
getCustomInstructionService().initialize()
|
||||
remote = new FakeSyncRemote(USER)
|
||||
clock = Date.parse('2026-09-27T00:00:00.000Z')
|
||||
engine = makeEngine()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
engine.dispose()
|
||||
resetRAGServiceForTests()
|
||||
unbindTestDatabase()
|
||||
resetInMemoryConfig()
|
||||
testDb.close()
|
||||
})
|
||||
|
||||
function historyRow(id: string, text: string): Record<string, unknown> {
|
||||
return { id, original_text: text, duration: 1, mode: 'dictation', status: 'completed' }
|
||||
}
|
||||
|
||||
function localHistoryIds(): string[] {
|
||||
return testDb.db.select({ id: history.id }).from(history).all().map((r) => r.id).sort()
|
||||
}
|
||||
|
||||
describe('tombstone 뒤 계정 보관본 복원', () => {
|
||||
it('삭제를 이미 반영한 데스크톱: 복원된 행을 받고, 이후 pull의 겹쳐 읽기가 다시 지우지 않는다', async () => {
|
||||
const x = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(x, 'keep me'))
|
||||
await engine.runFullSync()
|
||||
expect(localHistoryIds()).toEqual([x])
|
||||
|
||||
remote.mobileDelete('history', x)
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([])
|
||||
|
||||
// restore_account_portability: 같은 id로 다시 넣고, 0037 이후 서버가 updated_at을 새로 찍는다.
|
||||
remote.mobileInsert('history', historyRow(x, 'keep me'))
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([x])
|
||||
|
||||
// tombstone 커서는 여전히 T1 근처라 다음 pull은 같은 tombstone을 다시 읽는다.
|
||||
await engine.pull()
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([x])
|
||||
})
|
||||
|
||||
it('삭제와 복원 사이 오프라인이던 데스크톱: 첫 pull에서 복원된 행을 지우지 않는다', async () => {
|
||||
const x = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(x, 'v1'))
|
||||
await engine.runFullSync()
|
||||
|
||||
remote.mobileDelete('history', x)
|
||||
remote.mobileInsert('history', historyRow(x, 'restored'))
|
||||
const result = await engine.pull()
|
||||
|
||||
expect(result.deleted).toBe(0)
|
||||
expect(localHistoryIds()).toEqual([x])
|
||||
expect(testDb.db.select().from(history).where(eq(history.id, x)).get()?.originalText).toBe('restored')
|
||||
})
|
||||
|
||||
it('runFullSync(push 전 tombstone + pull 안의 tombstone)도 복원된 행을 지우지 않는다', async () => {
|
||||
const x = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(x, 'v1'))
|
||||
await engine.runFullSync()
|
||||
|
||||
remote.mobileDelete('history', x)
|
||||
remote.mobileInsert('history', historyRow(x, 'restored'))
|
||||
await engine.runFullSync()
|
||||
await engine.runFullSync()
|
||||
expect(localHistoryIds()).toEqual([x])
|
||||
})
|
||||
|
||||
it('복원되지 않은 삭제는 그대로 반영하고, 로컬 대기 변경보다 우선한다', async () => {
|
||||
const kept = crypto.randomUUID()
|
||||
const gone = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(kept, 'k'))
|
||||
remote.mobileInsert('history', historyRow(gone, 'g'))
|
||||
await engine.runFullSync()
|
||||
|
||||
enqueueChange('history', gone, 'upsert')
|
||||
remote.mobileDelete('history', gone)
|
||||
remote.mobileDelete('history', kept)
|
||||
remote.mobileInsert('history', historyRow(kept, 'k restored'))
|
||||
await engine.pull()
|
||||
|
||||
expect(localHistoryIds()).toEqual([kept])
|
||||
expect(pendingOps('history').has(gone)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('tombstone 보존 기간(180일) 초과', () => {
|
||||
function pruneAllTombstones(): void {
|
||||
remote.rows('sync_tombstones').splice(0)
|
||||
}
|
||||
|
||||
it('보존 기간보다 오래 쉰 기기는 서버에 없는 로컬 행을 지워 전체 대조한다', async () => {
|
||||
const stale = crypto.randomUUID()
|
||||
const alive = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(stale, 'deleted on phone long ago'))
|
||||
remote.mobileInsert('history', historyRow(alive, 'still here'))
|
||||
await engine.runFullSync()
|
||||
// 다음 tombstone 커서가 생기도록 다른 행 하나를 지워 둔다
|
||||
const other = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(other, 'other'))
|
||||
await engine.pull()
|
||||
remote.mobileDelete('history', other)
|
||||
await engine.pull()
|
||||
|
||||
remote.mobileDelete('history', stale)
|
||||
pruneAllTombstones()
|
||||
clock += TOMBSTONE_RETENTION_MS + DAY
|
||||
|
||||
const result = await engine.pull()
|
||||
expect(result.errors).toEqual([])
|
||||
expect(localHistoryIds()).toEqual([alive])
|
||||
expect(result.deleted).toBe(1)
|
||||
|
||||
// 편집해도 지운 행이 서버에 되살아나지 않는다(로컬에 없다)
|
||||
expect(remote.find('history', stale)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('아직 올리지 않은 로컬 변경이 있는 행은 전체 대조가 지우지 않는다', async () => {
|
||||
const seed = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(seed, 'seed'))
|
||||
await engine.runFullSync()
|
||||
remote.mobileDelete('history', seed)
|
||||
await engine.pull()
|
||||
|
||||
const local = crypto.randomUUID()
|
||||
const at = clock
|
||||
testDb.db.insert(history).values({ id: local, originalText: 'offline note', duration: 1, createdAt: at, updatedAt: at }).run()
|
||||
enqueueChange('history', local, 'upsert')
|
||||
pruneAllTombstones()
|
||||
clock += TOMBSTONE_RETENTION_MS + DAY
|
||||
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([local])
|
||||
})
|
||||
|
||||
it('보존 기간 안이면 전체 대조를 하지 않는다(서버에 없는 행을 함부로 지우지 않는다)', async () => {
|
||||
const seed = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(seed, 'seed'))
|
||||
await engine.runFullSync()
|
||||
remote.mobileDelete('history', seed)
|
||||
await engine.pull()
|
||||
|
||||
const local = crypto.randomUUID()
|
||||
testDb.db.insert(history).values({ id: local, originalText: 'x', duration: 1, createdAt: clock, updatedAt: clock }).run()
|
||||
clock += 30 * DAY
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([local])
|
||||
})
|
||||
|
||||
it('한 번 대조하면 다음 pull부터는 다시 대조하지 않는다', async () => {
|
||||
const seed = crypto.randomUUID()
|
||||
remote.mobileInsert('history', historyRow(seed, 'seed'))
|
||||
await engine.runFullSync()
|
||||
remote.mobileDelete('history', seed)
|
||||
await engine.pull()
|
||||
clock += TOMBSTONE_RETENTION_MS + DAY
|
||||
await engine.pull()
|
||||
|
||||
const fresh = crypto.randomUUID()
|
||||
testDb.db.insert(history).values({ id: fresh, originalText: 'y', duration: 1, createdAt: clock, updatedAt: clock }).run()
|
||||
clock += DAY
|
||||
await engine.pull()
|
||||
expect(localHistoryIds()).toEqual([fresh])
|
||||
})
|
||||
})
|
||||
|
||||
describe('tombstone 정책 (순수 함수)', () => {
|
||||
it('서버 시각 비교는 마이크로초까지 본다', () => {
|
||||
expect(remoteTimeAfter('2026-09-27T00:00:00.000200+00:00', '2026-09-27T00:00:00.000100+00:00')).toBe(true)
|
||||
expect(remoteTimeAfter('2026-09-27T00:00:01Z', '2026-09-27T00:00:02Z')).toBe(false)
|
||||
expect(remoteTimeAfter('bad', '2026-09-27T00:00:02Z')).toBe(false)
|
||||
})
|
||||
|
||||
it('복원 판정과 가장 이른 삭제 시각', () => {
|
||||
const ref = { rowId: 'a', deletedAt: '2026-09-27T00:00:05Z' }
|
||||
expect(isRestoredAfter(ref, '2026-09-27T00:00:06Z')).toBe(true)
|
||||
expect(isRestoredAfter(ref, '2026-09-27T00:00:04Z')).toBe(false)
|
||||
expect(isRestoredAfter({ rowId: 'a', deletedAt: null }, '2026-09-27T00:00:06Z')).toBe(false)
|
||||
expect(earliestDeletedAt([ref, { rowId: 'b', deletedAt: '2026-09-27T00:00:01Z' }, { rowId: 'c', deletedAt: null }])).toBe(
|
||||
'2026-09-27T00:00:01Z'
|
||||
)
|
||||
})
|
||||
|
||||
it('보존 기간 창', () => {
|
||||
const now = Date.parse('2026-09-27T00:00:00Z')
|
||||
expect(tombstoneWindowExpired(null, now)).toBe(false)
|
||||
expect(tombstoneWindowExpired(now - 30 * DAY, now)).toBe(false)
|
||||
expect(tombstoneWindowExpired(now - 179 * DAY, now)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('지식 문서 청크 완전성', () => {
|
||||
it('문서 행의 chunk_count보다 적은 청크만 올라와 있으면 미루고, 다 올라오면 전부 저장한다', async () => {
|
||||
const id = crypto.randomUUID()
|
||||
remote.mobileInsert('knowledge_documents', { id, title: 'Big', file_name: 'big.txt', file_type: 'txt', chunk_count: 3 })
|
||||
remote.rows('knowledge_chunks').push(
|
||||
{ id: crypto.randomUUID(), document_id: id, chunk_index: 0, content: 'c0' },
|
||||
{ id: crypto.randomUUID(), document_id: id, chunk_index: 1, content: 'c1' }
|
||||
)
|
||||
await engine.runFullSync()
|
||||
expect(testDb.db.select().from(ragDocuments).where(eq(ragDocuments.id, id)).get()).toBeUndefined()
|
||||
|
||||
remote.rows('knowledge_chunks').push({ id: crypto.randomUUID(), document_id: id, chunk_index: 2, content: 'c2' })
|
||||
await engine.pull()
|
||||
const doc = testDb.db.select().from(ragDocuments).where(eq(ragDocuments.id, id)).get()
|
||||
expect(doc?.chunkCount).toBe(3)
|
||||
const chunks = testDb.db.select().from(ragChunks).where(eq(ragChunks.documentId, id)).all()
|
||||
expect(chunks.sort((a, b) => a.chunkIndex - b.chunkIndex).map((c) => c.content)).toEqual(['c0', 'c1', 'c2'])
|
||||
})
|
||||
|
||||
it('completeKnowledgeChunks: 개수·연속 index를 확인한다', () => {
|
||||
const rows = [
|
||||
{ chunk_index: 1, content: 'b' },
|
||||
{ chunk_index: 0, content: 'a' },
|
||||
]
|
||||
expect(completeKnowledgeChunks(rows, 2)).toEqual(['a', 'b'])
|
||||
expect(completeKnowledgeChunks(rows, 3)).toBeNull()
|
||||
expect(completeKnowledgeChunks(rows, null)).toEqual(['a', 'b'])
|
||||
expect(completeKnowledgeChunks([{ chunk_index: 0, content: 'a' }, { chunk_index: 2, content: 'c' }], null)).toBeNull()
|
||||
expect(completeKnowledgeChunks([{ chunk_index: 0, content: 'a' }, { chunk_index: 0, content: 'a' }], 2)).toBeNull()
|
||||
expect(completeKnowledgeChunks([], 0)).toBeNull()
|
||||
expect(completeKnowledgeChunks([{ chunk_index: 0, content: null }], 1)).toBeNull()
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue