fix(desktop): add precise local OCR reading

This commit is contained in:
Yun Chan 2026-11-21 10:15:49 +09:00
parent 59cdddbecb
commit 25fd3ac004
30 changed files with 626 additions and 15 deletions

View file

@ -0,0 +1,113 @@
import { describe, expect, it, vi } from 'vitest'
import { ErrorCode } from '@d3ro/core/errors'
import { createOllamaPrecisionOcrEngine } from '../../../src/main/services/screen-ocr/ollama-precision-engine'
type Handler = (url: string, init?: RequestInit) => Promise<Response>
function json(body: unknown): Response {
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } })
}
function harness(handler: Handler) {
const calls: string[] = []
const fetchFn: typeof fetch = async (input, init) => {
const url = String(input)
calls.push(url)
return handler(url, init)
}
const ensureRunning = vi.fn(async () => 'running')
const engine = createOllamaPrecisionOcrEngine({
serverUrl: 'http://127.0.0.1:11434',
fetchFn,
ensureRunning,
readImage: async () => Buffer.from('image')
})
return { engine, calls, ensureRunning }
}
const modelPresent = json({ models: [{ name: 'glm-ocr:latest' }] })
const generated = (text: string, reason = 'stop') => json({ done: true, done_reason: reason, response: text })
describe('Ollama 정밀 OCR 엔진', () => {
it('정밀인식_모델이있으면_다운로드없이공식프롬프트와로컬이미지로호출한다', async () => {
const requests: Array<{ url: string; body: Record<string, unknown> }> = []
const { engine, calls } = harness(async (url, init) => {
if (url.endsWith('/api/tags')) return modelPresent
if (url.endsWith('/api/generate')) {
requests.push({ url, body: JSON.parse(String(init?.body)) as Record<string, unknown> })
return generated('정확한 결과')
}
throw new Error(`unexpected ${url}`)
})
const result = await engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 1024 })
expect(result).toBe('정확한 결과')
expect(calls.some((url) => url.endsWith('/api/pull'))).toBe(false)
expect(requests[0].body).toMatchObject({
model: 'glm-ocr:latest',
prompt: 'Text Recognition:',
images: [Buffer.from('image').toString('base64')],
stream: false,
keep_alive: '2m',
options: { temperature: 0, num_ctx: 8192, num_predict: 1024 }
})
})
it('정밀인식_모델이없으면_다운로드를완료한뒤인식한다', async () => {
const { engine, calls } = harness(async (url) => {
if (url.endsWith('/api/tags')) return json({ models: [] })
if (url.endsWith('/api/pull')) return new Response('{"status":"downloading"}\n{"status":"success"}\n')
if (url.endsWith('/api/generate')) return generated('결과')
throw new Error(`unexpected ${url}`)
})
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).resolves.toBe('결과')
expect(calls.findIndex((url) => url.endsWith('/api/pull'))).toBeLessThan(calls.findIndex((url) => url.endsWith('/api/generate')))
})
it('정밀인식_토큰한도로잘린결과_성공으로처리하지않는다', async () => {
const { engine } = harness(async (url) => url.endsWith('/api/tags') ? modelPresent : generated('불완전한 결과', 'length'))
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrRecognitionFailed
})
})
it('정밀인식_빈결과_성공으로처리하지않는다', async () => {
const { engine } = harness(async (url) => url.endsWith('/api/tags') ? modelPresent : generated(' \n'))
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrRecognitionFailed
})
})
it('정밀인식_원격Ollama주소_화면을볼내지않고차단한다', async () => {
const fetchFn = vi.fn(async () => json({}))
const engine = createOllamaPrecisionOcrEngine({
serverUrl: 'https://ollama.example.com',
fetchFn,
ensureRunning: async () => 'running',
readImage: async () => Buffer.from('image')
})
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrUnavailable
})
expect(fetchFn).not.toHaveBeenCalled()
})
it('정밀인식_다운로드중취소_인식을시작하지않는다', async () => {
const controller = new AbortController()
let pullStarted = false
const { engine, calls } = harness(async (url, init) => {
if (url.endsWith('/api/tags')) return json({ models: [] })
if (url.endsWith('/api/pull')) {
pullStarted = true
return new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener('abort', () => reject(new Error('aborted')), { once: true })
})
}
return generated('실행하면 안 되는 결과')
})
const pending = engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512, signal: controller.signal })
await vi.waitFor(() => expect(pullStarted).toBe(true))
controller.abort()
await expect(pending).rejects.toMatchObject({ code: ErrorCode.ScreenOcrRecognitionFailed })
expect(calls.some((url) => url.endsWith('/api/generate'))).toBe(false)
})
})

View file

@ -47,6 +47,7 @@ interface HarnessOptions {
/** 인식을 곧바로 끝낼지 (기본 true). false 면 finishRecognition 을 부를 때까지 걸려 있다. */
autoRecognize?: boolean
openExternal?: (url: string) => Promise<boolean>
precision?: ScreenOcrDeps['precision']['recognize']
translate?: ScreenOcrDeps['llm']['translate']
renderPdf?: ScreenOcrDeps['renderPdf']
clipboardError?: Error
@ -93,6 +94,7 @@ function createHarness(options: HarnessOptions = {}): Harness {
})
}
},
precision: { recognize: options.precision ?? (async () => '정밀 결과') },
overlay: {
show: (bounds, payload) => shown.push({ bounds, payload }),
sendRecognized: (payload) => recognized.push(payload),
@ -472,6 +474,77 @@ describe('ScreenOcrService 요청 처리', () => {
})
})
describe('GLM-OCR 선택 영역 정밀 읽기', () => {
it('정밀읽기_선택영역_크롭만모델에볼내고임시파일을정리한다', async () => {
const h = createHarness({
precision: async (request) => {
expect(h.temp.get(request.imagePath)?.toString()).toBe('crop-10-20-100-40')
expect(request.maxOutputTokens).toBe(256)
return '정확한 결과'
}
})
await open(h)
await h.service.handleRequest(h.sessionId(), 1, {
type: 'refine-text',
text: '원문',
rect: { x: 10, y: 20, width: 100, height: 40 }
})
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '정확한 결과' }])
expect(h.temp.size).toBe(0)
expect(h.clipboard.text).toBeNull()
})
it('정밀읽기_모델이추가한바깥코드펜스_본문과원문에펜스가없으면제거한다', async () => {
const h = createHarness({ precision: async () => '```javascript\nconst x = 1;\n```' })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: 'const x = 1;', rect: null })
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: 'const x = 1;' }])
})
it('정밀읽기_원문에코드펜스가있으면_제거하지않는다', async () => {
const text = '```javascript\nconst x = 1;\n```'
const h = createHarness({ precision: async () => text })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text, rect: null })
expect(h.responses).toEqual([{ requestId: 1, ok: true, text }])
})
it('정밀읽기_비정상적으로긴 반복결과_실패로처리하고임시파일을정리한다', async () => {
const h = createHarness({ precision: async () => 'x'.repeat(4001) })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '짧음', rect: null })
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'recognition' }])
expect(h.temp.size).toBe(0)
})
it('정밀읽기_요청이진행중일때_중복요청을거부한다', async () => {
let finish: ((text: string) => void) | undefined
const precision = vi.fn(() => new Promise<string>((resolve) => { finish = resolve }))
const h = createHarness({ precision })
await open(h)
const first = h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '첫 요청', rect: null })
const second = h.service.handleRequest(h.sessionId(), 2, { type: 'refine-text', text: '둘째 요청', rect: null })
await second
finish?.('첫 결과')
await first
expect(precision).toHaveBeenCalledTimes(1)
expect(h.responses).toContainEqual({ requestId: 2, ok: false, error: 'invalid' })
expect(h.responses).toContainEqual({ requestId: 1, ok: true, text: '첫 결과' })
})
it('정밀읽기_진행중닫기_응답을버리고임시파일을정리한다', async () => {
let finish: ((text: string) => void) | undefined
const h = createHarness({ precision: () => new Promise<string>((resolve) => { finish = resolve }) })
await open(h)
const pending = h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '원문', rect: null })
h.service.close()
finish?.('늦은 결과')
await pending
expect(h.responses).toEqual([])
expect(h.temp.size).toBe(0)
})
})
describe('Windows OCR 엔진 어댑터', () => {
it('엔진인식_정상출력_정본결과로옮기고입력은환경변수로만넘긴다', async () => {
const calls: Array<Record<string, string>> = []