fix(desktop): add precise local OCR reading
This commit is contained in:
parent
59cdddbecb
commit
25fd3ac004
30 changed files with 626 additions and 15 deletions
|
|
@ -21,6 +21,10 @@ const OVERLAY_STRING_KEYS = [
|
|||
'modeRect',
|
||||
'copy',
|
||||
'copyLine',
|
||||
'precision',
|
||||
'workingPrecision',
|
||||
'resultPrecision',
|
||||
'errorPrecision',
|
||||
'copyImage',
|
||||
'saveImage',
|
||||
'savePdf',
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
// 창 · 파일 · 클립보드 · LLM · TTS 는 전부 포트로 받는다(Electron 어댑터는 screen-ocr/electron-adapters.ts).
|
||||
//
|
||||
// 사생활: 캡처한 그림은 인식에 쓰는 임시 파일 하나로만 디스크에 닿고, 인식이 끝나면 곧바로 지운다.
|
||||
// 닫으면 메모리의 그림도 놓는다. 어디에도 보내지 않는다(번역·요약은 사용자가 고른 글자만 LLM 으로 간다).
|
||||
// 닫으면 메모리의 그림도 놓는다. 정밀 읽기는 선택 영역의 그림만 루프백 GLM-OCR로 본다.
|
||||
// 원격 Ollama는 차단한다. 어디에도 보내지 않는다(번역·요약은 사용자가 고른 글자만 LLM 으로 간다).
|
||||
|
||||
import { EventEmitter } from 'events'
|
||||
import { randomUUID } from 'crypto'
|
||||
|
|
@ -24,6 +25,8 @@ import {
|
|||
} from '@d3ro/core/screen-ocr'
|
||||
import { getLogger } from './LoggerService'
|
||||
import type { OcrEnginePort } from './screen-ocr/windows-ocr-engine'
|
||||
import type { PrecisionOcrEnginePort } from './screen-ocr/ollama-precision-engine'
|
||||
import { isPrecisionOutputWithinBounds, unwrapOcrCodeFence } from './screen-ocr/precision-output'
|
||||
|
||||
const logger = getLogger('ScreenOcrService')
|
||||
|
||||
|
|
@ -70,6 +73,7 @@ export interface ScreenOcrFilesPort {
|
|||
export interface ScreenOcrDeps {
|
||||
capture: () => Promise<ScreenShot>
|
||||
engine: OcrEnginePort
|
||||
precision: PrecisionOcrEnginePort
|
||||
overlay: ScreenOcrOverlayPort
|
||||
files: ScreenOcrFilesPort
|
||||
/** HTML 파일 → PDF */
|
||||
|
|
@ -130,6 +134,7 @@ export type ScreenOcrRequest =
|
|||
| { type: 'copy-image'; rect: OcrRect | null }
|
||||
| { type: 'save-image'; rect: OcrRect | null }
|
||||
| { type: 'save-pdf'; rect: OcrRect | null }
|
||||
| { type: 'refine-text'; text: string; rect: OcrRect | null }
|
||||
| { type: 'translate'; text: string }
|
||||
| { type: 'summarize'; text: string }
|
||||
| { type: 'speak'; text: string }
|
||||
|
|
@ -150,6 +155,7 @@ interface Session {
|
|||
abort: AbortController
|
||||
/** 인식을 다시 돌릴 때마다 올린다 — 늦게 끝난 옛 인식 결과를 버린다 */
|
||||
recognitionRound: number
|
||||
precisionPending: boolean
|
||||
/** 이 캡처에서 읽어 주기를 시작했는지 — 닫을 때 그 읽기만 멈춘다(다른 기능의 재생은 건드리지 않는다) */
|
||||
spoke: boolean
|
||||
}
|
||||
|
|
@ -230,6 +236,7 @@ export class ScreenOcrService extends EventEmitter {
|
|||
result: null,
|
||||
abort: new AbortController(),
|
||||
recognitionRound: 0,
|
||||
precisionPending: false,
|
||||
spoke: false
|
||||
}
|
||||
this._session = session
|
||||
|
|
@ -373,6 +380,8 @@ export class ScreenOcrService extends EventEmitter {
|
|||
}
|
||||
case 'save-pdf':
|
||||
return this._savePdf(session, request.rect)
|
||||
case 'refine-text':
|
||||
return this._refineText(session, request)
|
||||
case 'translate': {
|
||||
const text = this._requireText(request.text).slice(0, SCREEN_OCR_MAX_LLM_CHARS)
|
||||
const target = pickTranslationTarget(
|
||||
|
|
@ -426,6 +435,48 @@ export class ScreenOcrService extends EventEmitter {
|
|||
}
|
||||
}
|
||||
|
||||
private async _refineText(
|
||||
session: Session,
|
||||
request: Extract<ScreenOcrRequest, { type: 'refine-text' }>
|
||||
): Promise<{ ok: true; text: string } | { ok: false; error: ScreenOcrErrorKind }> {
|
||||
if (session.precisionPending) return { ok: false, error: 'invalid' }
|
||||
const source = this._requireText(request.text)
|
||||
const area = this._areaOf(session, request.rect)
|
||||
const png = area ? session.shot.crop(area) : session.shot.png
|
||||
session.precisionPending = true
|
||||
let tempPath: string | null = null
|
||||
try {
|
||||
tempPath = await this._deps.files.writeTemp(`${session.id}-precision-${randomUUID()}.png`, png)
|
||||
session.abort.signal.throwIfAborted()
|
||||
const raw = (await this._deps.precision.recognize({
|
||||
imagePath: tempPath,
|
||||
signal: session.abort.signal,
|
||||
maxOutputTokens: Math.min(4096, Math.max(256, Math.ceil(source.length * 2)))
|
||||
})).replace(/\r\n/gu, '\n').replace(/\n+$/u, '')
|
||||
if (!isPrecisionOutputWithinBounds(raw, source)) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'Precision OCR returned an implausibly long result')
|
||||
}
|
||||
const text = unwrapOcrCodeFence(raw, source)
|
||||
if (text.trim().length === 0) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'Precision OCR returned empty text')
|
||||
}
|
||||
return { ok: true, text }
|
||||
} catch (error) {
|
||||
if (error instanceof D3ROError) throw error
|
||||
throw new D3ROError(
|
||||
ErrorCode.ScreenOcrRecognitionFailed,
|
||||
`Precision OCR failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
} finally {
|
||||
session.precisionPending = false
|
||||
if (tempPath) {
|
||||
await this._deps.files.remove(tempPath).catch((error: unknown) => {
|
||||
logger.warn(`Precision OCR temp file cleanup failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _savePdf(
|
||||
session: Session,
|
||||
rect: OcrRect | null
|
||||
|
|
@ -507,6 +558,7 @@ export class ScreenOcrService extends EventEmitter {
|
|||
case 'search-web':
|
||||
return 'link'
|
||||
case 'set-language':
|
||||
case 'refine-text':
|
||||
return 'recognition'
|
||||
default:
|
||||
return 'export'
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import type {
|
|||
ScreenShot
|
||||
} from '../ScreenOcrService'
|
||||
import { createWindowsOcrEngine } from './windows-ocr-engine'
|
||||
import { createOllamaPrecisionOcrEngine } from './ollama-precision-engine'
|
||||
|
||||
const logger = getLogger('ScreenOcrAdapters')
|
||||
|
||||
|
|
@ -213,6 +214,7 @@ export function createScreenOcrDeps(
|
|||
return {
|
||||
capture: captureDisplayAtCursor,
|
||||
engine: createWindowsOcrEngine(),
|
||||
precision: createOllamaPrecisionOcrEngine(),
|
||||
overlay,
|
||||
files: createFilesPort(dialogTitle, guard),
|
||||
renderPdf: renderHtmlFileToPdf,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
import { readFile } from 'fs/promises'
|
||||
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
||||
import { getLocalLLMService } from '../LocalLLMService'
|
||||
import { getOllamaServerUrl } from '../LocalLLMService'
|
||||
import { getLogger } from '../LoggerService'
|
||||
import { readNdjsonLines } from '../../utils/ndjson-reader'
|
||||
|
||||
const MODEL = 'glm-ocr:latest'
|
||||
const GENERATE_TIMEOUT_MS = 120_000
|
||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
||||
const MAX_OUTPUT_TOKENS = 4096
|
||||
|
||||
export interface PrecisionOcrRequest {
|
||||
imagePath: string
|
||||
signal?: AbortSignal
|
||||
maxOutputTokens: number
|
||||
}
|
||||
|
||||
export interface PrecisionOcrEnginePort {
|
||||
recognize(request: PrecisionOcrRequest): Promise<string>
|
||||
}
|
||||
|
||||
interface Deps {
|
||||
serverUrl?: string
|
||||
model?: string
|
||||
fetchFn?: typeof fetch
|
||||
ensureRunning?: () => Promise<unknown>
|
||||
readImage?: (path: string) => Promise<Buffer>
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function loopbackOnly(url: string): boolean {
|
||||
try {
|
||||
return ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(new URL(url).hostname)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function timeoutSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
|
||||
return signal ? AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs)
|
||||
}
|
||||
|
||||
export function createOllamaPrecisionOcrEngine(deps: Deps = {}): PrecisionOcrEnginePort {
|
||||
const serverUrl = (deps.serverUrl ?? getOllamaServerUrl()).replace(/\/$/u, '')
|
||||
const model = deps.model ?? MODEL
|
||||
const fetchFn = deps.fetchFn ?? fetch
|
||||
const ensureRunning = deps.ensureRunning ?? (() => getLocalLLMService().ensureRunning())
|
||||
const readImage = deps.readImage ?? readFile
|
||||
const logger = getLogger('OllamaPrecisionOcr')
|
||||
|
||||
const request = async (path: string, init: RequestInit, signal: AbortSignal): Promise<Response> => {
|
||||
try {
|
||||
return await fetchFn(`${serverUrl}${path}`, { ...init, signal })
|
||||
} catch (error) {
|
||||
throw new D3ROError(
|
||||
ErrorCode.ScreenOcrRecognitionFailed,
|
||||
`Local precision OCR request failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const hasModel = async (signal?: AbortSignal): Promise<boolean> => {
|
||||
const response = await request('/api/tags', { method: 'GET' }, timeoutSignal(signal, 10_000))
|
||||
if (!response.ok) return false
|
||||
const body: unknown = await response.json()
|
||||
if (!isRecord(body) || !Array.isArray(body.models)) return false
|
||||
return body.models.some((entry) => isRecord(entry) && entry.name === model)
|
||||
}
|
||||
|
||||
const pullModel = async (signal?: AbortSignal): Promise<void> => {
|
||||
logger.info(`GLM-OCR 모델 다운로드 시작: ${model}`)
|
||||
const response = await request(
|
||||
'/api/pull',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model, stream: true })
|
||||
},
|
||||
timeoutSignal(signal, DOWNLOAD_TIMEOUT_MS)
|
||||
)
|
||||
if (!response.ok || !response.body) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrUnavailable, `GLM-OCR download failed with HTTP ${response.status}`)
|
||||
}
|
||||
const reader = response.body.getReader()
|
||||
const cancel = (): void => {
|
||||
void reader.cancel().catch(() => undefined)
|
||||
}
|
||||
signal?.addEventListener('abort', cancel, { once: true })
|
||||
let succeeded = false
|
||||
try {
|
||||
for await (const line of readNdjsonLines(reader)) {
|
||||
if (line.length > 64 * 1024) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR download returned an oversized status line')
|
||||
}
|
||||
const chunk: unknown = JSON.parse(line)
|
||||
if (!isRecord(chunk)) continue
|
||||
if (typeof chunk.error === 'string') {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrUnavailable, `GLM-OCR download failed: ${chunk.error}`)
|
||||
}
|
||||
if (chunk.status === 'success') succeeded = true
|
||||
}
|
||||
} finally {
|
||||
signal?.removeEventListener('abort', cancel)
|
||||
reader.releaseLock()
|
||||
}
|
||||
if (!succeeded) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrUnavailable, 'GLM-OCR download ended without success')
|
||||
}
|
||||
logger.info('GLM-OCR 모델 다운로드 완료')
|
||||
}
|
||||
|
||||
return {
|
||||
async recognize({ imagePath, signal, maxOutputTokens }): Promise<string> {
|
||||
try {
|
||||
if (!loopbackOnly(serverUrl)) {
|
||||
throw new D3ROError(
|
||||
ErrorCode.ScreenOcrUnavailable,
|
||||
'Precision OCR requires a loopback Ollama server so screenshots stay on this PC'
|
||||
)
|
||||
}
|
||||
await ensureRunning()
|
||||
if (!(await hasModel(signal))) await pullModel(signal)
|
||||
const image = await readImage(imagePath)
|
||||
const numPredict = Math.min(MAX_OUTPUT_TOKENS, Math.max(256, Math.floor(maxOutputTokens)))
|
||||
const response = await request(
|
||||
'/api/generate',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
prompt: 'Text Recognition:',
|
||||
images: [image.toString('base64')],
|
||||
stream: false,
|
||||
keep_alive: '2m',
|
||||
options: { temperature: 0, num_ctx: 8192, num_predict: numPredict }
|
||||
})
|
||||
},
|
||||
timeoutSignal(signal, GENERATE_TIMEOUT_MS)
|
||||
)
|
||||
if (!response.ok) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, `GLM-OCR returned HTTP ${response.status}`)
|
||||
}
|
||||
const body: unknown = await response.json()
|
||||
if (!isRecord(body) || body.done !== true || body.done_reason !== 'stop' || typeof body.response !== 'string') {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR returned an incomplete or truncated result')
|
||||
}
|
||||
if (body.response.trim().length === 0) {
|
||||
throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR returned empty text')
|
||||
}
|
||||
return body.response
|
||||
} catch (error) {
|
||||
if (error instanceof D3ROError) throw error
|
||||
throw new D3ROError(
|
||||
ErrorCode.ScreenOcrRecognitionFailed,
|
||||
`Precision OCR failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
export function unwrapOcrCodeFence(raw: string, source: string): string {
|
||||
const lines = raw.split('\n')
|
||||
if (lines.length < 3 || source.includes('```')) return raw
|
||||
if (!/^```[A-Za-z0-9_+.-]*$/u.test(lines[0].trim()) || lines[lines.length - 1].trim() !== '```') return raw
|
||||
const body = lines.slice(1, -1).join('\n')
|
||||
return body.includes('```') ? raw : body
|
||||
}
|
||||
|
||||
export function isPrecisionOutputWithinBounds(raw: string, source: string): boolean {
|
||||
return Array.from(raw).length <= Math.max(4000, Array.from(source).length * 8)
|
||||
}
|
||||
|
|
@ -31,6 +31,7 @@
|
|||
<div id="actions" class="d3-panel bar" role="toolbar">
|
||||
<button id="act-copy" class="act primary" type="button" data-i18n="copy"></button>
|
||||
<button id="act-copy-line" class="act" type="button" data-i18n="copyLine"></button>
|
||||
<button id="act-refine" class="act" type="button" data-i18n="precision" data-i18n-label="workingPrecision"></button>
|
||||
<button id="act-translate" class="act" type="button" data-i18n="translate"></button>
|
||||
<button id="act-summarize" class="act" type="button" data-i18n="summarize"></button>
|
||||
<button id="act-speak" class="act" type="button" data-i18n="speak"></button>
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ const state = {
|
|||
region: null,
|
||||
drag: null,
|
||||
speaking: false,
|
||||
precisionRunning: false,
|
||||
i18n: {},
|
||||
nextRequestId: 1,
|
||||
pending: new Map(),
|
||||
|
|
@ -193,6 +194,7 @@ function renderActions() {
|
|||
for (const id of ['act-copy', 'act-copy-line', 'act-translate', 'act-summarize', 'act-speak', 'act-link']) {
|
||||
$(id).disabled = !hasText
|
||||
}
|
||||
$('act-refine').disabled = !hasText || state.precisionRunning
|
||||
$('act-select-all').disabled = !state.result || state.result.words.length === 0
|
||||
$('act-link').textContent = hasText && detectLinks(text).length > 0 ? tr('openLink') : tr('search')
|
||||
$('act-speak').textContent = state.speaking ? tr('stopSpeak') : tr('speak')
|
||||
|
|
@ -460,6 +462,27 @@ async function runLlm(type, title) {
|
|||
}
|
||||
}
|
||||
|
||||
async function refineText() {
|
||||
const text = selectedText(false)
|
||||
if (!text.trim() || state.precisionRunning) return
|
||||
state.precisionRunning = true
|
||||
renderActions()
|
||||
openResult(tr('resultPrecision'))
|
||||
resultBody.textContent = tr('workingPrecision')
|
||||
const response = await request({ type: 'refine-text', text, rect: exportRegion() })
|
||||
state.precisionRunning = false
|
||||
renderActions()
|
||||
if (resultEl.hidden) return
|
||||
if (response.ok && typeof response.text === 'string') {
|
||||
resultBody.textContent = response.text
|
||||
resultBody.className = 'result-body'
|
||||
$('result-copy').disabled = false
|
||||
} else {
|
||||
resultBody.textContent = tr('errorPrecision')
|
||||
resultBody.className = 'result-body error'
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleSpeak() {
|
||||
if (state.speaking) {
|
||||
state.speaking = false
|
||||
|
|
@ -524,6 +547,7 @@ function resetSession() {
|
|||
state.region = null
|
||||
state.drag = null
|
||||
state.speaking = false
|
||||
state.precisionRunning = false
|
||||
shot.removeAttribute('src')
|
||||
toastEl.hidden = true
|
||||
closeResult()
|
||||
|
|
@ -647,6 +671,7 @@ click('flip', () => {
|
|||
})
|
||||
click('act-copy', () => void copyText(false))
|
||||
click('act-copy-line', () => void copyText(true))
|
||||
click('act-refine', () => void refineText())
|
||||
click('act-translate', () => void runLlm('translate', tr('resultTranslate')))
|
||||
click('act-summarize', () => void runLlm('summarize', tr('resultSummary')))
|
||||
click('act-speak', () => void toggleSpeak())
|
||||
|
|
|
|||
113
apps/desktop/tests/main/services/ollama-precision-engine.test.ts
Normal file
113
apps/desktop/tests/main/services/ollama-precision-engine.test.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { ErrorCode } from '@d3ro/core/errors'
|
||||
import { createOllamaPrecisionOcrEngine } from '../../../src/main/services/screen-ocr/ollama-precision-engine'
|
||||
|
||||
type Handler = (url: string, init?: RequestInit) => Promise<Response>
|
||||
|
||||
function json(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } })
|
||||
}
|
||||
|
||||
function harness(handler: Handler) {
|
||||
const calls: string[] = []
|
||||
const fetchFn: typeof fetch = async (input, init) => {
|
||||
const url = String(input)
|
||||
calls.push(url)
|
||||
return handler(url, init)
|
||||
}
|
||||
const ensureRunning = vi.fn(async () => 'running')
|
||||
const engine = createOllamaPrecisionOcrEngine({
|
||||
serverUrl: 'http://127.0.0.1:11434',
|
||||
fetchFn,
|
||||
ensureRunning,
|
||||
readImage: async () => Buffer.from('image')
|
||||
})
|
||||
return { engine, calls, ensureRunning }
|
||||
}
|
||||
|
||||
const modelPresent = json({ models: [{ name: 'glm-ocr:latest' }] })
|
||||
const generated = (text: string, reason = 'stop') => json({ done: true, done_reason: reason, response: text })
|
||||
|
||||
describe('Ollama 정밀 OCR 엔진', () => {
|
||||
it('정밀인식_모델이있으면_다운로드없이공식프롬프트와로컬이미지로호출한다', async () => {
|
||||
const requests: Array<{ url: string; body: Record<string, unknown> }> = []
|
||||
const { engine, calls } = harness(async (url, init) => {
|
||||
if (url.endsWith('/api/tags')) return modelPresent
|
||||
if (url.endsWith('/api/generate')) {
|
||||
requests.push({ url, body: JSON.parse(String(init?.body)) as Record<string, unknown> })
|
||||
return generated('정확한 결과')
|
||||
}
|
||||
throw new Error(`unexpected ${url}`)
|
||||
})
|
||||
const result = await engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 1024 })
|
||||
expect(result).toBe('정확한 결과')
|
||||
expect(calls.some((url) => url.endsWith('/api/pull'))).toBe(false)
|
||||
expect(requests[0].body).toMatchObject({
|
||||
model: 'glm-ocr:latest',
|
||||
prompt: 'Text Recognition:',
|
||||
images: [Buffer.from('image').toString('base64')],
|
||||
stream: false,
|
||||
keep_alive: '2m',
|
||||
options: { temperature: 0, num_ctx: 8192, num_predict: 1024 }
|
||||
})
|
||||
})
|
||||
|
||||
it('정밀인식_모델이없으면_다운로드를완료한뒤인식한다', async () => {
|
||||
const { engine, calls } = harness(async (url) => {
|
||||
if (url.endsWith('/api/tags')) return json({ models: [] })
|
||||
if (url.endsWith('/api/pull')) return new Response('{"status":"downloading"}\n{"status":"success"}\n')
|
||||
if (url.endsWith('/api/generate')) return generated('결과')
|
||||
throw new Error(`unexpected ${url}`)
|
||||
})
|
||||
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).resolves.toBe('결과')
|
||||
expect(calls.findIndex((url) => url.endsWith('/api/pull'))).toBeLessThan(calls.findIndex((url) => url.endsWith('/api/generate')))
|
||||
})
|
||||
|
||||
it('정밀인식_토큰한도로잘린결과_성공으로처리하지않는다', async () => {
|
||||
const { engine } = harness(async (url) => url.endsWith('/api/tags') ? modelPresent : generated('불완전한 결과', 'length'))
|
||||
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrRecognitionFailed
|
||||
})
|
||||
})
|
||||
|
||||
it('정밀인식_빈결과_성공으로처리하지않는다', async () => {
|
||||
const { engine } = harness(async (url) => url.endsWith('/api/tags') ? modelPresent : generated(' \n'))
|
||||
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrRecognitionFailed
|
||||
})
|
||||
})
|
||||
|
||||
it('정밀인식_원격Ollama주소_화면을볼내지않고차단한다', async () => {
|
||||
const fetchFn = vi.fn(async () => json({}))
|
||||
const engine = createOllamaPrecisionOcrEngine({
|
||||
serverUrl: 'https://ollama.example.com',
|
||||
fetchFn,
|
||||
ensureRunning: async () => 'running',
|
||||
readImage: async () => Buffer.from('image')
|
||||
})
|
||||
await expect(engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrUnavailable
|
||||
})
|
||||
expect(fetchFn).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('정밀인식_다운로드중취소_인식을시작하지않는다', async () => {
|
||||
const controller = new AbortController()
|
||||
let pullStarted = false
|
||||
const { engine, calls } = harness(async (url, init) => {
|
||||
if (url.endsWith('/api/tags')) return json({ models: [] })
|
||||
if (url.endsWith('/api/pull')) {
|
||||
pullStarted = true
|
||||
return new Promise<Response>((_resolve, reject) => {
|
||||
init?.signal?.addEventListener('abort', () => reject(new Error('aborted')), { once: true })
|
||||
})
|
||||
}
|
||||
return generated('실행하면 안 되는 결과')
|
||||
})
|
||||
const pending = engine.recognize({ imagePath: 'crop.png', maxOutputTokens: 512, signal: controller.signal })
|
||||
await vi.waitFor(() => expect(pullStarted).toBe(true))
|
||||
controller.abort()
|
||||
await expect(pending).rejects.toMatchObject({ code: ErrorCode.ScreenOcrRecognitionFailed })
|
||||
expect(calls.some((url) => url.endsWith('/api/generate'))).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
@ -47,6 +47,7 @@ interface HarnessOptions {
|
|||
/** 인식을 곧바로 끝낼지 (기본 true). false 면 finishRecognition 을 부를 때까지 걸려 있다. */
|
||||
autoRecognize?: boolean
|
||||
openExternal?: (url: string) => Promise<boolean>
|
||||
precision?: ScreenOcrDeps['precision']['recognize']
|
||||
translate?: ScreenOcrDeps['llm']['translate']
|
||||
renderPdf?: ScreenOcrDeps['renderPdf']
|
||||
clipboardError?: Error
|
||||
|
|
@ -93,6 +94,7 @@ function createHarness(options: HarnessOptions = {}): Harness {
|
|||
})
|
||||
}
|
||||
},
|
||||
precision: { recognize: options.precision ?? (async () => '정밀 결과') },
|
||||
overlay: {
|
||||
show: (bounds, payload) => shown.push({ bounds, payload }),
|
||||
sendRecognized: (payload) => recognized.push(payload),
|
||||
|
|
@ -472,6 +474,77 @@ describe('ScreenOcrService 요청 처리', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('GLM-OCR 선택 영역 정밀 읽기', () => {
|
||||
it('정밀읽기_선택영역_크롭만모델에볼내고임시파일을정리한다', async () => {
|
||||
const h = createHarness({
|
||||
precision: async (request) => {
|
||||
expect(h.temp.get(request.imagePath)?.toString()).toBe('crop-10-20-100-40')
|
||||
expect(request.maxOutputTokens).toBe(256)
|
||||
return '정확한 결과'
|
||||
}
|
||||
})
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, {
|
||||
type: 'refine-text',
|
||||
text: '원문',
|
||||
rect: { x: 10, y: 20, width: 100, height: 40 }
|
||||
})
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '정확한 결과' }])
|
||||
expect(h.temp.size).toBe(0)
|
||||
expect(h.clipboard.text).toBeNull()
|
||||
})
|
||||
|
||||
it('정밀읽기_모델이추가한바깥코드펜스_본문과원문에펜스가없으면제거한다', async () => {
|
||||
const h = createHarness({ precision: async () => '```javascript\nconst x = 1;\n```' })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: 'const x = 1;', rect: null })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: 'const x = 1;' }])
|
||||
})
|
||||
|
||||
it('정밀읽기_원문에코드펜스가있으면_제거하지않는다', async () => {
|
||||
const text = '```javascript\nconst x = 1;\n```'
|
||||
const h = createHarness({ precision: async () => text })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text, rect: null })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, text }])
|
||||
})
|
||||
|
||||
it('정밀읽기_비정상적으로긴 반복결과_실패로처리하고임시파일을정리한다', async () => {
|
||||
const h = createHarness({ precision: async () => 'x'.repeat(4001) })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '짧음', rect: null })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'recognition' }])
|
||||
expect(h.temp.size).toBe(0)
|
||||
})
|
||||
|
||||
it('정밀읽기_요청이진행중일때_중복요청을거부한다', async () => {
|
||||
let finish: ((text: string) => void) | undefined
|
||||
const precision = vi.fn(() => new Promise<string>((resolve) => { finish = resolve }))
|
||||
const h = createHarness({ precision })
|
||||
await open(h)
|
||||
const first = h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '첫 요청', rect: null })
|
||||
const second = h.service.handleRequest(h.sessionId(), 2, { type: 'refine-text', text: '둘째 요청', rect: null })
|
||||
await second
|
||||
finish?.('첫 결과')
|
||||
await first
|
||||
expect(precision).toHaveBeenCalledTimes(1)
|
||||
expect(h.responses).toContainEqual({ requestId: 2, ok: false, error: 'invalid' })
|
||||
expect(h.responses).toContainEqual({ requestId: 1, ok: true, text: '첫 결과' })
|
||||
})
|
||||
|
||||
it('정밀읽기_진행중닫기_응답을버리고임시파일을정리한다', async () => {
|
||||
let finish: ((text: string) => void) | undefined
|
||||
const h = createHarness({ precision: () => new Promise<string>((resolve) => { finish = resolve }) })
|
||||
await open(h)
|
||||
const pending = h.service.handleRequest(h.sessionId(), 1, { type: 'refine-text', text: '원문', rect: null })
|
||||
h.service.close()
|
||||
finish?.('늦은 결과')
|
||||
await pending
|
||||
expect(h.responses).toEqual([])
|
||||
expect(h.temp.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Windows OCR 엔진 어댑터', () => {
|
||||
it('엔진인식_정상출력_정본결과로옮기고입력은환경변수로만넘긴다', async () => {
|
||||
const calls: Array<Record<string, string>> = []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue