From 25fd3ac004b8b5fa861203ebefa41528e481dd57 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Sat, 21 Nov 2026 10:15:49 +0900 Subject: [PATCH] fix(desktop): add precise local OCR reading --- CHANGELOG.md | 4 + apps/desktop/src/main/screen-ocr-wiring.ts | 4 + .../src/main/services/ScreenOcrService.ts | 54 +++++- .../services/screen-ocr/electron-adapters.ts | 2 + .../screen-ocr/ollama-precision-engine.ts | 165 ++++++++++++++++++ .../services/screen-ocr/precision-output.ts | 11 ++ .../src/renderer/popups/screen-ocr/index.html | 1 + .../src/renderer/popups/screen-ocr/script.js | 25 +++ .../services/ollama-precision-engine.test.ts | 113 ++++++++++++ .../main/services/screen-ocr-service.test.ts | 73 ++++++++ .../evidence/2026-11-17-backlog-operations.md | 44 +++++ docs/map/00-index.md | 4 +- docs/map/02-infrastructure.md | 3 +- docs/map/04-desktop-app.md | 4 +- docs/map/10-feature-catalog.md | 4 +- docs/map/11-gap-backlog.md | 4 +- packages/core/__tests__/screen-ocr.test.ts | 58 ++++++ packages/core/src/screen-ocr.ts | 20 ++- packages/i18n/src/locales/de.json | 4 + packages/i18n/src/locales/en.json | 4 + packages/i18n/src/locales/es.json | 4 + packages/i18n/src/locales/fr.json | 4 + packages/i18n/src/locales/ja.json | 4 + packages/i18n/src/locales/ko.json | 4 + packages/i18n/src/locales/pt.json | 4 + packages/i18n/src/locales/ru.json | 4 + packages/i18n/src/locales/th.json | 4 + packages/i18n/src/locales/vi.json | 4 + packages/i18n/src/locales/zh-TW.json | 4 + packages/i18n/src/locales/zh.json | 4 + 30 files changed, 626 insertions(+), 15 deletions(-) create mode 100644 apps/desktop/src/main/services/screen-ocr/ollama-precision-engine.ts create mode 100644 apps/desktop/src/main/services/screen-ocr/precision-output.ts create mode 100644 apps/desktop/tests/main/services/ollama-precision-engine.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bfd03ae..52e49ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Additional Whisper model management UI - Plugin system for custom pipelines +### Fixed +- Screen OCR reconstructs tightly spaced URL, email and punctuation tokens from their pixel positions instead of inserting a space between every recognized word. Ordinary word spacing is preserved; unrecognized glyphs are not guessed or rewritten. +- Add an on-demand precise-reading action to screen OCR. Only the selected crop goes to a loopback GLM-OCR model; the first action downloads about 2.7 GB, original text remains available, and failed, truncated, repeated or cancelled results never replace it. + ## [1.18.4] - 2026-11-20 diff --git a/apps/desktop/src/main/screen-ocr-wiring.ts b/apps/desktop/src/main/screen-ocr-wiring.ts index 4d76638d..a654e9be 100644 --- a/apps/desktop/src/main/screen-ocr-wiring.ts +++ b/apps/desktop/src/main/screen-ocr-wiring.ts @@ -21,6 +21,10 @@ const OVERLAY_STRING_KEYS = [ 'modeRect', 'copy', 'copyLine', + 'precision', + 'workingPrecision', + 'resultPrecision', + 'errorPrecision', 'copyImage', 'saveImage', 'savePdf', diff --git a/apps/desktop/src/main/services/ScreenOcrService.ts b/apps/desktop/src/main/services/ScreenOcrService.ts index e0ef3155..38b46a0d 100644 --- a/apps/desktop/src/main/services/ScreenOcrService.ts +++ b/apps/desktop/src/main/services/ScreenOcrService.ts @@ -8,7 +8,8 @@ // 창 · 파일 · 클립보드 · LLM · TTS 는 전부 포트로 받는다(Electron 어댑터는 screen-ocr/electron-adapters.ts). // // 사생활: 캡처한 그림은 인식에 쓰는 임시 파일 하나로만 디스크에 닿고, 인식이 끝나면 곧바로 지운다. -// 닫으면 메모리의 그림도 놓는다. 어디에도 보내지 않는다(번역·요약은 사용자가 고른 글자만 LLM 으로 간다). +// 닫으면 메모리의 그림도 놓는다. 정밀 읽기는 선택 영역의 그림만 루프백 GLM-OCR로 본다. +// 원격 Ollama는 차단한다. 어디에도 보내지 않는다(번역·요약은 사용자가 고른 글자만 LLM 으로 간다). import { EventEmitter } from 'events' import { randomUUID } from 'crypto' @@ -24,6 +25,8 @@ import { } from '@d3ro/core/screen-ocr' import { getLogger } from './LoggerService' import type { OcrEnginePort } from './screen-ocr/windows-ocr-engine' +import type { PrecisionOcrEnginePort } from './screen-ocr/ollama-precision-engine' +import { isPrecisionOutputWithinBounds, unwrapOcrCodeFence } from './screen-ocr/precision-output' const logger = getLogger('ScreenOcrService') @@ -70,6 +73,7 @@ export interface ScreenOcrFilesPort { export interface ScreenOcrDeps { capture: () => Promise engine: OcrEnginePort + precision: PrecisionOcrEnginePort overlay: ScreenOcrOverlayPort files: ScreenOcrFilesPort /** HTML 파일 → PDF */ @@ -130,6 +134,7 @@ export type ScreenOcrRequest = | { type: 'copy-image'; rect: OcrRect | null } | { type: 'save-image'; rect: OcrRect | null } | { type: 'save-pdf'; rect: OcrRect | null } + | { type: 'refine-text'; text: string; rect: OcrRect | null } | { type: 'translate'; text: string } | { type: 'summarize'; text: string } | { type: 'speak'; text: string } @@ -150,6 +155,7 @@ interface Session { abort: AbortController /** 인식을 다시 돌릴 때마다 올린다 — 늦게 끝난 옛 인식 결과를 버린다 */ recognitionRound: number + precisionPending: boolean /** 이 캡처에서 읽어 주기를 시작했는지 — 닫을 때 그 읽기만 멈춘다(다른 기능의 재생은 건드리지 않는다) */ spoke: boolean } @@ -230,6 +236,7 @@ export class ScreenOcrService extends EventEmitter { result: null, abort: new AbortController(), recognitionRound: 0, + precisionPending: false, spoke: false } this._session = session @@ -373,6 +380,8 @@ export class ScreenOcrService extends EventEmitter { } case 'save-pdf': return this._savePdf(session, request.rect) + case 'refine-text': + return this._refineText(session, request) case 'translate': { const text = this._requireText(request.text).slice(0, SCREEN_OCR_MAX_LLM_CHARS) const target = pickTranslationTarget( @@ -426,6 +435,48 @@ export class ScreenOcrService extends EventEmitter { } } + private async _refineText( + session: Session, + request: Extract + ): Promise<{ ok: true; text: string } | { ok: false; error: ScreenOcrErrorKind }> { + if (session.precisionPending) return { ok: false, error: 'invalid' } + const source = this._requireText(request.text) + const area = this._areaOf(session, request.rect) + const png = area ? session.shot.crop(area) : session.shot.png + session.precisionPending = true + let tempPath: string | null = null + try { + tempPath = await this._deps.files.writeTemp(`${session.id}-precision-${randomUUID()}.png`, png) + session.abort.signal.throwIfAborted() + const raw = (await this._deps.precision.recognize({ + imagePath: tempPath, + signal: session.abort.signal, + maxOutputTokens: Math.min(4096, Math.max(256, Math.ceil(source.length * 2))) + })).replace(/\r\n/gu, '\n').replace(/\n+$/u, '') + if (!isPrecisionOutputWithinBounds(raw, source)) { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'Precision OCR returned an implausibly long result') + } + const text = unwrapOcrCodeFence(raw, source) + if (text.trim().length === 0) { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'Precision OCR returned empty text') + } + return { ok: true, text } + } catch (error) { + if (error instanceof D3ROError) throw error + throw new D3ROError( + ErrorCode.ScreenOcrRecognitionFailed, + `Precision OCR failed: ${error instanceof Error ? error.message : String(error)}` + ) + } finally { + session.precisionPending = false + if (tempPath) { + await this._deps.files.remove(tempPath).catch((error: unknown) => { + logger.warn(`Precision OCR temp file cleanup failed: ${error instanceof Error ? error.message : String(error)}`) + }) + } + } + } + private async _savePdf( session: Session, rect: OcrRect | null @@ -507,6 +558,7 @@ export class ScreenOcrService extends EventEmitter { case 'search-web': return 'link' case 'set-language': + case 'refine-text': return 'recognition' default: return 'export' diff --git a/apps/desktop/src/main/services/screen-ocr/electron-adapters.ts b/apps/desktop/src/main/services/screen-ocr/electron-adapters.ts index 6d557235..f34c187b 100644 --- a/apps/desktop/src/main/services/screen-ocr/electron-adapters.ts +++ b/apps/desktop/src/main/services/screen-ocr/electron-adapters.ts @@ -28,6 +28,7 @@ import type { ScreenShot } from '../ScreenOcrService' import { createWindowsOcrEngine } from './windows-ocr-engine' +import { createOllamaPrecisionOcrEngine } from './ollama-precision-engine' const logger = getLogger('ScreenOcrAdapters') @@ -213,6 +214,7 @@ export function createScreenOcrDeps( return { capture: captureDisplayAtCursor, engine: createWindowsOcrEngine(), + precision: createOllamaPrecisionOcrEngine(), overlay, files: createFilesPort(dialogTitle, guard), renderPdf: renderHtmlFileToPdf, diff --git a/apps/desktop/src/main/services/screen-ocr/ollama-precision-engine.ts b/apps/desktop/src/main/services/screen-ocr/ollama-precision-engine.ts new file mode 100644 index 00000000..bd927fc9 --- /dev/null +++ b/apps/desktop/src/main/services/screen-ocr/ollama-precision-engine.ts @@ -0,0 +1,165 @@ +import { readFile } from 'fs/promises' +import { D3ROError, ErrorCode } from '@d3ro/core/errors' +import { getLocalLLMService } from '../LocalLLMService' +import { getOllamaServerUrl } from '../LocalLLMService' +import { getLogger } from '../LoggerService' +import { readNdjsonLines } from '../../utils/ndjson-reader' + +const MODEL = 'glm-ocr:latest' +const GENERATE_TIMEOUT_MS = 120_000 +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const MAX_OUTPUT_TOKENS = 4096 + +export interface PrecisionOcrRequest { + imagePath: string + signal?: AbortSignal + maxOutputTokens: number +} + +export interface PrecisionOcrEnginePort { + recognize(request: PrecisionOcrRequest): Promise +} + +interface Deps { + serverUrl?: string + model?: string + fetchFn?: typeof fetch + ensureRunning?: () => Promise + readImage?: (path: string) => Promise +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function loopbackOnly(url: string): boolean { + try { + return ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(new URL(url).hostname) + } catch { + return false + } +} + +function timeoutSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + return signal ? AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) +} + +export function createOllamaPrecisionOcrEngine(deps: Deps = {}): PrecisionOcrEnginePort { + const serverUrl = (deps.serverUrl ?? getOllamaServerUrl()).replace(/\/$/u, '') + const model = deps.model ?? MODEL + const fetchFn = deps.fetchFn ?? fetch + const ensureRunning = deps.ensureRunning ?? (() => getLocalLLMService().ensureRunning()) + const readImage = deps.readImage ?? readFile + const logger = getLogger('OllamaPrecisionOcr') + + const request = async (path: string, init: RequestInit, signal: AbortSignal): Promise => { + try { + return await fetchFn(`${serverUrl}${path}`, { ...init, signal }) + } catch (error) { + throw new D3ROError( + ErrorCode.ScreenOcrRecognitionFailed, + `Local precision OCR request failed: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + + const hasModel = async (signal?: AbortSignal): Promise => { + const response = await request('/api/tags', { method: 'GET' }, timeoutSignal(signal, 10_000)) + if (!response.ok) return false + const body: unknown = await response.json() + if (!isRecord(body) || !Array.isArray(body.models)) return false + return body.models.some((entry) => isRecord(entry) && entry.name === model) + } + + const pullModel = async (signal?: AbortSignal): Promise => { + logger.info(`GLM-OCR 모델 다운로드 시작: ${model}`) + const response = await request( + '/api/pull', + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model, stream: true }) + }, + timeoutSignal(signal, DOWNLOAD_TIMEOUT_MS) + ) + if (!response.ok || !response.body) { + throw new D3ROError(ErrorCode.ScreenOcrUnavailable, `GLM-OCR download failed with HTTP ${response.status}`) + } + const reader = response.body.getReader() + const cancel = (): void => { + void reader.cancel().catch(() => undefined) + } + signal?.addEventListener('abort', cancel, { once: true }) + let succeeded = false + try { + for await (const line of readNdjsonLines(reader)) { + if (line.length > 64 * 1024) { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR download returned an oversized status line') + } + const chunk: unknown = JSON.parse(line) + if (!isRecord(chunk)) continue + if (typeof chunk.error === 'string') { + throw new D3ROError(ErrorCode.ScreenOcrUnavailable, `GLM-OCR download failed: ${chunk.error}`) + } + if (chunk.status === 'success') succeeded = true + } + } finally { + signal?.removeEventListener('abort', cancel) + reader.releaseLock() + } + if (!succeeded) { + throw new D3ROError(ErrorCode.ScreenOcrUnavailable, 'GLM-OCR download ended without success') + } + logger.info('GLM-OCR 모델 다운로드 완료') + } + + return { + async recognize({ imagePath, signal, maxOutputTokens }): Promise { + try { + if (!loopbackOnly(serverUrl)) { + throw new D3ROError( + ErrorCode.ScreenOcrUnavailable, + 'Precision OCR requires a loopback Ollama server so screenshots stay on this PC' + ) + } + await ensureRunning() + if (!(await hasModel(signal))) await pullModel(signal) + const image = await readImage(imagePath) + const numPredict = Math.min(MAX_OUTPUT_TOKENS, Math.max(256, Math.floor(maxOutputTokens))) + const response = await request( + '/api/generate', + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model, + prompt: 'Text Recognition:', + images: [image.toString('base64')], + stream: false, + keep_alive: '2m', + options: { temperature: 0, num_ctx: 8192, num_predict: numPredict } + }) + }, + timeoutSignal(signal, GENERATE_TIMEOUT_MS) + ) + if (!response.ok) { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, `GLM-OCR returned HTTP ${response.status}`) + } + const body: unknown = await response.json() + if (!isRecord(body) || body.done !== true || body.done_reason !== 'stop' || typeof body.response !== 'string') { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR returned an incomplete or truncated result') + } + if (body.response.trim().length === 0) { + throw new D3ROError(ErrorCode.ScreenOcrRecognitionFailed, 'GLM-OCR returned empty text') + } + return body.response + } catch (error) { + if (error instanceof D3ROError) throw error + throw new D3ROError( + ErrorCode.ScreenOcrRecognitionFailed, + `Precision OCR failed: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + } +} diff --git a/apps/desktop/src/main/services/screen-ocr/precision-output.ts b/apps/desktop/src/main/services/screen-ocr/precision-output.ts new file mode 100644 index 00000000..865794ed --- /dev/null +++ b/apps/desktop/src/main/services/screen-ocr/precision-output.ts @@ -0,0 +1,11 @@ +export function unwrapOcrCodeFence(raw: string, source: string): string { + const lines = raw.split('\n') + if (lines.length < 3 || source.includes('```')) return raw + if (!/^```[A-Za-z0-9_+.-]*$/u.test(lines[0].trim()) || lines[lines.length - 1].trim() !== '```') return raw + const body = lines.slice(1, -1).join('\n') + return body.includes('```') ? raw : body +} + +export function isPrecisionOutputWithinBounds(raw: string, source: string): boolean { + return Array.from(raw).length <= Math.max(4000, Array.from(source).length * 8) +} diff --git a/apps/desktop/src/renderer/popups/screen-ocr/index.html b/apps/desktop/src/renderer/popups/screen-ocr/index.html index 75cadfc8..7053c3ac 100644 --- a/apps/desktop/src/renderer/popups/screen-ocr/index.html +++ b/apps/desktop/src/renderer/popups/screen-ocr/index.html @@ -31,6 +31,7 @@