feat(desktop): screen text capture (OCR) overlay with selectable text

A rebindable screen-ocr action (default Ctrl+Alt+O) freezes the monitor under the cursor and
covers it with a focusable overlay. Text is selected by flow drag, rectangle drag, double/triple
click, Shift+click or Ctrl+A, then copied, saved as PNG or searchable PDF, translated,
summarized, read aloud, opened as a link or searched.

- Selection, text assembly and the PDF document are pure functions in @d3ro/core/screen-ocr,
  shared by the main process and the overlay.
- Recognition uses Windows.Media.Ocr through powershell.exe -EncodedCommand with 2x upscaling;
  the capture reaches disk only as a temp PNG that is removed right after recognition.
- Capture uses GDI BitBlt for the target monitor (39 ms) because desktopCapturer captured all
  three monitors and took 2.9 s when DXGI duplication timed out.
- The accelerator is reserved only where the feature is available, so macOS does not swallow
  the key for a feature it cannot run (fails closed with a message).

Map: CAP-17 [~], GAP-OCR-01 (live hotkey journey not yet driven in the running app),
GAP-OCR-02 (macOS engine, mixed-language accuracy).
This commit is contained in:
Yun Chan 2026-11-20 15:16:37 +09:00
parent 6791e29f27
commit 03cd8bf890
41 changed files with 4561 additions and 17 deletions

View file

@ -0,0 +1,561 @@
import { describe, expect, it, vi } from 'vitest'
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
import { parseOcrEngineOutput, type OcrRect, type OcrResult } from '@d3ro/core/screen-ocr'
import {
ScreenOcrService,
type ScreenOcrDeps,
type ScreenOcrRecognizedPayload,
type ScreenOcrResponse,
type ScreenOcrSessionPayload
} from '../../../src/main/services/ScreenOcrService'
import { createWindowsOcrEngine } from '../../../src/main/services/screen-ocr/windows-ocr-engine'
function ocrResult(): OcrResult {
const result = parseOcrEngineOutput({
lang: 'ko',
lines: [
{ words: [{ t: '화면', x: 10, y: 10, w: 40, h: 20 }, { t: 'OCR', x: 60, y: 10, w: 40, h: 20 }] },
{ words: [{ t: '둘째', x: 10, y: 50, w: 40, h: 20 }] }
]
})
if (!result) throw new Error('fixture did not parse')
return result
}
interface Harness {
service: ScreenOcrService
shown: Array<{ bounds: OcrRect; payload: ScreenOcrSessionPayload }>
recognized: ScreenOcrRecognizedPayload[]
responses: ScreenOcrResponse[]
hidden: { count: number }
temp: Map<string, Buffer | string>
written: Map<string, Buffer>
clipboard: { text: string | null; image: Buffer | null }
crops: OcrRect[]
opened: string[]
config: { language: string }
recognizeCalls: Array<{ imagePath: string; language: string; scale: number }>
finishRecognition: (value: OcrResult | Error) => void
sessionId: () => string
deps: ScreenOcrDeps
}
interface HarnessOptions {
supported?: boolean
savePath?: string | null
captureError?: Error
/** 인식을 곧바로 끝낼지 (기본 true). false 면 finishRecognition 을 부를 때까지 걸려 있다. */
autoRecognize?: boolean
openExternal?: (url: string) => Promise<boolean>
translate?: ScreenOcrDeps['llm']['translate']
renderPdf?: ScreenOcrDeps['renderPdf']
clipboardError?: Error
}
function createHarness(options: HarnessOptions = {}): Harness {
const shown: Harness['shown'] = []
const recognized: ScreenOcrRecognizedPayload[] = []
const responses: ScreenOcrResponse[] = []
const hidden = { count: 0 }
const temp = new Map<string, Buffer | string>()
const written = new Map<string, Buffer>()
const clipboard: Harness['clipboard'] = { text: null, image: null }
const crops: OcrRect[] = []
const opened: string[] = []
const config = { language: 'auto' }
const recognizeCalls: Harness['recognizeCalls'] = []
let pendingRecognition: Array<(value: OcrResult | Error) => void> = []
const deps: ScreenOcrDeps = {
capture: async () => {
if (options.captureError) throw options.captureError
return {
png: Buffer.from('full-png'),
width: 1920,
height: 1080,
scaleFactor: 1,
displayBounds: { x: 0, y: 0, width: 1920, height: 1080 },
crop: (rect) => {
crops.push(rect)
return Buffer.from(`crop-${rect.x}-${rect.y}-${rect.width}-${rect.height}`)
}
}
},
engine: {
maxDimension: 10000,
isSupported: () => options.supported ?? true,
listLanguages: async () => ['ko', 'en-US'],
recognize: (request) => {
recognizeCalls.push({ imagePath: request.imagePath, language: request.language, scale: request.scale })
if (options.autoRecognize ?? true) return Promise.resolve(ocrResult())
return new Promise<OcrResult>((resolve, reject) => {
pendingRecognition.push((value) => (value instanceof Error ? reject(value) : resolve(value)))
})
}
},
overlay: {
show: (bounds, payload) => shown.push({ bounds, payload }),
sendRecognized: (payload) => recognized.push(payload),
respond: (payload) => responses.push(payload),
hide: () => {
hidden.count += 1
}
},
files: {
writeTemp: async (name, data) => {
const path = `/tmp/${name}`
temp.set(path, data)
return path
},
remove: async (path) => {
temp.delete(path)
},
askSavePath: async () => (options.savePath === undefined ? '/out/file' : options.savePath),
write: async (path, data) => {
written.set(path, data)
}
},
renderPdf: options.renderPdf ?? (async () => Buffer.from('%PDF')),
clipboard: {
writeText: async (text) => {
if (options.clipboardError) throw options.clipboardError
clipboard.text = text
},
writeImage: async (png) => {
clipboard.image = png
}
},
llm: {
translate: options.translate ?? (async (text, target) => `[${target}] ${text}`),
summarize: async (text) => `요약: ${text}`
},
tts: { speak: vi.fn(async () => undefined), stop: vi.fn() },
openExternal:
options.openExternal ??
(async (url) => {
opened.push(url)
return true
}),
config: {
getOcrLanguage: () => config.language,
setOcrLanguage: (language) => {
config.language = language
},
getAppLocale: () => 'ko',
getTranslationTarget: () => 'English'
},
strings: () => ({ copy: '복사' }),
now: () => new Date(2026, 10, 20, 14, 5, 9).getTime()
}
return {
service: new ScreenOcrService(deps),
shown,
recognized,
responses,
hidden,
temp,
written,
clipboard,
crops,
opened,
config,
recognizeCalls,
finishRecognition: (value) => {
const waiting = pendingRecognition
pendingRecognition = []
for (const finish of waiting) finish(value)
},
sessionId: () => shown[shown.length - 1].payload.sessionId,
deps
}
}
/** 시작하고 인식이 끝나 오버레이에 결과가 갈 때까지 기다린다. */
async function open(h: Harness): Promise<void> {
await h.service.start()
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
}
describe('ScreenOcrService 캡처 수명주기', () => {
it('화면OCR시작_캡처성공_오버레이를화면크기로띄우고인식결과를보낸다', async () => {
const h = createHarness()
await open(h)
expect(h.shown).toHaveLength(1)
expect(h.shown[0].bounds).toEqual({ x: 0, y: 0, width: 1920, height: 1080 })
expect(h.shown[0].payload).toMatchObject({ width: 1920, height: 1080, language: 'auto', _i18n: { copy: '복사' } })
expect(h.recognized[0]).toMatchObject({ ok: true, languages: ['ko', 'en-US'] })
expect(h.recognizeCalls[0]).toMatchObject({ language: 'auto', scale: 2 })
expect(h.service.isOpen).toBe(true)
})
it('화면OCR시작_인식이끝난뒤_캡처임시파일이디스크에남지않는다', async () => {
const h = createHarness()
await open(h)
expect(h.recognizeCalls[0].imagePath).toMatch(/\.png$/u)
expect(h.temp.size).toBe(0)
})
it('화면OCR시작_인식이실패해도_캡처임시파일을지우고실패를오버레이에알린다', async () => {
const h = createHarness({ autoRecognize: false })
await h.service.start()
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
h.finishRecognition(new Error('engine crashed'))
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
expect(h.recognized[0]).toMatchObject({ ok: false, error: 'recognition' })
expect(h.temp.size).toBe(0)
})
it('화면OCR시작_설치된인식언어가없음_쓸수없음으로알린다', async () => {
const h = createHarness({ autoRecognize: false })
await h.service.start()
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
h.finishRecognition(new D3ROError(ErrorCode.ScreenOcrUnavailable, 'no language'))
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
expect(h.recognized[0]).toMatchObject({ ok: false, error: 'unavailable' })
})
it('화면OCR시작_지원하지않는플랫폼_오버레이를띄우지않고ScreenOcrUnavailable을던진다', async () => {
const h = createHarness({ supported: false })
await expect(h.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrUnavailable })
expect(h.shown).toHaveLength(0)
expect(h.service.isOpen).toBe(false)
})
it('화면OCR시작_캡처실패_ScreenOcrCaptureFailed를던지고열린상태로남지않는다', async () => {
const failing = createHarness({ captureError: new Error('no source') })
await expect(failing.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrCaptureFailed })
expect(failing.shown).toHaveLength(0)
expect(failing.service.isOpen).toBe(false)
})
it('화면OCR시작_이미열려있을때다시시작_새캡처를겹쳐만들지않는다', async () => {
const h = createHarness()
await open(h)
await h.service.start()
expect(h.shown).toHaveLength(1)
})
it('화면OCR토글_열려있을때단축키_닫고오버레이를숨긴다', async () => {
const h = createHarness()
await open(h)
await h.service.toggle()
expect(h.service.isOpen).toBe(false)
expect(h.hidden.count).toBe(1)
})
it('화면OCR닫기_인식이끝나기전에닫음_늦게온결과를오버레이에보내지않는다', async () => {
const h = createHarness({ autoRecognize: false })
await h.service.start()
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
h.service.close()
h.finishRecognition(ocrResult())
await vi.waitFor(() => expect(h.temp.size).toBe(0))
expect(h.recognized).toHaveLength(0)
})
it('화면OCR닫기_읽어주기를쓰지않은캡처_다른기능의음성재생을멈추지않는다', async () => {
const h = createHarness()
await open(h)
h.service.close()
expect(h.deps.tts.stop).not.toHaveBeenCalled()
})
it('화면OCR닫기_읽어주는중에닫음_읽기를멈춘다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'speak', text: '읽어 줘' })
expect(h.deps.tts.speak).toHaveBeenCalledWith('읽어 줘')
h.service.close()
expect(h.deps.tts.stop).toHaveBeenCalledTimes(1)
})
})
describe('ScreenOcrService 요청 처리', () => {
it('글자복사_선택한글_클립보드에쓰고성공으로응답한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 7, { type: 'copy-text', text: '화면 OCR\n둘째' })
expect(h.clipboard.text).toBe('화면 OCR\n둘째')
expect(h.responses).toEqual([{ requestId: 7, ok: true }])
})
it('글자복사_클립보드쓰기실패_성공으로꾸미지않고실패로응답한다', async () => {
const h = createHarness({ clipboardError: new Error('clipboard locked') })
await open(h)
await h.service.handleRequest(h.sessionId(), 2, { type: 'copy-text', text: '화면' })
expect(h.responses).toEqual([{ requestId: 2, ok: false, error: 'export' }])
})
it('글자복사_빈글_클립보드를건드리지않고선택없음으로거절한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-text', text: ' ' })
expect(h.clipboard.text).toBeNull()
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'empty' }])
})
it('요청처리_끝난캡처의요청_아무것도하지않고거절한다', async () => {
const h = createHarness()
await open(h)
const stale = h.sessionId()
h.service.close()
await h.service.handleRequest(stale, 3, { type: 'copy-text', text: '늦은 요청' })
expect(h.clipboard.text).toBeNull()
expect(h.responses).toEqual([{ requestId: 3, ok: false, error: 'invalid' }])
})
it('이미지복사_영역지정_그림밖부분을잘라낸영역만복사한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, {
type: 'copy-image',
rect: { x: 1900, y: -20, width: 100, height: 120 }
})
expect(h.crops).toEqual([{ x: 1900, y: 0, width: 20, height: 100 }])
expect(h.clipboard.image?.toString()).toBe('crop-1900-0-20-100')
})
it('이미지복사_영역없음_화면전체를복사한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-image', rect: null })
expect(h.clipboard.image?.toString()).toBe('full-png')
expect(h.crops).toEqual([])
})
it('이미지복사_그림밖영역_복사하지않고거절한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, {
type: 'copy-image',
rect: { x: 5000, y: 5000, width: 10, height: 10 }
})
expect(h.clipboard.image).toBeNull()
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }])
})
it('이미지저장_저장위치선택_그림을파일로쓴다', async () => {
const h = createHarness({ savePath: '/out/shot.png' })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null })
expect(h.written.get('/out/shot.png')?.toString()).toBe('full-png')
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
})
it('이미지저장_대화상자취소_파일을쓰지않고취소로응답한다', async () => {
const h = createHarness({ savePath: null })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null })
expect(h.written.size).toBe(0)
expect(h.responses).toEqual([{ requestId: 1, ok: true, cancelled: true }])
})
it('PDF저장_영역지정_그영역의글자만영역기준좌표로넣은문서를PDF로쓴다', async () => {
let html = ''
const h = createHarness({
savePath: '/out/doc.pdf',
renderPdf: async (htmlPath) => {
html = String(h.temp.get(htmlPath))
return Buffer.from('%PDF-1.7')
}
})
await open(h)
// 첫 줄(y 10~30)만 덮는 영역
await h.service.handleRequest(h.sessionId(), 1, {
type: 'save-pdf',
rect: { x: 0, y: 0, width: 200, height: 40 }
})
expect(h.written.get('/out/doc.pdf')?.toString()).toBe('%PDF-1.7')
expect(html).toContain('@page{size:200px 40px;margin:0}')
expect(html).toContain('>화면</text>')
expect(html).toContain('>OCR</text>')
expect(html).not.toContain('둘째')
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
})
it('PDF저장_PDF생성실패_임시HTML을지우고내보내기실패로응답한다', async () => {
const h = createHarness({
renderPdf: async () => {
throw new Error('print failed')
}
})
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-pdf', rect: null })
expect(h.written.size).toBe(0)
expect(h.temp.size).toBe(0)
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'export' }])
})
it('번역_외국어글_앱언어를대상으로번역한결과를돌려준다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'The quick brown fox' })
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '[Korean] The quick brown fox' }])
})
it('번역_LLM이빈응답_성공으로꾸미지않고실패로응답한다', async () => {
const h = createHarness({ translate: async () => ' ' })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }])
})
it('번역_LLM호출실패_실패종류를llm으로응답한다', async () => {
const h = createHarness({
translate: async () => {
throw new Error('server unreachable')
}
})
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }])
})
it('번역_처리중에닫음_닫힌오버레이로응답을보내지않는다', async () => {
let finish: (text: string) => void = () => undefined
const h = createHarness({
translate: () =>
new Promise<string>((resolve) => {
finish = resolve
})
})
await open(h)
const pending = h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
h.service.close()
finish('안녕')
await pending
expect(h.responses).toEqual([])
})
it('요약_선택한글_요약결과를돌려준다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'summarize', text: '긴 글' })
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '요약: 긴 글' }])
})
it('웹검색_선택한글_줄바꿈을정리한검색주소를연다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'search-web', text: '화면 OCR\n둘째' })
expect(h.opened).toEqual([`https://www.google.com/search?q=${encodeURIComponent('화면 OCR 둘째')}`])
})
it('링크열기_정책이막은주소_링크실패로응답한다', async () => {
const h = createHarness({ openExternal: async () => false })
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'open-link', url: 'file:///C:/secret' })
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'link' }])
})
it('인식언어변경_설치된언어_설정을바꾸고그언어로다시인식한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'en-US' })
await vi.waitFor(() => expect(h.recognized).toHaveLength(2))
expect(h.config.language).toBe('en-US')
expect(h.recognizeCalls[1].language).toBe('en-US')
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
})
it('인식언어변경_설치되지않은언어_설정을바꾸지않고거절한다', async () => {
const h = createHarness()
await open(h)
await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'xx-YY' })
expect(h.config.language).toBe('auto')
expect(h.recognizeCalls).toHaveLength(1)
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }])
})
})
describe('Windows OCR 엔진 어댑터', () => {
it('엔진인식_정상출력_정본결과로옮기고입력은환경변수로만넘긴다', async () => {
const calls: Array<Record<string, string>> = []
const engine = createWindowsOcrEngine({
platform: 'win32',
run: async (env) => {
calls.push(env)
return 'WARNING: noise\n{"lang":"ko","lines":[{"words":[{"t":"화면","x":1,"y":2,"w":30,"h":12}]}]}\n'
}
})
const result = await engine.recognize({ imagePath: 'C:\\tmp\\a b.png', language: 'ko', scale: 1.5 })
expect(result.words.map((w) => w.text)).toEqual(['화면'])
expect(calls[0]).toEqual({
D3RO_OCR_MODE: 'recognize',
D3RO_OCR_IMAGE: 'C:\\tmp\\a b.png',
D3RO_OCR_LANG: 'ko',
D3RO_OCR_SCALE: '1.5'
})
})
it('엔진인식_줄과단어가하나뿐이라배열이풀린출력_그대로읽는다', async () => {
const engine = createWindowsOcrEngine({
platform: 'win32',
run: async () => '{"lang":"en-US","lines":{"words":{"t":"Hello","x":1,"y":2,"w":30,"h":12}}}'
})
const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })
expect(result.words.map((w) => w.text)).toEqual(['Hello'])
expect(result.lines).toHaveLength(1)
})
it('엔진인식_글자가없는화면_빈결과를돌려준다', async () => {
const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"lang":"ko","lines":[]}' })
const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })
expect(result.words).toEqual([])
})
it('엔진인식_설치된언어없음_ScreenOcrUnavailable', async () => {
const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"error":"no-language"}' })
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrUnavailable
})
})
it('엔진인식_엔진예외와해석불가출력_ScreenOcrRecognitionFailed', async () => {
const failing = createWindowsOcrEngine({
platform: 'win32',
run: async () => '{"error":"exception","message":"bad image"}'
})
await expect(failing.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrRecognitionFailed
})
const garbage = createWindowsOcrEngine({ platform: 'win32', run: async () => 'not json at all' })
await expect(garbage.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrRecognitionFailed
})
})
it('엔진인식_프로세스실행실패_ScreenOcrRecognitionFailed', async () => {
const engine = createWindowsOcrEngine({
platform: 'win32',
run: async () => {
throw new Error('spawn ENOENT')
}
})
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrRecognitionFailed
})
})
it('엔진지원여부_Windows가아님_지원하지않고프로세스를띄우지않는다', async () => {
const run = vi.fn(async () => '{}')
const engine = createWindowsOcrEngine({ platform: 'darwin', run })
expect(engine.isSupported()).toBe(false)
expect(await engine.listLanguages()).toEqual([])
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
code: ErrorCode.ScreenOcrUnavailable
})
expect(run).not.toHaveBeenCalled()
})
it('언어목록_하나뿐이라문자열로온출력_목록으로돌려준다', async () => {
const single = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":"ko"}' })
expect(await single.listLanguages()).toEqual(['ko'])
const many = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":["ko","en-US"]}' })
expect(await many.listLanguages()).toEqual(['ko', 'en-US'])
})
})