d3ro-voice/packages/core/__tests__/screen-ocr.test.ts
Yun Chan 03cd8bf890 feat(desktop): screen text capture (OCR) overlay with selectable text
A rebindable screen-ocr action (default Ctrl+Alt+O) freezes the monitor under the cursor and
covers it with a focusable overlay. Text is selected by flow drag, rectangle drag, double/triple
click, Shift+click or Ctrl+A, then copied, saved as PNG or searchable PDF, translated,
summarized, read aloud, opened as a link or searched.

- Selection, text assembly and the PDF document are pure functions in @d3ro/core/screen-ocr,
  shared by the main process and the overlay.
- Recognition uses Windows.Media.Ocr through powershell.exe -EncodedCommand with 2x upscaling;
  the capture reaches disk only as a temp PNG that is removed right after recognition.
- Capture uses GDI BitBlt for the target monitor (39 ms) because desktopCapturer captured all
  three monitors and took 2.9 s when DXGI duplication timed out.
- The accelerator is reserved only where the feature is available, so macOS does not swallow
  the key for a feature it cannot run (fails closed with a message).

Map: CAP-17 [~], GAP-OCR-01 (live hotkey journey not yet driven in the running app),
GAP-OCR-02 (macOS engine, mixed-language accuracy).
2026-11-20 15:16:37 +09:00

279 lines
11 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import {
allWords,
buildSearchablePdfHtml,
chooseOcrScale,
clampRect,
detectLinks,
flowSelection,
lineSelection,
nearestWordIndex,
parseOcrEngineOutput,
pickTranslationTarget,
rectFromPoints,
selectionBounds,
selectionHighlightRects,
selectionText,
wordAtPoint,
wordsInRect,
wordsRelativeTo,
type OcrResult
} from '../src/screen-ocr'
/** 한 줄에 단어를 폭 40, 간격 10 으로 늘어놓는다. */
function line(y: number, texts: string[], startX = 10): { words: Array<Record<string, unknown>> } {
return { words: texts.map((t, i) => ({ t, x: startX + i * 50, y, w: 40, h: 20 })) }
}
function sample(): OcrResult {
const result = parseOcrEngineOutput({
lang: 'ko',
lines: [line(10, ['화면', 'OCR', '기능']), line(50, ['둘째', '줄']), line(90, ['Third', 'line', 'here'])]
})
if (!result) throw new Error('sample did not parse')
return result
}
describe('parseOcrEngineOutput', () => {
it('엔진출력해석_정상입력_읽는순서로단어번호와줄범위를매긴다', () => {
const result = sample()
expect(result.language).toBe('ko')
expect(result.words.map((w) => w.text)).toEqual(['화면', 'OCR', '기능', '둘째', '줄', 'Third', 'line', 'here'])
expect(result.words.map((w) => w.index)).toEqual([0, 1, 2, 3, 4, 5, 6, 7])
expect(result.lines.map((l) => [l.start, l.count])).toEqual([
[0, 3],
[3, 2],
[5, 3]
])
expect(result.lines[0]).toMatchObject({ x: 10, y: 10, width: 140, height: 20 })
})
it('엔진출력해석_모양이틀린최상위_null을돌려줘빈결과로꾸미지않는다', () => {
expect(parseOcrEngineOutput(null)).toBeNull()
expect(parseOcrEngineOutput({ lang: 'ko' })).toBeNull()
expect(parseOcrEngineOutput({ lines: 'nope' })).toBeNull()
})
it('엔진출력해석_망가진단어와빈줄_버리고나머지번호는이어진다', () => {
const result = parseOcrEngineOutput({
lang: 'en',
lines: [
{ words: [{ t: '', x: 0, y: 0, w: 10, h: 10 }, { t: 'ok', x: 'NaN', y: 0, w: 10, h: 10 }] },
{ words: [{ t: 'zero', x: 0, y: 0, w: 0, h: 10 }, { t: 'kept', x: 5, y: 5, w: 10, h: 10 }] },
'garbage'
]
})
expect(result?.words.map((w) => [w.text, w.index, w.line])).toEqual([['kept', 0, 0]])
expect(result?.lines).toHaveLength(1)
})
})
describe('사각형 선택', () => {
it('사각형선택_중심이든단어만_살짝스친옆줄은고르지않는다', () => {
const result = sample()
// 첫 줄 전체를 덮고 둘째 줄(y 50~70) 윗부분 4px 만 스친다.
expect(wordsInRect(result, { x: 0, y: 0, width: 200, height: 54 })).toEqual([0, 1, 2])
})
it('사각형선택_절반넘게걸친단어만_절반이안되면고르지않는다', () => {
const result = sample()
// 첫 단어(x 10~50)의 왼쪽 45% 만 덮으면 고르지 않는다. 높이 20 중 12 만 덮어도 폭을 다 덮으면(60%) 고른다.
expect(wordsInRect(result, { x: 0, y: 0, width: 28, height: 40 })).toEqual([])
expect(wordsInRect(result, { x: 0, y: 12, width: 60, height: 12 })).toEqual([0])
})
it('rectFromPoints_거꾸로끌어도_양수크기사각형이된다', () => {
expect(rectFromPoints({ x: 100, y: 80 }, { x: 20, y: 30 })).toEqual({ x: 20, y: 30, width: 80, height: 50 })
})
it('clampRect_그림밖으로나간영역_안쪽으로자르고완전히밖이면null', () => {
expect(clampRect({ x: -10, y: -5, width: 50.4, height: 30 }, { width: 100, height: 20 })).toEqual({
x: 0,
y: 0,
width: 41,
height: 20
})
expect(clampRect({ x: 200, y: 0, width: 50, height: 10 }, { width: 100, height: 100 })).toBeNull()
expect(clampRect({ x: 10, y: 10, width: 0, height: 0 }, { width: 100, height: 100 })).toBeNull()
})
})
describe('글 흐름 선택', () => {
it('wordAtPoint_단어위와빈곳_단어번호또는null', () => {
const result = sample()
expect(wordAtPoint(result, { x: 70, y: 20 })).toBe(1)
expect(wordAtPoint(result, { x: 300, y: 300 })).toBeNull()
})
it('nearestWordIndex_줄사이여백과줄오른쪽바깥_가까운줄의가까운단어를고른다', () => {
const result = sample()
// 둘째 줄(y 50~70) 오른쪽 바깥 → 둘째 줄의 마지막 단어
expect(nearestWordIndex(result, { x: 400, y: 60 })).toBe(4)
// 셋째 줄 아래 여백 → 셋째 줄에서 가로로 가장 가까운 단어
expect(nearestWordIndex(result, { x: 65, y: 400 })).toBe(6)
})
it('nearestWordIndex_단어가없는결과_null', () => {
const empty = parseOcrEngineOutput({ lang: 'ko', lines: [] })
expect(empty && nearestWordIndex(empty, { x: 0, y: 0 })).toBeNull()
})
it('flowSelection_뒤에서앞으로끌어도_사이단어를전부고른다', () => {
expect(flowSelection(6, 2)).toEqual([2, 3, 4, 5, 6])
expect(flowSelection(3, 3)).toEqual([3])
})
it('lineSelection과allWords_줄전체와전체단어', () => {
const result = sample()
expect(lineSelection(result, 4)).toEqual([3, 4])
expect(lineSelection(result, 99)).toEqual([])
expect(allWords(result)).toHaveLength(8)
})
})
describe('선택 → 텍스트', () => {
it('selectionText_여러줄선택_줄바꿈을유지하고단어는공백으로잇는다', () => {
const result = sample()
expect(selectionText(result, [1, 2, 3, 4, 5])).toBe('OCR 기능\n둘째 줄\nThird')
})
it('selectionText_한줄옵션_줄바꿈없이잇는다', () => {
const result = sample()
expect(selectionText(result, [2, 3, 4], { singleLine: true })).toBe('기능 둘째 줄')
})
it('selectionText_순서가뒤섞이고겹치고범위밖인번호_정리해서읽는순서로낸다', () => {
const result = sample()
expect(selectionText(result, [4, 0, 0, 99, -1, 3])).toBe('화면\n둘째 줄')
})
it('selectionText_일본어중국어처럼글자마다단어인줄_공백없이잇는다', () => {
const result = parseOcrEngineOutput({
lang: 'ja',
lines: [line(0, ['日', '本', '語', 'OCR', 'テ', 'ス', 'ト']), line(40, ['次', 'の', '行'])]
})
if (!result) throw new Error('no result')
expect(selectionText(result, allWords(result))).toBe('日本語 OCR テスト\n次の行')
expect(selectionText(result, allWords(result), { singleLine: true })).toBe('日本語 OCR テスト次の行')
})
it('selectionText_아무것도안고름_빈문자열', () => {
expect(selectionText(sample(), [])).toBe('')
})
})
describe('강조 사각형과 경계', () => {
it('selectionHighlightRects_같은줄에서이어진단어_하나로잇고끊긴곳은나눈다', () => {
const result = sample()
const rects = selectionHighlightRects(result, [0, 1, 3, 5, 7])
expect(rects).toEqual([
{ x: 10, y: 10, width: 90, height: 20 },
{ x: 10, y: 50, width: 40, height: 20 },
{ x: 10, y: 90, width: 40, height: 20 },
{ x: 110, y: 90, width: 40, height: 20 }
])
})
it('selectionBounds_고른단어를모두감싼다_없으면null', () => {
const result = sample()
expect(selectionBounds(result, [1, 4])).toEqual({ x: 60, y: 10, width: 40, height: 60 })
expect(selectionBounds(result, [])).toBeNull()
})
it('wordsRelativeTo_잘라낸영역기준좌표로옮긴다', () => {
const result = sample()
const moved = wordsRelativeTo(result, { x: 50, y: 40, width: 200, height: 40 })
expect(moved.map((w) => [w.text, w.x, w.y])).toEqual([['줄', 10, 10]])
})
})
describe('chooseOcrScale', () => {
it('인식배율_100퍼센트화면_2배로키운다', () => {
expect(chooseOcrScale(1920, 1080, 1, 10000)).toBe(2)
})
it('인식배율_고배율화면_이미큰글자는덜키운다', () => {
expect(chooseOcrScale(2880, 1800, 1.5, 10000)).toBe(1.33)
expect(chooseOcrScale(3840, 2160, 2, 10000)).toBe(1)
})
it('인식배율_엔진최대변길이를넘지않는다', () => {
const scale = chooseOcrScale(7680, 2160, 1, 10000)
expect(scale).toBe(1.3)
expect(7680 * scale).toBeLessThanOrEqual(10000)
})
it('인식배율_배율값이이상하면_1배율화면으로본다', () => {
expect(chooseOcrScale(1000, 1000, Number.NaN, 10000)).toBe(2)
expect(chooseOcrScale(0, 0, 1, 10000)).toBe(1)
})
})
describe('detectLinks', () => {
it('링크찾기_주소와메일_열수있는URL로돌려준다', () => {
expect(detectLinks('보기: https://example.com/path?q=1, 그리고 www.d3ro.io. 문의 help@example.com')).toEqual([
'mailto:help@example.com',
'https://example.com/path?q=1',
'https://www.d3ro.io'
])
})
it('링크찾기_주소가없는글_빈목록', () => {
expect(detectLinks('그냥 문장입니다 3.14 e.g.')).toEqual([])
})
it('링크찾기_같은주소가여러번_한번만', () => {
expect(detectLinks('https://a.io https://a.io')).toEqual(['https://a.io'])
})
})
describe('pickTranslationTarget', () => {
it('번역대상_외국어글_사용자의앱언어로옮긴다', () => {
expect(pickTranslationTarget('The quick brown fox', 'ko', 'English')).toBe('Korean')
expect(pickTranslationTarget('안녕하세요 반갑습니다', 'en', 'Japanese')).toBe('English')
})
it('번역대상_이미사용자언어인글_설정의번역대상언어로옮긴다', () => {
expect(pickTranslationTarget('안녕하세요 반갑습니다', 'ko', 'English')).toBe('English')
expect(pickTranslationTarget('Hello there, friend', 'en', 'Japanese')).toBe('Japanese')
expect(pickTranslationTarget('こんにちは、世界', 'ja', 'English')).toBe('English')
})
it('번역대상_모르는앱언어_설정값을그대로쓴다', () => {
expect(pickTranslationTarget('hello', 'xx', 'German')).toBe('German')
})
})
describe('buildSearchablePdfHtml', () => {
const html = buildSearchablePdfHtml({
imageDataUrl: 'data:image/png;base64,AAAA',
width: 800,
height: 600,
language: 'ko',
words: [
{ text: '화면 <b>&"', x: 10, y: 20, width: 100, height: 20 },
{ text: ' ', x: 0, y: 0, width: 10, height: 10 },
{ text: 'zero', x: 0, y: 0, width: 0, height: 10 }
]
})
it('PDF문서_쪽크기가그림크기와같다', () => {
expect(html).toContain('@page{size:800px 600px;margin:0}')
expect(html).toContain('viewBox="0 0 800 600"')
})
it('PDF문서_글자층이그림아래에깔린다', () => {
expect(html.indexOf('<svg')).toBeLessThan(html.indexOf('<img'))
expect(html).toContain('.page svg{z-index:0}.page img{z-index:1}')
})
it('PDF문서_단어상자폭에맞춘글자를넣고특수문자는이스케이프한다', () => {
expect(html).toContain('textLength="100.0"')
expect(html).toContain('화면 &lt;b&gt;&amp;&quot;')
expect(html).not.toContain('<b>')
})
it('PDF문서_빈글자와크기없는단어_글자층에넣지않는다', () => {
expect(html.match(/<text /gu)).toHaveLength(1)
})
})