feat(desktop): screen text capture (OCR) overlay with selectable text
A rebindable screen-ocr action (default Ctrl+Alt+O) freezes the monitor under the cursor and covers it with a focusable overlay. Text is selected by flow drag, rectangle drag, double/triple click, Shift+click or Ctrl+A, then copied, saved as PNG or searchable PDF, translated, summarized, read aloud, opened as a link or searched. - Selection, text assembly and the PDF document are pure functions in @d3ro/core/screen-ocr, shared by the main process and the overlay. - Recognition uses Windows.Media.Ocr through powershell.exe -EncodedCommand with 2x upscaling; the capture reaches disk only as a temp PNG that is removed right after recognition. - Capture uses GDI BitBlt for the target monitor (39 ms) because desktopCapturer captured all three monitors and took 2.9 s when DXGI duplication timed out. - The accelerator is reserved only where the feature is available, so macOS does not swallow the key for a feature it cannot run (fails closed with a message). Map: CAP-17 [~], GAP-OCR-01 (live hotkey journey not yet driven in the running app), GAP-OCR-02 (macOS engine, mixed-language accuracy).
This commit is contained in:
parent
6791e29f27
commit
03cd8bf890
41 changed files with 4561 additions and 17 deletions
279
packages/core/__tests__/screen-ocr.test.ts
Normal file
279
packages/core/__tests__/screen-ocr.test.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
allWords,
|
||||
buildSearchablePdfHtml,
|
||||
chooseOcrScale,
|
||||
clampRect,
|
||||
detectLinks,
|
||||
flowSelection,
|
||||
lineSelection,
|
||||
nearestWordIndex,
|
||||
parseOcrEngineOutput,
|
||||
pickTranslationTarget,
|
||||
rectFromPoints,
|
||||
selectionBounds,
|
||||
selectionHighlightRects,
|
||||
selectionText,
|
||||
wordAtPoint,
|
||||
wordsInRect,
|
||||
wordsRelativeTo,
|
||||
type OcrResult
|
||||
} from '../src/screen-ocr'
|
||||
|
||||
/** 한 줄에 단어를 폭 40, 간격 10 으로 늘어놓는다. */
|
||||
function line(y: number, texts: string[], startX = 10): { words: Array<Record<string, unknown>> } {
|
||||
return { words: texts.map((t, i) => ({ t, x: startX + i * 50, y, w: 40, h: 20 })) }
|
||||
}
|
||||
|
||||
function sample(): OcrResult {
|
||||
const result = parseOcrEngineOutput({
|
||||
lang: 'ko',
|
||||
lines: [line(10, ['화면', 'OCR', '기능']), line(50, ['둘째', '줄']), line(90, ['Third', 'line', 'here'])]
|
||||
})
|
||||
if (!result) throw new Error('sample did not parse')
|
||||
return result
|
||||
}
|
||||
|
||||
describe('parseOcrEngineOutput', () => {
|
||||
it('엔진출력해석_정상입력_읽는순서로단어번호와줄범위를매긴다', () => {
|
||||
const result = sample()
|
||||
expect(result.language).toBe('ko')
|
||||
expect(result.words.map((w) => w.text)).toEqual(['화면', 'OCR', '기능', '둘째', '줄', 'Third', 'line', 'here'])
|
||||
expect(result.words.map((w) => w.index)).toEqual([0, 1, 2, 3, 4, 5, 6, 7])
|
||||
expect(result.lines.map((l) => [l.start, l.count])).toEqual([
|
||||
[0, 3],
|
||||
[3, 2],
|
||||
[5, 3]
|
||||
])
|
||||
expect(result.lines[0]).toMatchObject({ x: 10, y: 10, width: 140, height: 20 })
|
||||
})
|
||||
|
||||
it('엔진출력해석_모양이틀린최상위_null을돌려줘빈결과로꾸미지않는다', () => {
|
||||
expect(parseOcrEngineOutput(null)).toBeNull()
|
||||
expect(parseOcrEngineOutput({ lang: 'ko' })).toBeNull()
|
||||
expect(parseOcrEngineOutput({ lines: 'nope' })).toBeNull()
|
||||
})
|
||||
|
||||
it('엔진출력해석_망가진단어와빈줄_버리고나머지번호는이어진다', () => {
|
||||
const result = parseOcrEngineOutput({
|
||||
lang: 'en',
|
||||
lines: [
|
||||
{ words: [{ t: '', x: 0, y: 0, w: 10, h: 10 }, { t: 'ok', x: 'NaN', y: 0, w: 10, h: 10 }] },
|
||||
{ words: [{ t: 'zero', x: 0, y: 0, w: 0, h: 10 }, { t: 'kept', x: 5, y: 5, w: 10, h: 10 }] },
|
||||
'garbage'
|
||||
]
|
||||
})
|
||||
expect(result?.words.map((w) => [w.text, w.index, w.line])).toEqual([['kept', 0, 0]])
|
||||
expect(result?.lines).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('사각형 선택', () => {
|
||||
it('사각형선택_중심이든단어만_살짝스친옆줄은고르지않는다', () => {
|
||||
const result = sample()
|
||||
// 첫 줄 전체를 덮고 둘째 줄(y 50~70) 윗부분 4px 만 스친다.
|
||||
expect(wordsInRect(result, { x: 0, y: 0, width: 200, height: 54 })).toEqual([0, 1, 2])
|
||||
})
|
||||
|
||||
it('사각형선택_절반넘게걸친단어만_절반이안되면고르지않는다', () => {
|
||||
const result = sample()
|
||||
// 첫 단어(x 10~50)의 왼쪽 45% 만 덮으면 고르지 않는다. 높이 20 중 12 만 덮어도 폭을 다 덮으면(60%) 고른다.
|
||||
expect(wordsInRect(result, { x: 0, y: 0, width: 28, height: 40 })).toEqual([])
|
||||
expect(wordsInRect(result, { x: 0, y: 12, width: 60, height: 12 })).toEqual([0])
|
||||
})
|
||||
|
||||
it('rectFromPoints_거꾸로끌어도_양수크기사각형이된다', () => {
|
||||
expect(rectFromPoints({ x: 100, y: 80 }, { x: 20, y: 30 })).toEqual({ x: 20, y: 30, width: 80, height: 50 })
|
||||
})
|
||||
|
||||
it('clampRect_그림밖으로나간영역_안쪽으로자르고완전히밖이면null', () => {
|
||||
expect(clampRect({ x: -10, y: -5, width: 50.4, height: 30 }, { width: 100, height: 20 })).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 41,
|
||||
height: 20
|
||||
})
|
||||
expect(clampRect({ x: 200, y: 0, width: 50, height: 10 }, { width: 100, height: 100 })).toBeNull()
|
||||
expect(clampRect({ x: 10, y: 10, width: 0, height: 0 }, { width: 100, height: 100 })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('글 흐름 선택', () => {
|
||||
it('wordAtPoint_단어위와빈곳_단어번호또는null', () => {
|
||||
const result = sample()
|
||||
expect(wordAtPoint(result, { x: 70, y: 20 })).toBe(1)
|
||||
expect(wordAtPoint(result, { x: 300, y: 300 })).toBeNull()
|
||||
})
|
||||
|
||||
it('nearestWordIndex_줄사이여백과줄오른쪽바깥_가까운줄의가까운단어를고른다', () => {
|
||||
const result = sample()
|
||||
// 둘째 줄(y 50~70) 오른쪽 바깥 → 둘째 줄의 마지막 단어
|
||||
expect(nearestWordIndex(result, { x: 400, y: 60 })).toBe(4)
|
||||
// 셋째 줄 아래 여백 → 셋째 줄에서 가로로 가장 가까운 단어
|
||||
expect(nearestWordIndex(result, { x: 65, y: 400 })).toBe(6)
|
||||
})
|
||||
|
||||
it('nearestWordIndex_단어가없는결과_null', () => {
|
||||
const empty = parseOcrEngineOutput({ lang: 'ko', lines: [] })
|
||||
expect(empty && nearestWordIndex(empty, { x: 0, y: 0 })).toBeNull()
|
||||
})
|
||||
|
||||
it('flowSelection_뒤에서앞으로끌어도_사이단어를전부고른다', () => {
|
||||
expect(flowSelection(6, 2)).toEqual([2, 3, 4, 5, 6])
|
||||
expect(flowSelection(3, 3)).toEqual([3])
|
||||
})
|
||||
|
||||
it('lineSelection과allWords_줄전체와전체단어', () => {
|
||||
const result = sample()
|
||||
expect(lineSelection(result, 4)).toEqual([3, 4])
|
||||
expect(lineSelection(result, 99)).toEqual([])
|
||||
expect(allWords(result)).toHaveLength(8)
|
||||
})
|
||||
})
|
||||
|
||||
describe('선택 → 텍스트', () => {
|
||||
it('selectionText_여러줄선택_줄바꿈을유지하고단어는공백으로잇는다', () => {
|
||||
const result = sample()
|
||||
expect(selectionText(result, [1, 2, 3, 4, 5])).toBe('OCR 기능\n둘째 줄\nThird')
|
||||
})
|
||||
|
||||
it('selectionText_한줄옵션_줄바꿈없이잇는다', () => {
|
||||
const result = sample()
|
||||
expect(selectionText(result, [2, 3, 4], { singleLine: true })).toBe('기능 둘째 줄')
|
||||
})
|
||||
|
||||
it('selectionText_순서가뒤섞이고겹치고범위밖인번호_정리해서읽는순서로낸다', () => {
|
||||
const result = sample()
|
||||
expect(selectionText(result, [4, 0, 0, 99, -1, 3])).toBe('화면\n둘째 줄')
|
||||
})
|
||||
|
||||
it('selectionText_일본어중국어처럼글자마다단어인줄_공백없이잇는다', () => {
|
||||
const result = parseOcrEngineOutput({
|
||||
lang: 'ja',
|
||||
lines: [line(0, ['日', '本', '語', 'OCR', 'テ', 'ス', 'ト']), line(40, ['次', 'の', '行'])]
|
||||
})
|
||||
if (!result) throw new Error('no result')
|
||||
expect(selectionText(result, allWords(result))).toBe('日本語 OCR テスト\n次の行')
|
||||
expect(selectionText(result, allWords(result), { singleLine: true })).toBe('日本語 OCR テスト次の行')
|
||||
})
|
||||
|
||||
it('selectionText_아무것도안고름_빈문자열', () => {
|
||||
expect(selectionText(sample(), [])).toBe('')
|
||||
})
|
||||
})
|
||||
|
||||
describe('강조 사각형과 경계', () => {
|
||||
it('selectionHighlightRects_같은줄에서이어진단어_하나로잇고끊긴곳은나눈다', () => {
|
||||
const result = sample()
|
||||
const rects = selectionHighlightRects(result, [0, 1, 3, 5, 7])
|
||||
expect(rects).toEqual([
|
||||
{ x: 10, y: 10, width: 90, height: 20 },
|
||||
{ x: 10, y: 50, width: 40, height: 20 },
|
||||
{ x: 10, y: 90, width: 40, height: 20 },
|
||||
{ x: 110, y: 90, width: 40, height: 20 }
|
||||
])
|
||||
})
|
||||
|
||||
it('selectionBounds_고른단어를모두감싼다_없으면null', () => {
|
||||
const result = sample()
|
||||
expect(selectionBounds(result, [1, 4])).toEqual({ x: 60, y: 10, width: 40, height: 60 })
|
||||
expect(selectionBounds(result, [])).toBeNull()
|
||||
})
|
||||
|
||||
it('wordsRelativeTo_잘라낸영역기준좌표로옮긴다', () => {
|
||||
const result = sample()
|
||||
const moved = wordsRelativeTo(result, { x: 50, y: 40, width: 200, height: 40 })
|
||||
expect(moved.map((w) => [w.text, w.x, w.y])).toEqual([['줄', 10, 10]])
|
||||
})
|
||||
})
|
||||
|
||||
describe('chooseOcrScale', () => {
|
||||
it('인식배율_100퍼센트화면_2배로키운다', () => {
|
||||
expect(chooseOcrScale(1920, 1080, 1, 10000)).toBe(2)
|
||||
})
|
||||
|
||||
it('인식배율_고배율화면_이미큰글자는덜키운다', () => {
|
||||
expect(chooseOcrScale(2880, 1800, 1.5, 10000)).toBe(1.33)
|
||||
expect(chooseOcrScale(3840, 2160, 2, 10000)).toBe(1)
|
||||
})
|
||||
|
||||
it('인식배율_엔진최대변길이를넘지않는다', () => {
|
||||
const scale = chooseOcrScale(7680, 2160, 1, 10000)
|
||||
expect(scale).toBe(1.3)
|
||||
expect(7680 * scale).toBeLessThanOrEqual(10000)
|
||||
})
|
||||
|
||||
it('인식배율_배율값이이상하면_1배율화면으로본다', () => {
|
||||
expect(chooseOcrScale(1000, 1000, Number.NaN, 10000)).toBe(2)
|
||||
expect(chooseOcrScale(0, 0, 1, 10000)).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('detectLinks', () => {
|
||||
it('링크찾기_주소와메일_열수있는URL로돌려준다', () => {
|
||||
expect(detectLinks('보기: https://example.com/path?q=1, 그리고 www.d3ro.io. 문의 help@example.com')).toEqual([
|
||||
'mailto:help@example.com',
|
||||
'https://example.com/path?q=1',
|
||||
'https://www.d3ro.io'
|
||||
])
|
||||
})
|
||||
|
||||
it('링크찾기_주소가없는글_빈목록', () => {
|
||||
expect(detectLinks('그냥 문장입니다 3.14 e.g.')).toEqual([])
|
||||
})
|
||||
|
||||
it('링크찾기_같은주소가여러번_한번만', () => {
|
||||
expect(detectLinks('https://a.io https://a.io')).toEqual(['https://a.io'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('pickTranslationTarget', () => {
|
||||
it('번역대상_외국어글_사용자의앱언어로옮긴다', () => {
|
||||
expect(pickTranslationTarget('The quick brown fox', 'ko', 'English')).toBe('Korean')
|
||||
expect(pickTranslationTarget('안녕하세요 반갑습니다', 'en', 'Japanese')).toBe('English')
|
||||
})
|
||||
|
||||
it('번역대상_이미사용자언어인글_설정의번역대상언어로옮긴다', () => {
|
||||
expect(pickTranslationTarget('안녕하세요 반갑습니다', 'ko', 'English')).toBe('English')
|
||||
expect(pickTranslationTarget('Hello there, friend', 'en', 'Japanese')).toBe('Japanese')
|
||||
expect(pickTranslationTarget('こんにちは、世界', 'ja', 'English')).toBe('English')
|
||||
})
|
||||
|
||||
it('번역대상_모르는앱언어_설정값을그대로쓴다', () => {
|
||||
expect(pickTranslationTarget('hello', 'xx', 'German')).toBe('German')
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildSearchablePdfHtml', () => {
|
||||
const html = buildSearchablePdfHtml({
|
||||
imageDataUrl: 'data:image/png;base64,AAAA',
|
||||
width: 800,
|
||||
height: 600,
|
||||
language: 'ko',
|
||||
words: [
|
||||
{ text: '화면 <b>&"', x: 10, y: 20, width: 100, height: 20 },
|
||||
{ text: ' ', x: 0, y: 0, width: 10, height: 10 },
|
||||
{ text: 'zero', x: 0, y: 0, width: 0, height: 10 }
|
||||
]
|
||||
})
|
||||
|
||||
it('PDF문서_쪽크기가그림크기와같다', () => {
|
||||
expect(html).toContain('@page{size:800px 600px;margin:0}')
|
||||
expect(html).toContain('viewBox="0 0 800 600"')
|
||||
})
|
||||
|
||||
it('PDF문서_글자층이그림아래에깔린다', () => {
|
||||
expect(html.indexOf('<svg')).toBeLessThan(html.indexOf('<img'))
|
||||
expect(html).toContain('.page svg{z-index:0}.page img{z-index:1}')
|
||||
})
|
||||
|
||||
it('PDF문서_단어상자폭에맞춘글자를넣고특수문자는이스케이프한다', () => {
|
||||
expect(html).toContain('textLength="100.0"')
|
||||
expect(html).toContain('화면 <b>&"')
|
||||
expect(html).not.toContain('<b>')
|
||||
})
|
||||
|
||||
it('PDF문서_빈글자와크기없는단어_글자층에넣지않는다', () => {
|
||||
expect(html.match(/<text /gu)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue