feat(desktop): screen text capture (OCR) overlay with selectable text
A rebindable screen-ocr action (default Ctrl+Alt+O) freezes the monitor under the cursor and covers it with a focusable overlay. Text is selected by flow drag, rectangle drag, double/triple click, Shift+click or Ctrl+A, then copied, saved as PNG or searchable PDF, translated, summarized, read aloud, opened as a link or searched. - Selection, text assembly and the PDF document are pure functions in @d3ro/core/screen-ocr, shared by the main process and the overlay. - Recognition uses Windows.Media.Ocr through powershell.exe -EncodedCommand with 2x upscaling; the capture reaches disk only as a temp PNG that is removed right after recognition. - Capture uses GDI BitBlt for the target monitor (39 ms) because desktopCapturer captured all three monitors and took 2.9 s when DXGI duplication timed out. - The accelerator is reserved only where the feature is available, so macOS does not swallow the key for a feature it cannot run (fails closed with a message). Map: CAP-17 [~], GAP-OCR-01 (live hotkey journey not yet driven in the running app), GAP-OCR-02 (macOS engine, mixed-language accuracy).
This commit is contained in:
parent
6791e29f27
commit
03cd8bf890
41 changed files with 4561 additions and 17 deletions
|
|
@ -70,6 +70,18 @@ describe('acceleratorsToReserve', () => {
|
|||
).toEqual(['Control+Shift+V', 'Control+Shift+C'])
|
||||
})
|
||||
|
||||
it('화면OCR단축키예약_쓸수없는환경_키만삼키지않게예약하지않고쓸수있을때만예약한다', () => {
|
||||
const reserved = (screenOcrAvailable: boolean | undefined): string[] =>
|
||||
acceleratorsToReserve(defaultReservable(), {
|
||||
platform: 'darwin',
|
||||
flags: { suggestionEnabled: false, screenOcrAvailable }
|
||||
})
|
||||
expect(reserved(undefined)).not.toContain('Control+Alt+O')
|
||||
expect(reserved(false)).not.toContain('Control+Alt+O')
|
||||
expect(reserved(true)).toContain('Control+Alt+O')
|
||||
expect(isKeyBindingActionFeatureEnabled('screen-ocr', { suggestionEnabled: true })).toBe(false)
|
||||
})
|
||||
|
||||
it('macOS 에서 제안 기능이 켜져 있으면 suggestion-* 바인딩도 예약한다', () => {
|
||||
expect(
|
||||
acceleratorsToReserve(defaultReservable(), { platform: 'darwin', flags: { suggestionEnabled: true } })
|
||||
|
|
|
|||
561
apps/desktop/tests/main/services/screen-ocr-service.test.ts
Normal file
561
apps/desktop/tests/main/services/screen-ocr-service.test.ts
Normal file
|
|
@ -0,0 +1,561 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
||||
import { parseOcrEngineOutput, type OcrRect, type OcrResult } from '@d3ro/core/screen-ocr'
|
||||
import {
|
||||
ScreenOcrService,
|
||||
type ScreenOcrDeps,
|
||||
type ScreenOcrRecognizedPayload,
|
||||
type ScreenOcrResponse,
|
||||
type ScreenOcrSessionPayload
|
||||
} from '../../../src/main/services/ScreenOcrService'
|
||||
import { createWindowsOcrEngine } from '../../../src/main/services/screen-ocr/windows-ocr-engine'
|
||||
|
||||
function ocrResult(): OcrResult {
|
||||
const result = parseOcrEngineOutput({
|
||||
lang: 'ko',
|
||||
lines: [
|
||||
{ words: [{ t: '화면', x: 10, y: 10, w: 40, h: 20 }, { t: 'OCR', x: 60, y: 10, w: 40, h: 20 }] },
|
||||
{ words: [{ t: '둘째', x: 10, y: 50, w: 40, h: 20 }] }
|
||||
]
|
||||
})
|
||||
if (!result) throw new Error('fixture did not parse')
|
||||
return result
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
service: ScreenOcrService
|
||||
shown: Array<{ bounds: OcrRect; payload: ScreenOcrSessionPayload }>
|
||||
recognized: ScreenOcrRecognizedPayload[]
|
||||
responses: ScreenOcrResponse[]
|
||||
hidden: { count: number }
|
||||
temp: Map<string, Buffer | string>
|
||||
written: Map<string, Buffer>
|
||||
clipboard: { text: string | null; image: Buffer | null }
|
||||
crops: OcrRect[]
|
||||
opened: string[]
|
||||
config: { language: string }
|
||||
recognizeCalls: Array<{ imagePath: string; language: string; scale: number }>
|
||||
finishRecognition: (value: OcrResult | Error) => void
|
||||
sessionId: () => string
|
||||
deps: ScreenOcrDeps
|
||||
}
|
||||
|
||||
interface HarnessOptions {
|
||||
supported?: boolean
|
||||
savePath?: string | null
|
||||
captureError?: Error
|
||||
/** 인식을 곧바로 끝낼지 (기본 true). false 면 finishRecognition 을 부를 때까지 걸려 있다. */
|
||||
autoRecognize?: boolean
|
||||
openExternal?: (url: string) => Promise<boolean>
|
||||
translate?: ScreenOcrDeps['llm']['translate']
|
||||
renderPdf?: ScreenOcrDeps['renderPdf']
|
||||
clipboardError?: Error
|
||||
}
|
||||
|
||||
function createHarness(options: HarnessOptions = {}): Harness {
|
||||
const shown: Harness['shown'] = []
|
||||
const recognized: ScreenOcrRecognizedPayload[] = []
|
||||
const responses: ScreenOcrResponse[] = []
|
||||
const hidden = { count: 0 }
|
||||
const temp = new Map<string, Buffer | string>()
|
||||
const written = new Map<string, Buffer>()
|
||||
const clipboard: Harness['clipboard'] = { text: null, image: null }
|
||||
const crops: OcrRect[] = []
|
||||
const opened: string[] = []
|
||||
const config = { language: 'auto' }
|
||||
const recognizeCalls: Harness['recognizeCalls'] = []
|
||||
let pendingRecognition: Array<(value: OcrResult | Error) => void> = []
|
||||
|
||||
const deps: ScreenOcrDeps = {
|
||||
capture: async () => {
|
||||
if (options.captureError) throw options.captureError
|
||||
return {
|
||||
png: Buffer.from('full-png'),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
scaleFactor: 1,
|
||||
displayBounds: { x: 0, y: 0, width: 1920, height: 1080 },
|
||||
crop: (rect) => {
|
||||
crops.push(rect)
|
||||
return Buffer.from(`crop-${rect.x}-${rect.y}-${rect.width}-${rect.height}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
engine: {
|
||||
maxDimension: 10000,
|
||||
isSupported: () => options.supported ?? true,
|
||||
listLanguages: async () => ['ko', 'en-US'],
|
||||
recognize: (request) => {
|
||||
recognizeCalls.push({ imagePath: request.imagePath, language: request.language, scale: request.scale })
|
||||
if (options.autoRecognize ?? true) return Promise.resolve(ocrResult())
|
||||
return new Promise<OcrResult>((resolve, reject) => {
|
||||
pendingRecognition.push((value) => (value instanceof Error ? reject(value) : resolve(value)))
|
||||
})
|
||||
}
|
||||
},
|
||||
overlay: {
|
||||
show: (bounds, payload) => shown.push({ bounds, payload }),
|
||||
sendRecognized: (payload) => recognized.push(payload),
|
||||
respond: (payload) => responses.push(payload),
|
||||
hide: () => {
|
||||
hidden.count += 1
|
||||
}
|
||||
},
|
||||
files: {
|
||||
writeTemp: async (name, data) => {
|
||||
const path = `/tmp/${name}`
|
||||
temp.set(path, data)
|
||||
return path
|
||||
},
|
||||
remove: async (path) => {
|
||||
temp.delete(path)
|
||||
},
|
||||
askSavePath: async () => (options.savePath === undefined ? '/out/file' : options.savePath),
|
||||
write: async (path, data) => {
|
||||
written.set(path, data)
|
||||
}
|
||||
},
|
||||
renderPdf: options.renderPdf ?? (async () => Buffer.from('%PDF')),
|
||||
clipboard: {
|
||||
writeText: async (text) => {
|
||||
if (options.clipboardError) throw options.clipboardError
|
||||
clipboard.text = text
|
||||
},
|
||||
writeImage: async (png) => {
|
||||
clipboard.image = png
|
||||
}
|
||||
},
|
||||
llm: {
|
||||
translate: options.translate ?? (async (text, target) => `[${target}] ${text}`),
|
||||
summarize: async (text) => `요약: ${text}`
|
||||
},
|
||||
tts: { speak: vi.fn(async () => undefined), stop: vi.fn() },
|
||||
openExternal:
|
||||
options.openExternal ??
|
||||
(async (url) => {
|
||||
opened.push(url)
|
||||
return true
|
||||
}),
|
||||
config: {
|
||||
getOcrLanguage: () => config.language,
|
||||
setOcrLanguage: (language) => {
|
||||
config.language = language
|
||||
},
|
||||
getAppLocale: () => 'ko',
|
||||
getTranslationTarget: () => 'English'
|
||||
},
|
||||
strings: () => ({ copy: '복사' }),
|
||||
now: () => new Date(2026, 10, 20, 14, 5, 9).getTime()
|
||||
}
|
||||
|
||||
return {
|
||||
service: new ScreenOcrService(deps),
|
||||
shown,
|
||||
recognized,
|
||||
responses,
|
||||
hidden,
|
||||
temp,
|
||||
written,
|
||||
clipboard,
|
||||
crops,
|
||||
opened,
|
||||
config,
|
||||
recognizeCalls,
|
||||
finishRecognition: (value) => {
|
||||
const waiting = pendingRecognition
|
||||
pendingRecognition = []
|
||||
for (const finish of waiting) finish(value)
|
||||
},
|
||||
sessionId: () => shown[shown.length - 1].payload.sessionId,
|
||||
deps
|
||||
}
|
||||
}
|
||||
|
||||
/** 시작하고 인식이 끝나 오버레이에 결과가 갈 때까지 기다린다. */
|
||||
async function open(h: Harness): Promise<void> {
|
||||
await h.service.start()
|
||||
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
|
||||
}
|
||||
|
||||
describe('ScreenOcrService 캡처 수명주기', () => {
|
||||
it('화면OCR시작_캡처성공_오버레이를화면크기로띄우고인식결과를보낸다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
|
||||
expect(h.shown).toHaveLength(1)
|
||||
expect(h.shown[0].bounds).toEqual({ x: 0, y: 0, width: 1920, height: 1080 })
|
||||
expect(h.shown[0].payload).toMatchObject({ width: 1920, height: 1080, language: 'auto', _i18n: { copy: '복사' } })
|
||||
expect(h.recognized[0]).toMatchObject({ ok: true, languages: ['ko', 'en-US'] })
|
||||
expect(h.recognizeCalls[0]).toMatchObject({ language: 'auto', scale: 2 })
|
||||
expect(h.service.isOpen).toBe(true)
|
||||
})
|
||||
|
||||
it('화면OCR시작_인식이끝난뒤_캡처임시파일이디스크에남지않는다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
expect(h.recognizeCalls[0].imagePath).toMatch(/\.png$/u)
|
||||
expect(h.temp.size).toBe(0)
|
||||
})
|
||||
|
||||
it('화면OCR시작_인식이실패해도_캡처임시파일을지우고실패를오버레이에알린다', async () => {
|
||||
const h = createHarness({ autoRecognize: false })
|
||||
await h.service.start()
|
||||
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
|
||||
h.finishRecognition(new Error('engine crashed'))
|
||||
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
|
||||
|
||||
expect(h.recognized[0]).toMatchObject({ ok: false, error: 'recognition' })
|
||||
expect(h.temp.size).toBe(0)
|
||||
})
|
||||
|
||||
it('화면OCR시작_설치된인식언어가없음_쓸수없음으로알린다', async () => {
|
||||
const h = createHarness({ autoRecognize: false })
|
||||
await h.service.start()
|
||||
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
|
||||
h.finishRecognition(new D3ROError(ErrorCode.ScreenOcrUnavailable, 'no language'))
|
||||
await vi.waitFor(() => expect(h.recognized).toHaveLength(1))
|
||||
expect(h.recognized[0]).toMatchObject({ ok: false, error: 'unavailable' })
|
||||
})
|
||||
|
||||
it('화면OCR시작_지원하지않는플랫폼_오버레이를띄우지않고ScreenOcrUnavailable을던진다', async () => {
|
||||
const h = createHarness({ supported: false })
|
||||
await expect(h.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrUnavailable })
|
||||
expect(h.shown).toHaveLength(0)
|
||||
expect(h.service.isOpen).toBe(false)
|
||||
})
|
||||
|
||||
it('화면OCR시작_캡처실패_ScreenOcrCaptureFailed를던지고열린상태로남지않는다', async () => {
|
||||
const failing = createHarness({ captureError: new Error('no source') })
|
||||
await expect(failing.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrCaptureFailed })
|
||||
expect(failing.shown).toHaveLength(0)
|
||||
expect(failing.service.isOpen).toBe(false)
|
||||
})
|
||||
|
||||
it('화면OCR시작_이미열려있을때다시시작_새캡처를겹쳐만들지않는다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.start()
|
||||
expect(h.shown).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('화면OCR토글_열려있을때단축키_닫고오버레이를숨긴다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.toggle()
|
||||
expect(h.service.isOpen).toBe(false)
|
||||
expect(h.hidden.count).toBe(1)
|
||||
})
|
||||
|
||||
it('화면OCR닫기_인식이끝나기전에닫음_늦게온결과를오버레이에보내지않는다', async () => {
|
||||
const h = createHarness({ autoRecognize: false })
|
||||
await h.service.start()
|
||||
await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1))
|
||||
h.service.close()
|
||||
h.finishRecognition(ocrResult())
|
||||
await vi.waitFor(() => expect(h.temp.size).toBe(0))
|
||||
expect(h.recognized).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('화면OCR닫기_읽어주기를쓰지않은캡처_다른기능의음성재생을멈추지않는다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
h.service.close()
|
||||
expect(h.deps.tts.stop).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('화면OCR닫기_읽어주는중에닫음_읽기를멈춘다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'speak', text: '읽어 줘' })
|
||||
expect(h.deps.tts.speak).toHaveBeenCalledWith('읽어 줘')
|
||||
h.service.close()
|
||||
expect(h.deps.tts.stop).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('ScreenOcrService 요청 처리', () => {
|
||||
it('글자복사_선택한글_클립보드에쓰고성공으로응답한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 7, { type: 'copy-text', text: '화면 OCR\n둘째' })
|
||||
expect(h.clipboard.text).toBe('화면 OCR\n둘째')
|
||||
expect(h.responses).toEqual([{ requestId: 7, ok: true }])
|
||||
})
|
||||
|
||||
it('글자복사_클립보드쓰기실패_성공으로꾸미지않고실패로응답한다', async () => {
|
||||
const h = createHarness({ clipboardError: new Error('clipboard locked') })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 2, { type: 'copy-text', text: '화면' })
|
||||
expect(h.responses).toEqual([{ requestId: 2, ok: false, error: 'export' }])
|
||||
})
|
||||
|
||||
it('글자복사_빈글_클립보드를건드리지않고선택없음으로거절한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-text', text: ' ' })
|
||||
expect(h.clipboard.text).toBeNull()
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'empty' }])
|
||||
})
|
||||
|
||||
it('요청처리_끝난캡처의요청_아무것도하지않고거절한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
const stale = h.sessionId()
|
||||
h.service.close()
|
||||
await h.service.handleRequest(stale, 3, { type: 'copy-text', text: '늦은 요청' })
|
||||
expect(h.clipboard.text).toBeNull()
|
||||
expect(h.responses).toEqual([{ requestId: 3, ok: false, error: 'invalid' }])
|
||||
})
|
||||
|
||||
it('이미지복사_영역지정_그림밖부분을잘라낸영역만복사한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, {
|
||||
type: 'copy-image',
|
||||
rect: { x: 1900, y: -20, width: 100, height: 120 }
|
||||
})
|
||||
expect(h.crops).toEqual([{ x: 1900, y: 0, width: 20, height: 100 }])
|
||||
expect(h.clipboard.image?.toString()).toBe('crop-1900-0-20-100')
|
||||
})
|
||||
|
||||
it('이미지복사_영역없음_화면전체를복사한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-image', rect: null })
|
||||
expect(h.clipboard.image?.toString()).toBe('full-png')
|
||||
expect(h.crops).toEqual([])
|
||||
})
|
||||
|
||||
it('이미지복사_그림밖영역_복사하지않고거절한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, {
|
||||
type: 'copy-image',
|
||||
rect: { x: 5000, y: 5000, width: 10, height: 10 }
|
||||
})
|
||||
expect(h.clipboard.image).toBeNull()
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }])
|
||||
})
|
||||
|
||||
it('이미지저장_저장위치선택_그림을파일로쓴다', async () => {
|
||||
const h = createHarness({ savePath: '/out/shot.png' })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null })
|
||||
expect(h.written.get('/out/shot.png')?.toString()).toBe('full-png')
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
|
||||
})
|
||||
|
||||
it('이미지저장_대화상자취소_파일을쓰지않고취소로응답한다', async () => {
|
||||
const h = createHarness({ savePath: null })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null })
|
||||
expect(h.written.size).toBe(0)
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, cancelled: true }])
|
||||
})
|
||||
|
||||
it('PDF저장_영역지정_그영역의글자만영역기준좌표로넣은문서를PDF로쓴다', async () => {
|
||||
let html = ''
|
||||
const h = createHarness({
|
||||
savePath: '/out/doc.pdf',
|
||||
renderPdf: async (htmlPath) => {
|
||||
html = String(h.temp.get(htmlPath))
|
||||
return Buffer.from('%PDF-1.7')
|
||||
}
|
||||
})
|
||||
await open(h)
|
||||
// 첫 줄(y 10~30)만 덮는 영역
|
||||
await h.service.handleRequest(h.sessionId(), 1, {
|
||||
type: 'save-pdf',
|
||||
rect: { x: 0, y: 0, width: 200, height: 40 }
|
||||
})
|
||||
|
||||
expect(h.written.get('/out/doc.pdf')?.toString()).toBe('%PDF-1.7')
|
||||
expect(html).toContain('@page{size:200px 40px;margin:0}')
|
||||
expect(html).toContain('>화면</text>')
|
||||
expect(html).toContain('>OCR</text>')
|
||||
expect(html).not.toContain('둘째')
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
|
||||
})
|
||||
|
||||
it('PDF저장_PDF생성실패_임시HTML을지우고내보내기실패로응답한다', async () => {
|
||||
const h = createHarness({
|
||||
renderPdf: async () => {
|
||||
throw new Error('print failed')
|
||||
}
|
||||
})
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'save-pdf', rect: null })
|
||||
expect(h.written.size).toBe(0)
|
||||
expect(h.temp.size).toBe(0)
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'export' }])
|
||||
})
|
||||
|
||||
it('번역_외국어글_앱언어를대상으로번역한결과를돌려준다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'The quick brown fox' })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '[Korean] The quick brown fox' }])
|
||||
})
|
||||
|
||||
it('번역_LLM이빈응답_성공으로꾸미지않고실패로응답한다', async () => {
|
||||
const h = createHarness({ translate: async () => ' ' })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }])
|
||||
})
|
||||
|
||||
it('번역_LLM호출실패_실패종류를llm으로응답한다', async () => {
|
||||
const h = createHarness({
|
||||
translate: async () => {
|
||||
throw new Error('server unreachable')
|
||||
}
|
||||
})
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }])
|
||||
})
|
||||
|
||||
it('번역_처리중에닫음_닫힌오버레이로응답을보내지않는다', async () => {
|
||||
let finish: (text: string) => void = () => undefined
|
||||
const h = createHarness({
|
||||
translate: () =>
|
||||
new Promise<string>((resolve) => {
|
||||
finish = resolve
|
||||
})
|
||||
})
|
||||
await open(h)
|
||||
const pending = h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' })
|
||||
h.service.close()
|
||||
finish('안녕')
|
||||
await pending
|
||||
expect(h.responses).toEqual([])
|
||||
})
|
||||
|
||||
it('요약_선택한글_요약결과를돌려준다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'summarize', text: '긴 글' })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '요약: 긴 글' }])
|
||||
})
|
||||
|
||||
it('웹검색_선택한글_줄바꿈을정리한검색주소를연다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'search-web', text: '화면 OCR\n둘째' })
|
||||
expect(h.opened).toEqual([`https://www.google.com/search?q=${encodeURIComponent('화면 OCR 둘째')}`])
|
||||
})
|
||||
|
||||
it('링크열기_정책이막은주소_링크실패로응답한다', async () => {
|
||||
const h = createHarness({ openExternal: async () => false })
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'open-link', url: 'file:///C:/secret' })
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'link' }])
|
||||
})
|
||||
|
||||
it('인식언어변경_설치된언어_설정을바꾸고그언어로다시인식한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'en-US' })
|
||||
await vi.waitFor(() => expect(h.recognized).toHaveLength(2))
|
||||
expect(h.config.language).toBe('en-US')
|
||||
expect(h.recognizeCalls[1].language).toBe('en-US')
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: true }])
|
||||
})
|
||||
|
||||
it('인식언어변경_설치되지않은언어_설정을바꾸지않고거절한다', async () => {
|
||||
const h = createHarness()
|
||||
await open(h)
|
||||
await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'xx-YY' })
|
||||
expect(h.config.language).toBe('auto')
|
||||
expect(h.recognizeCalls).toHaveLength(1)
|
||||
expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Windows OCR 엔진 어댑터', () => {
|
||||
it('엔진인식_정상출력_정본결과로옮기고입력은환경변수로만넘긴다', async () => {
|
||||
const calls: Array<Record<string, string>> = []
|
||||
const engine = createWindowsOcrEngine({
|
||||
platform: 'win32',
|
||||
run: async (env) => {
|
||||
calls.push(env)
|
||||
return 'WARNING: noise\n{"lang":"ko","lines":[{"words":[{"t":"화면","x":1,"y":2,"w":30,"h":12}]}]}\n'
|
||||
}
|
||||
})
|
||||
const result = await engine.recognize({ imagePath: 'C:\\tmp\\a b.png', language: 'ko', scale: 1.5 })
|
||||
expect(result.words.map((w) => w.text)).toEqual(['화면'])
|
||||
expect(calls[0]).toEqual({
|
||||
D3RO_OCR_MODE: 'recognize',
|
||||
D3RO_OCR_IMAGE: 'C:\\tmp\\a b.png',
|
||||
D3RO_OCR_LANG: 'ko',
|
||||
D3RO_OCR_SCALE: '1.5'
|
||||
})
|
||||
})
|
||||
|
||||
it('엔진인식_줄과단어가하나뿐이라배열이풀린출력_그대로읽는다', async () => {
|
||||
const engine = createWindowsOcrEngine({
|
||||
platform: 'win32',
|
||||
run: async () => '{"lang":"en-US","lines":{"words":{"t":"Hello","x":1,"y":2,"w":30,"h":12}}}'
|
||||
})
|
||||
const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })
|
||||
expect(result.words.map((w) => w.text)).toEqual(['Hello'])
|
||||
expect(result.lines).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('엔진인식_글자가없는화면_빈결과를돌려준다', async () => {
|
||||
const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"lang":"ko","lines":[]}' })
|
||||
const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })
|
||||
expect(result.words).toEqual([])
|
||||
})
|
||||
|
||||
it('엔진인식_설치된언어없음_ScreenOcrUnavailable', async () => {
|
||||
const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"error":"no-language"}' })
|
||||
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrUnavailable
|
||||
})
|
||||
})
|
||||
|
||||
it('엔진인식_엔진예외와해석불가출력_ScreenOcrRecognitionFailed', async () => {
|
||||
const failing = createWindowsOcrEngine({
|
||||
platform: 'win32',
|
||||
run: async () => '{"error":"exception","message":"bad image"}'
|
||||
})
|
||||
await expect(failing.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrRecognitionFailed
|
||||
})
|
||||
const garbage = createWindowsOcrEngine({ platform: 'win32', run: async () => 'not json at all' })
|
||||
await expect(garbage.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrRecognitionFailed
|
||||
})
|
||||
})
|
||||
|
||||
it('엔진인식_프로세스실행실패_ScreenOcrRecognitionFailed', async () => {
|
||||
const engine = createWindowsOcrEngine({
|
||||
platform: 'win32',
|
||||
run: async () => {
|
||||
throw new Error('spawn ENOENT')
|
||||
}
|
||||
})
|
||||
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrRecognitionFailed
|
||||
})
|
||||
})
|
||||
|
||||
it('엔진지원여부_Windows가아님_지원하지않고프로세스를띄우지않는다', async () => {
|
||||
const run = vi.fn(async () => '{}')
|
||||
const engine = createWindowsOcrEngine({ platform: 'darwin', run })
|
||||
expect(engine.isSupported()).toBe(false)
|
||||
expect(await engine.listLanguages()).toEqual([])
|
||||
await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({
|
||||
code: ErrorCode.ScreenOcrUnavailable
|
||||
})
|
||||
expect(run).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('언어목록_하나뿐이라문자열로온출력_목록으로돌려준다', async () => {
|
||||
const single = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":"ko"}' })
|
||||
expect(await single.listLanguages()).toEqual(['ko'])
|
||||
const many = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":["ko","en-US"]}' })
|
||||
expect(await many.listLanguages()).toEqual(['ko', 'en-US'])
|
||||
})
|
||||
})
|
||||
45
apps/desktop/tests/main/windows/popup-window-options.test.ts
Normal file
45
apps/desktop/tests/main/windows/popup-window-options.test.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
BrowserWindow: class {},
|
||||
nativeTheme: { shouldUseDarkColors: true },
|
||||
shell: { openExternal: vi.fn() },
|
||||
app: { getPath: () => '', isPackaged: false }
|
||||
}))
|
||||
|
||||
vi.mock('@electron-toolkit/utils', () => ({ is: { dev: false } }))
|
||||
|
||||
import { buildPopupWindowOptions } from '../../../src/main/windows/popup-window-factory'
|
||||
|
||||
const spec = { name: 'history-popup', bounds: { width: 380, height: 200 }, onClosed: () => undefined }
|
||||
|
||||
describe('buildPopupWindowOptions', () => {
|
||||
it('팝업창옵션_기본팝업_포커스를빼앗지않는투명창이다', () => {
|
||||
const options = buildPopupWindowOptions(spec, '/preload/popup.js')
|
||||
expect(options).toMatchObject({
|
||||
width: 380,
|
||||
height: 200,
|
||||
show: false,
|
||||
frame: false,
|
||||
transparent: true,
|
||||
focusable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true
|
||||
})
|
||||
expect(options.webPreferences).toMatchObject({
|
||||
preload: '/preload/popup.js',
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
backgroundThrottling: false
|
||||
})
|
||||
})
|
||||
|
||||
it('팝업창옵션_화면OCR오버레이_키를받는불투명창이되고보안설정은그대로다', () => {
|
||||
const options = buildPopupWindowOptions(
|
||||
{ ...spec, name: 'screen-ocr', focusable: true, transparent: false },
|
||||
'/preload/popup.js'
|
||||
)
|
||||
expect(options).toMatchObject({ focusable: true, transparent: false, frame: false, alwaysOnTop: true })
|
||||
expect(options.webPreferences).toMatchObject({ contextIsolation: true, nodeIntegration: false })
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue