import { describe, expect, it, vi } from 'vitest' import { D3ROError, ErrorCode } from '@d3ro/core/errors' import { parseOcrEngineOutput, type OcrRect, type OcrResult } from '@d3ro/core/screen-ocr' import { ScreenOcrService, type ScreenOcrDeps, type ScreenOcrRecognizedPayload, type ScreenOcrResponse, type ScreenOcrSessionPayload } from '../../../src/main/services/ScreenOcrService' import { createWindowsOcrEngine } from '../../../src/main/services/screen-ocr/windows-ocr-engine' function ocrResult(): OcrResult { const result = parseOcrEngineOutput({ lang: 'ko', lines: [ { words: [{ t: '화면', x: 10, y: 10, w: 40, h: 20 }, { t: 'OCR', x: 60, y: 10, w: 40, h: 20 }] }, { words: [{ t: '둘째', x: 10, y: 50, w: 40, h: 20 }] } ] }) if (!result) throw new Error('fixture did not parse') return result } interface Harness { service: ScreenOcrService shown: Array<{ bounds: OcrRect; payload: ScreenOcrSessionPayload }> recognized: ScreenOcrRecognizedPayload[] responses: ScreenOcrResponse[] hidden: { count: number } temp: Map written: Map clipboard: { text: string | null; image: Buffer | null } crops: OcrRect[] opened: string[] config: { language: string } recognizeCalls: Array<{ imagePath: string; language: string; scale: number }> finishRecognition: (value: OcrResult | Error) => void sessionId: () => string deps: ScreenOcrDeps } interface HarnessOptions { supported?: boolean savePath?: string | null captureError?: Error /** 인식을 곧바로 끝낼지 (기본 true). false 면 finishRecognition 을 부를 때까지 걸려 있다. */ autoRecognize?: boolean openExternal?: (url: string) => Promise translate?: ScreenOcrDeps['llm']['translate'] renderPdf?: ScreenOcrDeps['renderPdf'] clipboardError?: Error } function createHarness(options: HarnessOptions = {}): Harness { const shown: Harness['shown'] = [] const recognized: ScreenOcrRecognizedPayload[] = [] const responses: ScreenOcrResponse[] = [] const hidden = { count: 0 } const temp = new Map() const written = new Map() const clipboard: Harness['clipboard'] = { text: null, image: null } const crops: OcrRect[] = [] const opened: string[] = [] const config = { language: 'auto' } const recognizeCalls: Harness['recognizeCalls'] = [] let pendingRecognition: Array<(value: OcrResult | Error) => void> = [] const deps: ScreenOcrDeps = { capture: async () => { if (options.captureError) throw options.captureError return { png: Buffer.from('full-png'), width: 1920, height: 1080, scaleFactor: 1, displayBounds: { x: 0, y: 0, width: 1920, height: 1080 }, crop: (rect) => { crops.push(rect) return Buffer.from(`crop-${rect.x}-${rect.y}-${rect.width}-${rect.height}`) } } }, engine: { maxDimension: 10000, isSupported: () => options.supported ?? true, listLanguages: async () => ['ko', 'en-US'], recognize: (request) => { recognizeCalls.push({ imagePath: request.imagePath, language: request.language, scale: request.scale }) if (options.autoRecognize ?? true) return Promise.resolve(ocrResult()) return new Promise((resolve, reject) => { pendingRecognition.push((value) => (value instanceof Error ? reject(value) : resolve(value))) }) } }, overlay: { show: (bounds, payload) => shown.push({ bounds, payload }), sendRecognized: (payload) => recognized.push(payload), respond: (payload) => responses.push(payload), hide: () => { hidden.count += 1 } }, files: { writeTemp: async (name, data) => { const path = `/tmp/${name}` temp.set(path, data) return path }, remove: async (path) => { temp.delete(path) }, askSavePath: async () => (options.savePath === undefined ? '/out/file' : options.savePath), write: async (path, data) => { written.set(path, data) } }, renderPdf: options.renderPdf ?? (async () => Buffer.from('%PDF')), clipboard: { writeText: async (text) => { if (options.clipboardError) throw options.clipboardError clipboard.text = text }, writeImage: async (png) => { clipboard.image = png } }, llm: { translate: options.translate ?? (async (text, target) => `[${target}] ${text}`), summarize: async (text) => `요약: ${text}` }, tts: { speak: vi.fn(async () => undefined), stop: vi.fn() }, openExternal: options.openExternal ?? (async (url) => { opened.push(url) return true }), config: { getOcrLanguage: () => config.language, setOcrLanguage: (language) => { config.language = language }, getAppLocale: () => 'ko', getTranslationTarget: () => 'English' }, strings: () => ({ copy: '복사' }), now: () => new Date(2026, 10, 20, 14, 5, 9).getTime() } return { service: new ScreenOcrService(deps), shown, recognized, responses, hidden, temp, written, clipboard, crops, opened, config, recognizeCalls, finishRecognition: (value) => { const waiting = pendingRecognition pendingRecognition = [] for (const finish of waiting) finish(value) }, sessionId: () => shown[shown.length - 1].payload.sessionId, deps } } /** 시작하고 인식이 끝나 오버레이에 결과가 갈 때까지 기다린다. */ async function open(h: Harness): Promise { await h.service.start() await vi.waitFor(() => expect(h.recognized).toHaveLength(1)) } describe('ScreenOcrService 캡처 수명주기', () => { it('화면OCR시작_캡처성공_오버레이를화면크기로띄우고인식결과를보낸다', async () => { const h = createHarness() await open(h) expect(h.shown).toHaveLength(1) expect(h.shown[0].bounds).toEqual({ x: 0, y: 0, width: 1920, height: 1080 }) expect(h.shown[0].payload).toMatchObject({ width: 1920, height: 1080, language: 'auto', _i18n: { copy: '복사' } }) expect(h.recognized[0]).toMatchObject({ ok: true, languages: ['ko', 'en-US'] }) expect(h.recognizeCalls[0]).toMatchObject({ language: 'auto', scale: 2 }) expect(h.service.isOpen).toBe(true) }) it('화면OCR시작_인식이끝난뒤_캡처임시파일이디스크에남지않는다', async () => { const h = createHarness() await open(h) expect(h.recognizeCalls[0].imagePath).toMatch(/\.png$/u) expect(h.temp.size).toBe(0) }) it('화면OCR시작_인식이실패해도_캡처임시파일을지우고실패를오버레이에알린다', async () => { const h = createHarness({ autoRecognize: false }) await h.service.start() await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1)) h.finishRecognition(new Error('engine crashed')) await vi.waitFor(() => expect(h.recognized).toHaveLength(1)) expect(h.recognized[0]).toMatchObject({ ok: false, error: 'recognition' }) expect(h.temp.size).toBe(0) }) it('화면OCR시작_설치된인식언어가없음_쓸수없음으로알린다', async () => { const h = createHarness({ autoRecognize: false }) await h.service.start() await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1)) h.finishRecognition(new D3ROError(ErrorCode.ScreenOcrUnavailable, 'no language')) await vi.waitFor(() => expect(h.recognized).toHaveLength(1)) expect(h.recognized[0]).toMatchObject({ ok: false, error: 'unavailable' }) }) it('화면OCR시작_지원하지않는플랫폼_오버레이를띄우지않고ScreenOcrUnavailable을던진다', async () => { const h = createHarness({ supported: false }) await expect(h.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrUnavailable }) expect(h.shown).toHaveLength(0) expect(h.service.isOpen).toBe(false) }) it('화면OCR시작_캡처실패_ScreenOcrCaptureFailed를던지고열린상태로남지않는다', async () => { const failing = createHarness({ captureError: new Error('no source') }) await expect(failing.service.start()).rejects.toMatchObject({ code: ErrorCode.ScreenOcrCaptureFailed }) expect(failing.shown).toHaveLength(0) expect(failing.service.isOpen).toBe(false) }) it('화면OCR시작_이미열려있을때다시시작_새캡처를겹쳐만들지않는다', async () => { const h = createHarness() await open(h) await h.service.start() expect(h.shown).toHaveLength(1) }) it('화면OCR토글_열려있을때단축키_닫고오버레이를숨긴다', async () => { const h = createHarness() await open(h) await h.service.toggle() expect(h.service.isOpen).toBe(false) expect(h.hidden.count).toBe(1) }) it('화면OCR닫기_인식이끝나기전에닫음_늦게온결과를오버레이에보내지않는다', async () => { const h = createHarness({ autoRecognize: false }) await h.service.start() await vi.waitFor(() => expect(h.recognizeCalls).toHaveLength(1)) h.service.close() h.finishRecognition(ocrResult()) await vi.waitFor(() => expect(h.temp.size).toBe(0)) expect(h.recognized).toHaveLength(0) }) it('화면OCR닫기_읽어주기를쓰지않은캡처_다른기능의음성재생을멈추지않는다', async () => { const h = createHarness() await open(h) h.service.close() expect(h.deps.tts.stop).not.toHaveBeenCalled() }) it('화면OCR닫기_읽어주는중에닫음_읽기를멈춘다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'speak', text: '읽어 줘' }) expect(h.deps.tts.speak).toHaveBeenCalledWith('읽어 줘') h.service.close() expect(h.deps.tts.stop).toHaveBeenCalledTimes(1) }) }) describe('ScreenOcrService 요청 처리', () => { it('글자복사_선택한글_클립보드에쓰고성공으로응답한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 7, { type: 'copy-text', text: '화면 OCR\n둘째' }) expect(h.clipboard.text).toBe('화면 OCR\n둘째') expect(h.responses).toEqual([{ requestId: 7, ok: true }]) }) it('글자복사_클립보드쓰기실패_성공으로꾸미지않고실패로응답한다', async () => { const h = createHarness({ clipboardError: new Error('clipboard locked') }) await open(h) await h.service.handleRequest(h.sessionId(), 2, { type: 'copy-text', text: '화면' }) expect(h.responses).toEqual([{ requestId: 2, ok: false, error: 'export' }]) }) it('글자복사_빈글_클립보드를건드리지않고선택없음으로거절한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-text', text: ' ' }) expect(h.clipboard.text).toBeNull() expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'empty' }]) }) it('요청처리_끝난캡처의요청_아무것도하지않고거절한다', async () => { const h = createHarness() await open(h) const stale = h.sessionId() h.service.close() await h.service.handleRequest(stale, 3, { type: 'copy-text', text: '늦은 요청' }) expect(h.clipboard.text).toBeNull() expect(h.responses).toEqual([{ requestId: 3, ok: false, error: 'invalid' }]) }) it('이미지복사_영역지정_그림밖부분을잘라낸영역만복사한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-image', rect: { x: 1900, y: -20, width: 100, height: 120 } }) expect(h.crops).toEqual([{ x: 1900, y: 0, width: 20, height: 100 }]) expect(h.clipboard.image?.toString()).toBe('crop-1900-0-20-100') }) it('이미지복사_영역없음_화면전체를복사한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-image', rect: null }) expect(h.clipboard.image?.toString()).toBe('full-png') expect(h.crops).toEqual([]) }) it('이미지복사_그림밖영역_복사하지않고거절한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'copy-image', rect: { x: 5000, y: 5000, width: 10, height: 10 } }) expect(h.clipboard.image).toBeNull() expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }]) }) it('이미지저장_저장위치선택_그림을파일로쓴다', async () => { const h = createHarness({ savePath: '/out/shot.png' }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null }) expect(h.written.get('/out/shot.png')?.toString()).toBe('full-png') expect(h.responses).toEqual([{ requestId: 1, ok: true }]) }) it('이미지저장_대화상자취소_파일을쓰지않고취소로응답한다', async () => { const h = createHarness({ savePath: null }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'save-image', rect: null }) expect(h.written.size).toBe(0) expect(h.responses).toEqual([{ requestId: 1, ok: true, cancelled: true }]) }) it('PDF저장_영역지정_그영역의글자만영역기준좌표로넣은문서를PDF로쓴다', async () => { let html = '' const h = createHarness({ savePath: '/out/doc.pdf', renderPdf: async (htmlPath) => { html = String(h.temp.get(htmlPath)) return Buffer.from('%PDF-1.7') } }) await open(h) // 첫 줄(y 10~30)만 덮는 영역 await h.service.handleRequest(h.sessionId(), 1, { type: 'save-pdf', rect: { x: 0, y: 0, width: 200, height: 40 } }) expect(h.written.get('/out/doc.pdf')?.toString()).toBe('%PDF-1.7') expect(html).toContain('@page{size:200px 40px;margin:0}') expect(html).toContain('>화면') expect(html).toContain('>OCR') expect(html).not.toContain('둘째') expect(h.responses).toEqual([{ requestId: 1, ok: true }]) }) it('PDF저장_PDF생성실패_임시HTML을지우고내보내기실패로응답한다', async () => { const h = createHarness({ renderPdf: async () => { throw new Error('print failed') } }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'save-pdf', rect: null }) expect(h.written.size).toBe(0) expect(h.temp.size).toBe(0) expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'export' }]) }) it('번역_외국어글_앱언어를대상으로번역한결과를돌려준다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'The quick brown fox' }) expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '[Korean] The quick brown fox' }]) }) it('번역_LLM이빈응답_성공으로꾸미지않고실패로응답한다', async () => { const h = createHarness({ translate: async () => ' ' }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' }) expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }]) }) it('번역_LLM호출실패_실패종류를llm으로응답한다', async () => { const h = createHarness({ translate: async () => { throw new Error('server unreachable') } }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' }) expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'llm' }]) }) it('번역_처리중에닫음_닫힌오버레이로응답을보내지않는다', async () => { let finish: (text: string) => void = () => undefined const h = createHarness({ translate: () => new Promise((resolve) => { finish = resolve }) }) await open(h) const pending = h.service.handleRequest(h.sessionId(), 1, { type: 'translate', text: 'hello' }) h.service.close() finish('안녕') await pending expect(h.responses).toEqual([]) }) it('요약_선택한글_요약결과를돌려준다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'summarize', text: '긴 글' }) expect(h.responses).toEqual([{ requestId: 1, ok: true, text: '요약: 긴 글' }]) }) it('웹검색_선택한글_줄바꿈을정리한검색주소를연다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'search-web', text: '화면 OCR\n둘째' }) expect(h.opened).toEqual([`https://www.google.com/search?q=${encodeURIComponent('화면 OCR 둘째')}`]) }) it('링크열기_정책이막은주소_링크실패로응답한다', async () => { const h = createHarness({ openExternal: async () => false }) await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'open-link', url: 'file:///C:/secret' }) expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'link' }]) }) it('인식언어변경_설치된언어_설정을바꾸고그언어로다시인식한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'en-US' }) await vi.waitFor(() => expect(h.recognized).toHaveLength(2)) expect(h.config.language).toBe('en-US') expect(h.recognizeCalls[1].language).toBe('en-US') expect(h.responses).toEqual([{ requestId: 1, ok: true }]) }) it('인식언어변경_설치되지않은언어_설정을바꾸지않고거절한다', async () => { const h = createHarness() await open(h) await h.service.handleRequest(h.sessionId(), 1, { type: 'set-language', language: 'xx-YY' }) expect(h.config.language).toBe('auto') expect(h.recognizeCalls).toHaveLength(1) expect(h.responses).toEqual([{ requestId: 1, ok: false, error: 'invalid' }]) }) }) describe('Windows OCR 엔진 어댑터', () => { it('엔진인식_정상출력_정본결과로옮기고입력은환경변수로만넘긴다', async () => { const calls: Array> = [] const engine = createWindowsOcrEngine({ platform: 'win32', run: async (env) => { calls.push(env) return 'WARNING: noise\n{"lang":"ko","lines":[{"words":[{"t":"화면","x":1,"y":2,"w":30,"h":12}]}]}\n' } }) const result = await engine.recognize({ imagePath: 'C:\\tmp\\a b.png', language: 'ko', scale: 1.5 }) expect(result.words.map((w) => w.text)).toEqual(['화면']) expect(calls[0]).toEqual({ D3RO_OCR_MODE: 'recognize', D3RO_OCR_IMAGE: 'C:\\tmp\\a b.png', D3RO_OCR_LANG: 'ko', D3RO_OCR_SCALE: '1.5' }) }) it('엔진인식_줄과단어가하나뿐이라배열이풀린출력_그대로읽는다', async () => { const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"lang":"en-US","lines":{"words":{"t":"Hello","x":1,"y":2,"w":30,"h":12}}}' }) const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 }) expect(result.words.map((w) => w.text)).toEqual(['Hello']) expect(result.lines).toHaveLength(1) }) it('엔진인식_글자가없는화면_빈결과를돌려준다', async () => { const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"lang":"ko","lines":[]}' }) const result = await engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 }) expect(result.words).toEqual([]) }) it('엔진인식_설치된언어없음_ScreenOcrUnavailable', async () => { const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"error":"no-language"}' }) await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({ code: ErrorCode.ScreenOcrUnavailable }) }) it('엔진인식_엔진예외와해석불가출력_ScreenOcrRecognitionFailed', async () => { const failing = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"error":"exception","message":"bad image"}' }) await expect(failing.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({ code: ErrorCode.ScreenOcrRecognitionFailed }) const garbage = createWindowsOcrEngine({ platform: 'win32', run: async () => 'not json at all' }) await expect(garbage.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({ code: ErrorCode.ScreenOcrRecognitionFailed }) }) it('엔진인식_프로세스실행실패_ScreenOcrRecognitionFailed', async () => { const engine = createWindowsOcrEngine({ platform: 'win32', run: async () => { throw new Error('spawn ENOENT') } }) await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({ code: ErrorCode.ScreenOcrRecognitionFailed }) }) it('엔진지원여부_Windows가아님_지원하지않고프로세스를띄우지않는다', async () => { const run = vi.fn(async () => '{}') const engine = createWindowsOcrEngine({ platform: 'darwin', run }) expect(engine.isSupported()).toBe(false) expect(await engine.listLanguages()).toEqual([]) await expect(engine.recognize({ imagePath: 'x.png', language: 'auto', scale: 2 })).rejects.toMatchObject({ code: ErrorCode.ScreenOcrUnavailable }) expect(run).not.toHaveBeenCalled() }) it('언어목록_하나뿐이라문자열로온출력_목록으로돌려준다', async () => { const single = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":"ko"}' }) expect(await single.listLanguages()).toEqual(['ko']) const many = createWindowsOcrEngine({ platform: 'win32', run: async () => '{"languages":["ko","en-US"]}' }) expect(await many.listLanguages()).toEqual(['ko', 'en-US']) }) })