From 5c11ee2fde767153cf1cd6e591660f85924c6a09 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Wed, 23 Sep 2026 16:04:27 +0900 Subject: [PATCH] release: ship v1.5.0 with on-device writing suggestions Adds next-sentence suggestions while typing, weekly input insights and a personal phrase memory to the desktop app, and fixes custom instructions so they process the text instead of inserting the instruction's own wording. Local model requests are now bounded and individually cancellable. Bumps the product version to 1.5.0 (Android/iOS build 1050000), refreshes the landing and web download links, and records the new INPUT feature rows and the open verification gaps in the infrastructure map. --- CHANGELOG.md | 63 + apps/admin-swagger/openapi.json | 2 +- apps/admin/package.json | 2 +- apps/api-server/D3ROVoice.Api.csproj | 2 +- apps/desktop/electron-builder.yml | 3 + apps/desktop/electron.vite.config.ts | 4 + apps/desktop/package.json | 3 +- apps/desktop/scripts/build-sidecar.mjs | 5 + apps/desktop/sidecar/main.py | 25 + apps/desktop/sidecar/requirements.txt | 3 + apps/desktop/sidecar/uia_bridge.py | 428 ++++++ apps/desktop/src/main/bootstrap.ts | 91 ++ apps/desktop/src/main/db/index.ts | 93 ++ apps/desktop/src/main/db/schema.ts | 131 ++ apps/desktop/src/main/ipc/index.ts | 4 + .../src/main/ipc/input-telemetry-handlers.ts | 164 +++ .../src/main/ipc/suggestion-handlers.ts | 190 +++ apps/desktop/src/main/lifecycle.ts | 2 + .../src/main/services/AudioCaptureService.ts | 2 +- .../src/main/services/ConfigService.ts | 70 +- .../main/services/FileTranscriptionService.ts | 6 +- .../src/main/services/HistoryService.ts | 13 + .../main/services/InputTelemetryService.ts | 1296 +++++++++++++++++ .../src/main/services/KeyBindingService.ts | 165 ++- .../src/main/services/LocalLLMService.ts | 319 +++- .../src/main/services/LocalSTTService.ts | 11 + .../src/main/services/PersonalGraphService.ts | 392 +++++ .../src/main/services/ScreenContextService.ts | 2 +- .../src/main/services/SuggestionService.ts | 1199 +++++++++++++++ .../src/main/services/TTSPlaybackService.ts | 2 +- .../src/main/services/UiaContextService.ts | 214 +++ .../src/main/services/VoiceActionService.ts | 12 +- .../main/services/VoiceConversationService.ts | 49 +- .../src/main/services/global-input-hook.ts | 69 + apps/desktop/src/main/services/llm-prompts.ts | 95 ++ .../src/main/utils/win32-foreground.ts | 180 +++ .../desktop/src/main/windows/WindowManager.ts | 190 ++- apps/desktop/src/preload/index.ts | 108 ++ .../src/renderer/components/SettingsModal.tsx | 64 +- .../components/input-insights/BarChart.tsx | 163 +++ .../input-insights/InputConsentPanel.tsx | 555 +++++++ .../input-insights/InputInsightsView.tsx | 701 +++++++++ .../src/renderer/hooks/useInputInsights.ts | 83 ++ .../src/renderer/pages/DashboardPage.tsx | 53 + .../src/renderer/pages/KnowledgeBasePage.tsx | 35 +- .../popups/suggestion-overlay/index.html | 31 + .../popups/suggestion-overlay/script.js | 200 +++ .../popups/suggestion-overlay/style.css | 208 +++ .../main/ipc/suggestion-handlers.test.ts | 80 + .../tests/main/services/ConfigService.test.ts | 61 + .../main/services/SuggestionService.test.ts | 274 ++++ .../services/VoiceConversationService.test.ts | 194 +++ .../main/services/input-flow-domain.test.ts | 197 +++ .../main/services/input-flow-services.test.ts | 581 ++++++++ .../main/services/input-intelligence.test.ts | 470 ++++++ .../tests/main/services/llm-prompts.test.ts | 46 + .../main/services/local-llm-service.test.ts | 181 +++ .../windows-child-process-hide.test.ts | 164 +++ apps/mobile-rn/android/app/build.gradle | 4 +- .../ios/D3ROVoice.xcodeproj/project.pbxproj | 8 +- apps/mobile-rn/package-lock.json | 14 +- apps/mobile-rn/package.json | 2 +- apps/web/package.json | 2 +- apps/web/src/components/layout/sidebar.tsx | 2 +- apps/web/src/lib/desktop-release.ts | 4 +- design.md | 6 + docs/map/00-index.md | 8 +- docs/map/02-infrastructure.md | 4 +- docs/map/03-shared-packages.md | 5 +- docs/map/04-desktop-app.md | 45 +- docs/map/10-feature-catalog.md | 32 +- docs/map/11-gap-backlog.md | 11 + memory/project_status.md | 140 ++ package-lock.json | 373 ++++- package.json | 2 +- packages/api-client/package.json | 2 +- .../core/__tests__/personal-graph.test.ts | 123 ++ packages/core/package.json | 6 +- packages/core/src/errors.ts | 14 + packages/core/src/index.ts | 2 + packages/core/src/input-intelligence.ts | 1113 ++++++++++++++ packages/core/src/ipc-channels.ts | 44 + packages/core/src/keybinding.ts | 105 +- packages/core/src/personal-graph.ts | 222 +++ packages/core/src/types.ts | 49 + packages/i18n/package.json | 2 +- packages/i18n/src/locales/de.json | 189 ++- packages/i18n/src/locales/en.json | 189 ++- packages/i18n/src/locales/es.json | 189 ++- packages/i18n/src/locales/fr.json | 189 ++- packages/i18n/src/locales/ja.json | 189 ++- packages/i18n/src/locales/ko.json | 189 ++- packages/i18n/src/locales/pt.json | 189 ++- packages/i18n/src/locales/ru.json | 189 ++- packages/i18n/src/locales/th.json | 189 ++- packages/i18n/src/locales/vi.json | 189 ++- packages/i18n/src/locales/zh-TW.json | 189 ++- packages/i18n/src/locales/zh.json | 189 ++- packages/ui-native/package.json | 2 +- packages/ui/package.json | 2 +- release/product-version.json | 8 +- site/package-lock.json | 4 +- site/package.json | 2 +- site/src/release.ts | 4 +- 104 files changed, 14410 insertions(+), 174 deletions(-) create mode 100644 apps/desktop/sidecar/uia_bridge.py create mode 100644 apps/desktop/src/main/ipc/input-telemetry-handlers.ts create mode 100644 apps/desktop/src/main/ipc/suggestion-handlers.ts create mode 100644 apps/desktop/src/main/services/InputTelemetryService.ts create mode 100644 apps/desktop/src/main/services/PersonalGraphService.ts create mode 100644 apps/desktop/src/main/services/SuggestionService.ts create mode 100644 apps/desktop/src/main/services/UiaContextService.ts create mode 100644 apps/desktop/src/main/services/global-input-hook.ts create mode 100644 apps/desktop/src/main/utils/win32-foreground.ts create mode 100644 apps/desktop/src/renderer/components/input-insights/BarChart.tsx create mode 100644 apps/desktop/src/renderer/components/input-insights/InputConsentPanel.tsx create mode 100644 apps/desktop/src/renderer/components/input-insights/InputInsightsView.tsx create mode 100644 apps/desktop/src/renderer/hooks/useInputInsights.ts create mode 100644 apps/desktop/src/renderer/popups/suggestion-overlay/index.html create mode 100644 apps/desktop/src/renderer/popups/suggestion-overlay/script.js create mode 100644 apps/desktop/src/renderer/popups/suggestion-overlay/style.css create mode 100644 apps/desktop/tests/main/ipc/suggestion-handlers.test.ts create mode 100644 apps/desktop/tests/main/services/ConfigService.test.ts create mode 100644 apps/desktop/tests/main/services/SuggestionService.test.ts create mode 100644 apps/desktop/tests/main/services/VoiceConversationService.test.ts create mode 100644 apps/desktop/tests/main/services/input-flow-domain.test.ts create mode 100644 apps/desktop/tests/main/services/input-flow-services.test.ts create mode 100644 apps/desktop/tests/main/services/input-intelligence.test.ts create mode 100644 apps/desktop/tests/main/services/local-llm-service.test.ts create mode 100644 apps/desktop/tests/main/services/windows-child-process-hide.test.ts create mode 100644 packages/core/__tests__/personal-graph.test.ts create mode 100644 packages/core/src/input-intelligence.ts create mode 100644 packages/core/src/personal-graph.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f5eac..a5d73da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,69 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Cloud-optional backup (encrypted, opt-in) - Plugin system for custom pipelines +## [1.5.0] - 2026-09-23 + +> Published from an annotated tag. Installer and update metadata are served by the +> canonical Forgejo feed; no binaries are committed to this repository. + +### Added +- **Next-sentence suggestions while you type.** With the feature turned on, a small + ghost-text panel sits next to the caret in whatever app has focus and offers up to + three ways to continue the sentence. `Ctrl+Alt+Right` accepts, `Down`/`Up` move + between candidates, `Left` or the panel's X dismisses, and all four are ordinary + rebindable shortcuts. The panel never takes focus and is click-through until you + turn interaction on. +- **Typed-text learning from the field you are in,** read through UI Automation and + diffed as a snapshot, so committed Korean and Japanese IME text is counted + correctly (key codes cannot reconstruct it). Password fields are refused before any + read, and IME composition suppresses learning and suggestions until the character + is committed. +- **Weekly input insights** (`Settings > Input` and a dashboard card): keystroke, + click and scroll totals, mouse travel converted to meters, top hours and apps, flow + windows, edit friction (edits per 100 characters) and per-app suggestion quality. +- **A personal phrase memory** built from your own typed sentences and voice history, + ranked by frequency, recency (30-day half-life) and same-app usage, and offered as + prompt hints. Phrases can be deleted individually or all at once. +- **A privacy receipt** that reports the real number of retained rows and the 30-day + retention rule, plus **smart exclusion**: only the app you are in can be suggested + for the exclusion list, and only after repeated unreadable fields, never + automatically. +- **A shortcut safety audit** in Settings that flags invalid or conflicting bindings + across every action. +- **A local personal graph** that links your sentences by "follows" and shared-terms + relationships, so related context can surface even when the current prefix differs. + It is local SQLite only, with no embeddings or network. + +### Changed +- **Local model requests are now bounded and individually cancellable.** + `LocalLLMService` gives every request its own abort controller, relays external + cancellation, requires a `done` frame before a stream counts as successful, and + clears incomplete streams. Generation and streaming are capped at 2048 tokens / + 120 seconds and chat at 512 / 60 seconds. Text suggestions use `keep_alive: 2m` and + no longer warm the model at startup. +- **Voice conversations are single-flight** and pass their cancellation through to + the active local chat request, so ending a conversation aborts its own inference + instead of only changing what the UI shows. +- Console windows are suppressed for the app's Windows helper processes (text to + speech, voice actions, audio-device and active-window enumeration, and every ffmpeg + path). + +### Fixed +- **Custom instructions ran the instruction instead of your text.** Every built-in + instruction (translate, summarize, rephrase, explain code, free prompt) inserted the + instruction's own wording rather than the processed result, and only `{{text}}` was + ever substituted, which none of the presets use. The instruction is now the system + prompt, the transcript is the text, and `{{text}}`, `{{userPrompt}}` and + `{{targetLanguage}}` are resolved in one shared place. This path had never worked in + any shipped release. +- **The suggestion panel could refuse to close, revive itself, or appear on a bare + click.** Holding the modifier that Windows turns into `Ctrl+Alt` no longer fires the + voice shortcut underneath, closing the panel during generation discards that request + instead of re-showing it, a click without typing no longer triggers a suggestion, + and pressing Enter stops the same text from being suggested again. +- **The global input hook is reference-counted**, so the shortcut service and input + telemetry can attach at the same time without one disabling the other. + ## [1.4.0] - 2026-09-21 ### Added diff --git a/apps/admin-swagger/openapi.json b/apps/admin-swagger/openapi.json index 48e374c..d3ad687 100644 --- a/apps/admin-swagger/openapi.json +++ b/apps/admin-swagger/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "D3RO-VOICE Admin API", "description": "Admin CRM Edge Functions for user management, subscription CRUD, payment history, and audit logs.", - "version": "1.4.0" + "version": "1.5.0" }, "servers": [ { diff --git a/apps/admin/package.json b/apps/admin/package.json index 2f364de..4c915af 100644 --- a/apps/admin/package.json +++ b/apps/admin/package.json @@ -1,6 +1,6 @@ { "name": "@d3ro/admin", - "version": "1.4.0", + "version": "1.5.0", "private": true, "description": "D3RO Voice Admin CRM — SaaS 관리 도구", "scripts": { diff --git a/apps/api-server/D3ROVoice.Api.csproj b/apps/api-server/D3ROVoice.Api.csproj index 7d8757f..92e9fea 100644 --- a/apps/api-server/D3ROVoice.Api.csproj +++ b/apps/api-server/D3ROVoice.Api.csproj @@ -2,7 +2,7 @@ net10.0 - 1.4.0 + 1.5.0 enable enable diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index d762e2f..1dc6217 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -44,6 +44,9 @@ asarUnpack: - "node_modules/better-sqlite3/**" - "node_modules/uiohook-napi/**" - "node_modules/@nut-tree-fork/**" + # koffi — 포그라운드 창 제목/프로세스 조회(User32 FFI). prebuilt 바이너리 포함. + - "node_modules/koffi/**" + - "node_modules/@koromix/**" # ──────────────────────────────────────────────────────────────────── # Windows diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts index 4a3975e..6b8b6b1 100644 --- a/apps/desktop/electron.vite.config.ts +++ b/apps/desktop/electron.vite.config.ts @@ -67,6 +67,10 @@ export default defineConfig({ 'popups/caption-overlay': resolve( __dirname, 'src/renderer/popups/caption-overlay/index.html' + ), + 'popups/suggestion-overlay': resolve( + __dirname, + 'src/renderer/popups/suggestion-overlay/index.html' ) } } diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 04ccbec..da1617d 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@d3ro/desktop", - "version": "1.4.0", + "version": "1.5.0", "productName": "d3ro-voice", "description": "로컬 AI 음성 어시스턴트 (Electron)", "main": "./out/main/index.js", @@ -58,6 +58,7 @@ "electron-log": "^5.2.0", "electron-store": "^10.0.0", "electron-updater": "^6.3.9", + "koffi": "^3.3.1", "lucide-react": "^1.25.0", "overlayscrollbars": "^2.16.0", "overlayscrollbars-react": "^0.5.6", diff --git a/apps/desktop/scripts/build-sidecar.mjs b/apps/desktop/scripts/build-sidecar.mjs index bec2783..8801094 100644 --- a/apps/desktop/scripts/build-sidecar.mjs +++ b/apps/desktop/scripts/build-sidecar.mjs @@ -83,6 +83,11 @@ const args = [ 'tokenizers', '--collect-all', 'huggingface_hub', + // UIA 브리지(입력 인텔리전스). comtypes 는 실행 시점에 타입 라이브러리를 + // 동적으로 생성하므로 데이터 파일까지 통째로 모아야 한다 (Windows 전용). + ...(process.platform === 'win32' + ? ['--collect-all', 'uiautomation', '--collect-all', 'comtypes', '--hidden-import', 'comtypes.client'] + : []), // uvicorn은 동적 임포트를 사용하므로 명시 '--hidden-import', 'uvicorn.logging', diff --git a/apps/desktop/sidecar/main.py b/apps/desktop/sidecar/main.py index 5504939..9bfe944 100644 --- a/apps/desktop/sidecar/main.py +++ b/apps/desktop/sidecar/main.py @@ -12,6 +12,7 @@ faster-whisper + CTranslate2만 사용. torch/pyannote 비의존으로 슬림 POST /download - 모델 다운로드 시작 (백그라운드) GET /download/status - 다운로드 진행률 조회 POST /download/cancel - 다운로드 취소 + GET /uia/focus - 포커스 입력 요소 UIA 스냅샷 (제안/학습용) POST /shutdown - 서버 종료 주: 화자 구분(diarization)은 Phase 15.5에서 LLM 추정 경로가 primary이며, @@ -21,6 +22,7 @@ pyannote 기반 고정밀 화자 구분은 추후 서버 사이드 API로 제공 from __future__ import annotations import argparse +import asyncio import fnmatch import logging import os @@ -582,11 +584,34 @@ async def download_cancel() -> JSONResponse: return JSONResponse(content={"status": "idle"}) +@app.get("/uia/focus") +async def uia_focus(timeoutMs: int = 1500) -> JSONResponse: + """포커스된 입력 요소의 UIA 스냅샷 (텍스트/케어렛/비밀번호 여부). + + 제안(ghost text)과 이핑 학습이 쓰는 유일한 입력창 읽기 경로다. + 비밀번호 필드는 브리지 안에서 fail-closed 로 차단한다. + """ + try: + import uia_bridge + except Exception as exc: # pragma: no cover - 파일 누락 등 + return JSONResponse(content={"available": False, "reason": f"bridge-import:{exc}"}) + + snapshot = await asyncio.to_thread(uia_bridge.snapshot_focus, timeoutMs) + return JSONResponse(content=snapshot) + + @app.post("/shutdown") async def shutdown() -> JSONResponse: """서버를 graceful하게 종료한다.""" logger.info("종료 요청 수신") + try: + import uia_bridge + + await asyncio.to_thread(uia_bridge.shutdown) + except Exception: + pass + if _server is not None: _server.should_exit = True diff --git a/apps/desktop/sidecar/requirements.txt b/apps/desktop/sidecar/requirements.txt index 2d52c81..ed59de7 100644 --- a/apps/desktop/sidecar/requirements.txt +++ b/apps/desktop/sidecar/requirements.txt @@ -5,3 +5,6 @@ python-multipart>=0.0.6 numpy>=1.24.0 huggingface_hub>=0.23.0 requests>=2.31.0 +# UIA 브리지(/uia/focus) — 포커스 입력창 텍스트·케어렛 읽기. Windows 전용. +uiautomation>=2.0.29; sys_platform == "win32" +comtypes>=1.4.0; sys_platform == "win32" diff --git a/apps/desktop/sidecar/uia_bridge.py b/apps/desktop/sidecar/uia_bridge.py new file mode 100644 index 0000000..53605fa --- /dev/null +++ b/apps/desktop/sidecar/uia_bridge.py @@ -0,0 +1,428 @@ +"""UIA(UI Automation) 포커스 스냅샷 브리지. + +목적: 사용자가 지금 어느 입력창에 무엇을 치고 있는지, 어렛이 어디 있는지를 +읽어 "다음 문장 제안" 과 "타이핑 학습" 에 넘긴다. + +설계 근거 (조사 기반, 뇌피셜 아님): + - Win32 `GetGUIThreadInfo` 의 `rcCaret` 은 Chromium/Electron/VS Code 처럼 + 커서를 직접 그리는 앱에서 아무것도 돌려주지 않는다 (AutoHotkey CaretGetPos 문서, + MS Learn "GetGUIThreadInfo" 의 rcCaret 주의 문구). 그래서 UIA 를 정본으로 쓴다. + - 케어렛 rect 는 `TextPattern.GetSelection()` (축소된 caret range) → + `GetBoundingRectangles()` 가 이식성 있는 경로다. Chromium 은 `ITextProvider` 는 + 구현하지만 `ITextPattern2::GetCaretRange` 는 구현하지 않는다 (Chromium 소스). + - 비밀번호 필드는 `UIA_IsPasswordPropertyId(30019)` 로 반드시 먼저 검사한다. + (MS Learn "IUIAutomationElement::get_CurrentIsPassword") + - 트리 전체 순회는 금지한다. Chrome/VS Code 에서 UIA 트리 크가 10~30초 걸린다는 + 실측 보고가 있다 (PowerToys #46385). 포커스 요소 + 패턴 2~3회 호출만 한다. + - IME(한/일) 조합 중에는 스트가 커밋되지 않고 케어렛도 부정확하다. 조합 중이면 + 호출자는 제안을 억제해야 하므로 `is_composing` 을 함께 돌려준다. + +모든 호출은 전용 단일 스레드에서 실행한다 (comtypes COM 아파트먼트 친화성). +""" + +from __future__ import annotations + +import logging +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor, TimeoutError as FutureTimeout +from typing import Any, Optional + +logger = logging.getLogger("sidecar.uia") + +# 텍스트를 그대로 들고 오는 상한. 입력창 전체 복사는 위험하므로 잘라 쓴다. +MAX_TEXT_CHARS = 4000 + +# UIA 호출 예산. 넘기면 이번 주기는 실패로 처리하고 다음 기회를 기다린다. +DEFAULT_TIMEOUT_MS = 1500 + +_uia_available: Optional[bool] = None +_executor: Optional[ThreadPoolExecutor] = None +_executor_lock = threading.Lock() +_inflight = threading.Lock() + + +def _uia_module() -> Any: + """uiautomation 모듈을 지연 임포트한다 (미설치 환경에서 임포트 자체를 막지 않기 위해).""" + global _uia_available + try: + import uiautomation # noqa: PLC0415 + + _uia_available = True + return uiautomation + except Exception as exc: # pragma: no cover - 설치 환경 의존 + _uia_available = False + logger.warning("uiautomation 미설치 — UIA 컨텍스트 비활성 (%s)", exc) + raise + + +def is_available() -> bool: + """uiautomation 사용 가능 여부 (실제 임포트 시도 결과).""" + global _uia_available + if _uia_available is not None: + return _uia_available + try: + _uia_module() + except Exception: + return False + return bool(_uia_available) + + +def _worker_init() -> None: + """UIA 전용 스레드의 COM 초기화.""" + try: + import comtypes # noqa: PLC0415 + + comtypes.CoInitializeEx(comtypes.COINIT_MULTITHREADED) + except Exception: + # 이미 초기화됐거나 comtypes 가 없더라도 uiautomation 자체 초기화에 맡긴다. + try: + import comtypes # noqa: PLC0415 + + comtypes.CoInitialize() + except Exception: + pass + + +def _get_executor() -> ThreadPoolExecutor: + global _executor + with _executor_lock: + if _executor is None: + _executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="uia") + _executor.submit(_worker_init).result(timeout=5) + return _executor + + +def _rect_to_dict(rect: Any) -> Optional[dict]: + if rect is None: + return None + try: + left = int(rect.left) + top = int(rect.top) + right = int(rect.right) + bottom = int(rect.bottom) + except Exception: + return None + width = right - left + height = bottom - top + if width < 0 or height < 0: + return None + return {"x": left, "y": top, "width": width, "height": height} + + +def _first_rect(rects: Any) -> Optional[dict]: + """GetBoundingRectangles 결과에서 케어 rect 를 는다 (첫 사각형 우선).""" + try: + for rect in rects or []: + converted = _rect_to_dict(rect) + if converted is not None: + return converted + except Exception: + return None + return None + + +def _read_text(auto: Any, control: Any) -> tuple[str, str, Optional[Any]]: + """(텍스트, 출처, value_pattern) 을 돌려준다. 지원 패턴이 없으면 ('', 'none', None).""" + value_pattern = None + try: + value_pattern = control.GetValuePattern() + except Exception: + value_pattern = None + + if value_pattern is not None: + try: + text = value_pattern.Value or "" + return (str(text)[:MAX_TEXT_CHARS], "value", value_pattern) + except Exception: + pass + + try: + text_pattern = control.GetTextPattern() + except Exception: + text_pattern = None + if text_pattern is not None: + try: + text = text_pattern.DocumentRange.GetText(-1) or "" + return (str(text)[:MAX_TEXT_CHARS], "text", value_pattern) + except Exception: + # Chromium/CEF 입력창에서 COMError 가 나는 사례가 보고돼 있다 (issue #135/#138). + pass + + try: + legacy = control.GetLegacyIAccessiblePattern() + if legacy is not None: + text = legacy.Value or "" + return (str(text)[:MAX_TEXT_CHARS], "legacy", value_pattern) + except Exception: + pass + + return ("", "none", value_pattern) + + +def _caret_rect_and_offset(auto: Any, control: Any) -> tuple[Optional[dict], Optional[int], bool, bool]: + """케어렛 rect / 문서 시작 기준 오프셋 / 조합 진행 여부 / 비축소 선택 여부.""" + caret_rect: Optional[dict] = None + caret_offset: Optional[int] = None + composing = False + has_selection = False + + text_pattern = None + try: + text_pattern = control.GetTextPattern() + except Exception: + text_pattern = None + + if text_pattern is not None: + try: + selection = text_pattern.GetSelection() + except Exception: + selection = None + + if selection: + for selected_range in selection: + try: + has_selection = ( + selected_range.CompareEndpoints( + auto.TextPatternRangeEndpoint.Start, + selected_range, + auto.TextPatternRangeEndpoint.End, + ) + != 0 + ) + except Exception: + has_selection = False + if has_selection: + break + caret_range = selection[0] + try: + caret_rect = _first_rect(caret_range.GetBoundingRectangles()) + except Exception: + caret_rect = None + + if caret_rect is None: + # 축소된 caret range 는 빈 사각형 배열을 돌려주는 경우가 많다. + # 문자 단위로 확장한 뒤 다시 시도한다 (Windows Terminal 레시피). + try: + expanded = caret_range.Clone() + expanded.ExpandToEnclosingUnit(auto.TextUnit.Character) + caret_rect = _first_rect(expanded.GetBoundingRectangles()) + except Exception: + caret_rect = None + + try: + caret_offset = int( + caret_range.CompareEndpoints( + auto.TextPatternRangeEndpoint.Start, + text_pattern.DocumentRange, + auto.TextPatternRangeEndpoint.Start, + ) + ) + except Exception: + caret_offset = None + + # IME 조합 여부 — TextEditPattern 이 지원되는 제공자에서만 확인 가능하다. + try: + pattern_id = getattr(auto.PatternId, "TextEditPattern", None) + if pattern_id is not None: + text_edit = control.GetPattern(pattern_id) + if text_edit is not None: + composition = text_edit.GetActiveComposition() + if composition is not None: + composing_text = composition.GetText(-1) or "" + composing = len(str(composing_text)) > 0 + except Exception: + composing = False + + return (caret_rect, caret_offset, composing, has_selection) + + + +def _wake_chromium_accessibility() -> bool: + """Chromium 계열에 "보조기술이 있다" 고 알려 접근성 트리를 열게 한다. + + Chromium 문서: 브라우저가 EVENT_SYSTEM_ALERT + custom object id 1 을 알리고, + 그 id 에 대해 WM_GETOBJECT 를 받으면 보조기술이 실행 중이라고 판단해 AX 트리를 + 만든다. NVDA 등이 쓰는 방식이며, 이걸 하지 않으면 Chrome/Edge 의 웹 입력창은 + LegacyIAccessible 만 노출돼 텍스트를 읽을 수 없다(실측: edit=false src=legacy len=0). + + SendMessage 대신 SendMessageTimeout 을 쓰는 이유: 대상이 바쁘면 무한 대기한다. + """ + try: + import ctypes # noqa: PLC0415 + from ctypes import wintypes # noqa: PLC0415 + + user32 = ctypes.windll.user32 + WM_GETOBJECT = 0x003D + OBJID_CLIENT = 0xFFFFFFFC + + hwnd = user32.GetForegroundWindow() + if not hwnd: + return False + + result = ctypes.c_void_p() + # 1 = 보조기술 감지용 custom object id + user32.SendMessageTimeoutW( + wintypes.HWND(hwnd), + WM_GETOBJECT, + wintypes.WPARAM(0), + wintypes.LPARAM(1), + 0x0002, # SMTO_ABORTIFHUNG + 250, + ctypes.byref(result), + ) + # 클라이언트 영역 객체도 함께 요청해 AX 트리를 확실히 깨운다. + user32.SendMessageTimeoutW( + wintypes.HWND(hwnd), + WM_GETOBJECT, + wintypes.WPARAM(0), + wintypes.LPARAM(OBJID_CLIENT & 0xFFFFFFFF), + 0x0002, + 250, + ctypes.byref(result), + ) + return True + except Exception as exc: # pragma: no cover - 플랫폼 의존 + logger.debug("Chromium 접근성 깨우기 실패: %s", exc) + return False + + +def _snapshot() -> dict: + """전용 스레드에서 실행되는 실제 UIA 수집.""" + auto = _uia_module() + + # Chromium/Electron 계열은 접근성 트리를 요청받아야 연다 — 먼저 깨운다. + _wake_chromium_accessibility() + + started = time.monotonic() + control = auto.GetFocusedControl() + if control is None: + return {"available": False, "reason": "no-focused-control"} + + result: dict[str, Any] = { + "available": True, + "isPassword": False, + "isEditable": False, + "isComposing": False, + "hasSelection": False, + "controlType": None, + "controlName": None, + "className": None, + "textSource": "none", + "text": "", + "caretOffset": None, + "caretRect": None, + "elementRect": None, + "windowTitle": None, + "processId": None, + } + + try: + result["controlType"] = str(control.ControlTypeName or "") + result["controlName"] = str(control.Name or "")[:200] + result["className"] = str(control.ClassName or "")[:120] + except Exception: + pass + + # 1) 비밀번호 필드는 어떤 읽기도 하지 않는다 (fail-closed). + try: + is_password = bool(control.IsPassword) + except Exception: + # 판별 실패를 "안전" 으로 넘기면 안 된다 — 읽기 불가로 처리한다. + return {"available": False, "reason": "is-password-unavailable"} + result["isPassword"] = is_password + if is_password: + result["reason"] = "password-field" + return result + + try: + result["processId"] = int(control.ProcessId) + except Exception: + result["processId"] = None + + try: + top = control.GetTopLevelControl() + if top is not None: + result["windowTitle"] = str(top.Name or "")[:300] + result["elementRect"] = _rect_to_dict(top.BoundingRectangle) or result["elementRect"] + except Exception: + pass + + try: + result["elementRect"] = _rect_to_dict(control.BoundingRectangle) or result["elementRect"] + except Exception: + pass + + text, source, value_pattern = _read_text(auto, control) + result["textSource"] = source + result["text"] = text + + is_readonly = False + try: + if value_pattern is not None: + is_readonly = bool(value_pattern.IsReadOnly) + except Exception: + is_readonly = False + + control_type = result["controlType"] or "" + + # 터미널(Windows Terminal 등)은 ValuePattern 이 없고 TextPattern 만 있다. + # 값 패턴 부재만 보고 "편집 불가" 로 판정하면 파워셸/터미널처럼 실제로 가장 + # 많이 타이핑하는 표면에서 제안이 통째로 죽는다(실측: src=text, edit=false). + # 텍스트 패턴이 선택/삽입 지점을 제공하면 입력 가능으로 본다. + # + # TextPattern 이 있으면 입력 가능으로 본다. 처음에는 GetSelection() 이 비어 + # 있지 않은지까지 봤는데, Windows Terminal 은 선택이 없으면 0개를 돌려주는 + # 탓에 터미널이 계속 "편집 불가" 로 판정됐다(실측 프로브: TextPattern=yes, + # GetSelection=0 ranges). 값 패턴조차 없는 표면이라 다른 단서가 없다. + text_readable = False + try: + text_readable = control.GetTextPattern() is not None + except Exception: + text_readable = False + + result["isEditable"] = bool( + (value_pattern is not None and not is_readonly) + or control_type in ("EditControl", "DocumentControl") + or text_readable + ) + + caret_rect, caret_offset, composing, has_selection = _caret_rect_and_offset(auto, control) + result["caretRect"] = caret_rect + result["caretOffset"] = caret_offset + result["isComposing"] = composing + result["hasSelection"] = has_selection + result["capturedAt"] = int(time.time() * 1000) + result["elapsedMs"] = int((time.monotonic() - started) * 1000) + return result + + +def snapshot_focus(timeout_ms: int = DEFAULT_TIMEOUT_MS) -> dict: + """포커스 스냅샷. 실패/타임아웃은 예외 대신 available=False 로 돌려준다.""" + if not is_available(): + return {"available": False, "reason": "uiautomation-missing"} + + # 이전 호출이 아직 춰 있으면 새 작업을 쌓지 않고 즉시 포기한다. + if not _inflight.acquire(blocking=False): + return {"available": False, "reason": "busy"} + try: + executor = _get_executor() + future: Future = executor.submit(_snapshot) + try: + return future.result(timeout=max(0.2, timeout_ms / 1000)) + except FutureTimeout: + logger.warning("UIA 스냅샷 타임아웃 (%sms) — 이번 주기 건너뜀", timeout_ms) + return {"available": False, "reason": "timeout"} + except Exception as exc: # pragma: no cover - 환경 의존 + logger.warning("UIA 스냅샷 실패: %s", exc) + return {"available": False, "reason": f"error:{type(exc).__name__}"} + finally: + _inflight.release() + + +def shutdown() -> None: + global _executor + with _executor_lock: + if _executor is not None: + _executor.shutdown(wait=False, cancel_futures=True) + _executor = None diff --git a/apps/desktop/src/main/bootstrap.ts b/apps/desktop/src/main/bootstrap.ts index c406db6..b68a059 100644 --- a/apps/desktop/src/main/bootstrap.ts +++ b/apps/desktop/src/main/bootstrap.ts @@ -11,6 +11,8 @@ import { persistCompletedVoiceSessionSafe } from './voice-session-persist' import { getTextInsertService } from './services/TextInsertService' import { getCustomInstructionService } from './services/CustomInstructionService' import { getVoiceCommandService } from './services/VoiceCommandService' +import { getInputTelemetryService } from './services/InputTelemetryService' +import { getSuggestionService } from './services/SuggestionService' import { getSoundEffectService } from './services/SoundEffectService' import { getAutoLaunchService } from './services/AutoLaunchService' import { getAudioCaptureService } from './services/AudioCaptureService' @@ -30,6 +32,8 @@ import { isCommandPopupVisible, hideRecordingTip, updateRecordingTipState, + showSuggestionOverlay, + hideSuggestionOverlay, } from './windows/WindowManager' import { createTray } from './windows/TrayManager' import { registerAllIpcHandlers } from './ipc' @@ -62,6 +66,7 @@ export async function bootstrap(): Promise { { name: 'auto-launch', critical: false, fn: initAutoLaunch }, { name: 'popup-preload', critical: false, fn: initPopupWindows }, { name: 'key-bindings', critical: false, fn: initKeyBindings }, + { name: 'input-intelligence', critical: false, fn: initInputIntelligence }, { name: 'voice-mode', critical: false, fn: initVoiceMode }, { name: 'stt-warmup', critical: false, fn: initSTTWarmup }, { name: 'llm-polling', critical: false, fn: initLLMPolling }, @@ -125,6 +130,91 @@ async function initKeyBindings(): Promise { keyBindings.start() } +/** + * 입력 인텔리전스 배선 — 텔레메트리 → 제안 → 오버레이. + * + * 두 서비스를 직접 서로 import 하지 않고 여기서 이벤트로 는다 + * (서비스 간 결합을 늘리지 않으면서 교체 가능성을 남긴다). + */ +async function initInputIntelligence(): Promise { + const telemetry = getInputTelemetryService() + const suggestion = getSuggestionService() + + telemetry.on('typed-context', (context) => suggestion.handleTypingContext(context)) + telemetry.on('state-changed', (state) => + sendToMainWindow(IPC_CHANNELS.INPUT_TELEMETRY.STATE_CHANGED, state) + ) + telemetry.on('activity', (payload) => + sendToMainWindow(IPC_CHANNELS.INPUT_TELEMETRY.ACTIVITY, payload) + ) + + suggestion.on('updated', (state) => { + // 오버레이/윈도우 호출이 예외를 던지면 서비스의 진행 플래그가 굳을 수 있다. + // 이벤트 경로는 어떤 경우에도 조용히 삼킨다 (UI 문제로 기능 전체가 멈추면 안 된다). + try { + const hasSomethingToShow = + state.candidates.length > 0 || state.generating || state.warmingUp || !!state.partialText + const shouldPresent = hasSomethingToShow && !!state.anchor + telemetry.setSuggestionPresentationActive(shouldPresent) + if (!shouldPresent) { + hideSuggestionOverlay() + } else { + showSuggestionOverlay({ + candidates: state.candidates, + activeIndex: state.activeIndex, + generating: state.generating, + warmingUp: state.warmingUp, + partialText: state.partialText, + anchor: state.anchor, + appName: state.appName, + provenance: state.provenance + }) + } + } catch (error) { + logger.warn(`제안 오버레이 표시 실패: ${error instanceof Error ? error.message : String(error)}`) + } + sendToMainWindow(IPC_CHANNELS.SUGGESTION.UPDATED, state) + }) + suggestion.on('cleared', (payload) => { + telemetry.setSuggestionPresentationActive(false) + try { + hideSuggestionOverlay() + } catch (error) { + logger.warn(`제안 오버레이 숨기기 실패: ${error instanceof Error ? error.message : String(error)}`) + } + sendToMainWindow(IPC_CHANNELS.SUGGESTION.CLEARED, payload) + }) + suggestion.on('state-changed', (state) => + sendToMainWindow(IPC_CHANNELS.SUGGESTION.STATE_CHANGED, state) + ) + + // 제안 수락/순환/닫기는 전역 키바인딩으로만 들어온다 — 오버레이는 포커스를 갖지 않는다. + getKeyBindingService().on('triggered', (payload) => { + if (payload.type !== 'pressed') return + if (payload.actionId === 'suggestion-accept') void suggestion.accept() + else if (payload.actionId === 'suggestion-next') suggestion.next() + else if (payload.actionId === 'suggestion-prev') suggestion.previous() + else if (payload.actionId === 'suggestion-dismiss') suggestion.dismiss('dismissed') + }) + + // Chromium(Electron)은 접근성 지원이 감지될 때만 AX 트리를 만든다. 켜지 않으면 + // 우리 앱 자신의 입력창은 UIA 로 읽히지 않아 D3RO 안에서 타이핑할 때 제안이 죽는다. + if (configGet('inputTelemetryEnabled') || configGet('suggestionEnabled')) { + app.setAccessibilitySupportEnabled(true) + logger.info('[bootstrap] accessibility tree enabled for UIA context capture') + } + + telemetry.start() +} + +/** 메인 도우로만 이벤트 전송 (팝업은 자체 내부 채널을 다) */ +function sendToMainWindow(channel: IPCChannel, payload: unknown): void { + const win = getMainWindow() + if (win && !win.isDestroyed() && !win.webContents.isDestroyed()) { + win.webContents.send(channel, payload) + } +} + async function initCustomInstructions(): Promise { getCustomInstructionService().initialize() } @@ -228,6 +318,7 @@ function unregisterPopupNavKeys(): void { } } + async function initVoiceMode(): Promise { const voiceMode = getVoiceModeService() voiceMode.connectKeyBindings() diff --git a/apps/desktop/src/main/db/index.ts b/apps/desktop/src/main/db/index.ts index 8ce1c8c..d46aa9a 100644 --- a/apps/desktop/src/main/db/index.ts +++ b/apps/desktop/src/main/db/index.ts @@ -246,6 +246,99 @@ function applySchema(s: Database.Database): void { CREATE INDEX IF NOT EXISTS idx_meeting_documents_session_id ON meeting_documents(session_id); `) + // 입력 인텔리전스: 텔레메트리 집계 / 학습 문장 / 개인 문구 / 제안 이력 + s.exec(` + CREATE TABLE IF NOT EXISTS input_activity ( + id TEXT PRIMARY KEY, + date TEXT NOT NULL, + hour INTEGER NOT NULL, + app_name TEXT NOT NULL DEFAULT '', + keystrokes INTEGER NOT NULL DEFAULT 0, + shortcuts INTEGER NOT NULL DEFAULT 0, + backspaces INTEGER NOT NULL DEFAULT 0, + clicks INTEGER NOT NULL DEFAULT 0, + double_clicks INTEGER NOT NULL DEFAULT 0, + scroll_ticks INTEGER NOT NULL DEFAULT 0, + mouse_distance_px INTEGER NOT NULL DEFAULT 0, + chars INTEGER NOT NULL DEFAULT 0, + words INTEGER NOT NULL DEFAULT 0, + sentences INTEGER NOT NULL DEFAULT 0, + active_ms INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_input_activity_bucket + ON input_activity(date, hour, app_name); + CREATE INDEX IF NOT EXISTS idx_input_activity_date ON input_activity(date); + + CREATE TABLE IF NOT EXISTS typing_samples ( + id TEXT PRIMARY KEY, + text TEXT NOT NULL, + word_count INTEGER NOT NULL DEFAULT 0, + char_count INTEGER NOT NULL DEFAULT 0, + app_name TEXT, + window_title TEXT, + source TEXT NOT NULL DEFAULT 'typed', + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_typing_samples_created_at ON typing_samples(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_typing_samples_source ON typing_samples(source); + + CREATE TABLE IF NOT EXISTS personal_phrases ( + id TEXT PRIMARY KEY, + phrase TEXT NOT NULL, + count INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'typed', + last_used_at INTEGER, + created_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_personal_phrases_phrase ON personal_phrases(phrase); + CREATE INDEX IF NOT EXISTS idx_personal_phrases_count ON personal_phrases(count DESC); + + CREATE TABLE IF NOT EXISTS suggestions ( + id TEXT PRIMARY KEY, + app_name TEXT, + prefix_text TEXT NOT NULL DEFAULT '', + suggestion_text TEXT NOT NULL, + candidate_count INTEGER NOT NULL DEFAULT 1, + model TEXT, + latency_ms INTEGER, + accepted INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_suggestions_created_at ON suggestions(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_suggestions_accepted ON suggestions(accepted); + + CREATE TABLE IF NOT EXISTS phrase_edges ( + id TEXT PRIMARY KEY, + from_phrase TEXT NOT NULL, + to_phrase TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'follows', + weight INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_phrase_edges_unique + ON phrase_edges(from_phrase, to_phrase, kind); + CREATE INDEX IF NOT EXISTS idx_phrase_edges_from ON phrase_edges(from_phrase); + CREATE INDEX IF NOT EXISTS idx_phrase_edges_weight ON phrase_edges(weight DESC); + `) + + // 개인 그래프: personal_phrases 에 terms/app_name 컬럼 (기존 DB 마이그레이션) + try { + const phraseCols = s.pragma('table_info(personal_phrases)') as Array<{ name: string }> + if (!phraseCols.some((c) => c.name === 'terms')) { + s.exec('ALTER TABLE personal_phrases ADD COLUMN terms TEXT') + logger.info('Migrated: added terms column to personal_phrases') + } + if (!phraseCols.some((c) => c.name === 'app_name')) { + s.exec('ALTER TABLE personal_phrases ADD COLUMN app_name TEXT') + logger.info('Migrated: added app_name column to personal_phrases') + } + } catch (err) { + logger.warn( + `personal_phrases graph migration failed: ${err instanceof Error ? err.message : String(err)}` + ) + } + // Phase 14.5: meeting_sessions에 edited_transcript 컬럼 try { const msCols = s.pragma('table_info(meeting_sessions)') as Array<{ name: string }> diff --git a/apps/desktop/src/main/db/schema.ts b/apps/desktop/src/main/db/schema.ts index dd29247..c868dba 100644 --- a/apps/desktop/src/main/db/schema.ts +++ b/apps/desktop/src/main/db/schema.ts @@ -229,6 +229,137 @@ export const meetingDocuments = sqliteTable( export type MeetingDocumentRow = typeof meetingDocuments.$inferSelect export type NewMeetingDocumentRow = typeof meetingDocuments.$inferInsert +// ── input_activity (입력 텔레메트리) ───────────────────── +// 시간(hour)·앱 단위 집계 행. 키 내용은 저장하지 않는다 — +// ActivityWatch aw-watcher-input 과 같은 데이터 최소화 정책 (카운터 + 거리만). +export const inputActivity = sqliteTable( + 'input_activity', + { + id: text('id').primaryKey(), + /** 로컬 날짜 YYYY-MM-DD */ + date: text('date').notNull(), + /** 0~23 */ + hour: integer('hour').notNull(), + /** 포그라운드 프로세스명 (없으면 '') */ + appName: text('app_name').notNull().default(''), + keystrokes: integer('keystrokes').notNull().default(0), + shortcuts: integer('shortcuts').notNull().default(0), + backspaces: integer('backspaces').notNull().default(0), + clicks: integer('clicks').notNull().default(0), + doubleClicks: integer('double_clicks').notNull().default(0), + scrollTicks: integer('scroll_ticks').notNull().default(0), + mouseDistancePx: integer('mouse_distance_px').notNull().default(0), + chars: integer('chars').notNull().default(0), + words: integer('words').notNull().default(0), + sentences: integer('sentences').notNull().default(0), + activeMs: integer('active_ms').notNull().default(0), + updatedAt: integer('updated_at').notNull(), + }, + (table) => [ + uniqueIndex('idx_input_activity_bucket').on(table.date, table.hour, table.appName), + index('idx_input_activity_date').on(table.date), + ] +) + +// ── typing_samples (학습된 이핑/음성 문장) ───────────── +export const typingSamples = sqliteTable( + 'typing_samples', + { + id: text('id').primaryKey(), + text: text('text').notNull(), + wordCount: integer('word_count').notNull().default(0), + charCount: integer('char_count').notNull().default(0), + appName: text('app_name'), + windowTitle: text('window_title'), + source: text('source', { enum: ['typed', 'voice', 'suggestion', 'clipboard'] }) + .notNull() + .default('typed'), + createdAt: integer('created_at').notNull(), + }, + (table) => [ + index('idx_typing_samples_created_at').on(table.createdAt), + index('idx_typing_samples_source').on(table.source), + ] +) + +// ── personal_phrases (개인화 문구) ────────────────────── +export const personalPhrases = sqliteTable( + 'personal_phrases', + { + id: text('id').primaryKey(), + phrase: text('phrase').notNull(), + count: integer('count').notNull().default(1), + source: text('source', { enum: ['typed', 'voice', 'suggestion', 'clipboard'] }) + .notNull() + .default('typed'), + lastUsedAt: integer('last_used_at'), + createdAt: integer('created_at').notNull(), + /** 개인 그래프: 비교용 용어 (JSON string[]) */ + terms: text('terms'), + /** 마지막으로 쓰인 앱 (그래프 문맥 표시용) */ + appName: text('app_name'), + }, + (table) => [ + uniqueIndex('idx_personal_phrases_phrase').on(table.phrase), + index('idx_personal_phrases_count').on(table.count), + ] +) + +// ── suggestions (제안 이력 · 수락률 측정) ──────────────── +export const suggestions = sqliteTable( + 'suggestions', + { + id: text('id').primaryKey(), + appName: text('app_name'), + /** 어렛 앞 접두 (프롬프트 스트) */ + prefixText: text('prefix_text').notNull().default(''), + suggestionText: text('suggestion_text').notNull(), + candidateCount: integer('candidate_count').notNull().default(1), + model: text('model'), + latencyMs: integer('latency_ms'), + accepted: integer('accepted', { mode: 'boolean' }).notNull().default(false), + createdAt: integer('created_at').notNull(), + }, + (table) => [ + index('idx_suggestions_created_at').on(table.createdAt), + index('idx_suggestions_accepted').on(table.accepted), + ] +) + +// ── phrase_edges (개인 그래프 엣지) ────────────────────── +// 문장(노드) 사이의 관계. follows = 한 텍스트 안에서 연달아 나온 문장 쌍, +// shares_terms = 용어를 충분히 공유하는 문장 쌍. 관계형 개인화의 근거다. +export const phraseEdges = sqliteTable( + 'phrase_edges', + { + id: text('id').primaryKey(), + fromPhrase: text('from_phrase').notNull(), + toPhrase: text('to_phrase').notNull(), + kind: text('kind', { enum: ['follows', 'shares_terms'] }) + .notNull() + .default('follows'), + weight: integer('weight').notNull().default(1), + updatedAt: integer('updated_at').notNull(), + }, + (table) => [ + uniqueIndex('idx_phrase_edges_unique').on(table.fromPhrase, table.toPhrase, table.kind), + index('idx_phrase_edges_from').on(table.fromPhrase), + index('idx_phrase_edges_weight').on(table.weight), + ] +) + +export type PhraseEdgeRow = typeof phraseEdges.$inferSelect +export type NewPhraseEdgeRow = typeof phraseEdges.$inferInsert + +export type InputActivityRow = typeof inputActivity.$inferSelect +export type NewInputActivityRow = typeof inputActivity.$inferInsert +export type TypingSampleRow = typeof typingSamples.$inferSelect +export type NewTypingSampleRow = typeof typingSamples.$inferInsert +export type PersonalPhraseRow = typeof personalPhrases.$inferSelect +export type NewPersonalPhraseRow = typeof personalPhrases.$inferInsert +export type SuggestionRow = typeof suggestions.$inferSelect +export type NewSuggestionRow = typeof suggestions.$inferInsert + // ── 타입 추출 ──────────────────────────────────────────── export type History = typeof history.$inferSelect export type NewHistory = typeof history.$inferInsert diff --git a/apps/desktop/src/main/ipc/index.ts b/apps/desktop/src/main/ipc/index.ts index 33cb23d..809deb4 100644 --- a/apps/desktop/src/main/ipc/index.ts +++ b/apps/desktop/src/main/ipc/index.ts @@ -29,6 +29,8 @@ import { registerCloudSyncHandlers } from './cloud-sync-handlers' import { registerAdsHandlers } from './ads-handlers' import { registerSupportHandlers } from './support-handlers' import { registerPaymentHandlers } from './payment-handlers' +import { registerInputTelemetryHandlers } from './input-telemetry-handlers' +import { registerSuggestionHandlers } from './suggestion-handlers' import { getLogger } from '../services/LoggerService' const logger = getLogger('ipc') @@ -63,5 +65,7 @@ export function registerAllIpcHandlers(): void { registerAdsHandlers() registerSupportHandlers() registerPaymentHandlers() + registerInputTelemetryHandlers() + registerSuggestionHandlers() logger.info('All IPC handlers registered') } diff --git a/apps/desktop/src/main/ipc/input-telemetry-handlers.ts b/apps/desktop/src/main/ipc/input-telemetry-handlers.ts new file mode 100644 index 0000000..63b31cb --- /dev/null +++ b/apps/desktop/src/main/ipc/input-telemetry-handlers.ts @@ -0,0 +1,164 @@ +// src/main/ipc/input-telemetry-handlers.ts +// 입력 텔레메트리 수집 동의 · 리포트 · 개인 문구 관리. + +import { ipcMain } from 'electron' +import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' +import { ipcSuccess, ipcError, ErrorCode } from '@d3ro/core/errors' +import { getInputTelemetryService } from '../services/InputTelemetryService' +import { getPersonalGraphService } from '../services/PersonalGraphService' +import { getSuggestionService } from '../services/SuggestionService' + +export interface SetInputTelemetryEnabledParams { + enabled: boolean +} + +export interface SetInputTelemetryPausedParams { + paused: boolean +} + +export interface InputSummaryParams { + days?: number +} + +export interface InputPhrasesParams { + limit?: number +} + +export interface DeletePhraseParams { + id: string +} + +export function registerInputTelemetryHandlers(): void { + ipcMain.handle(IPC_CHANNELS.INPUT_TELEMETRY.GET_STATE, async () => { + try { + return ipcSuccess(getInputTelemetryService().getState()) + } catch (error) { + return ipcError( + ErrorCode.InputTelemetryConfigFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.SET_ENABLED, + async (_event, params: SetInputTelemetryEnabledParams) => { + try { + getInputTelemetryService().setEnabled(params.enabled === true) + // 동의를 끄면 제안도 더 이상 입력 맥락을 받을 수 없다. + if (params.enabled !== true) getSuggestionService().applyConfig() + return ipcSuccess(getInputTelemetryService().getState()) + } catch (error) { + return ipcError( + ErrorCode.InputTelemetryConfigFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.SET_PAUSED, + async (_event, params: SetInputTelemetryPausedParams) => { + try { + getInputTelemetryService().setPaused(params.paused === true) + return ipcSuccess(getInputTelemetryService().getState()) + } catch (error) { + return ipcError( + ErrorCode.InputTelemetryConfigFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.GET_SUMMARY, + async (_event, params?: InputSummaryParams) => { + try { + return ipcSuccess(getInputTelemetryService().getSummary(params?.days ?? 7)) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle(IPC_CHANNELS.INPUT_TELEMETRY.GET_PRIVACY_RECEIPT, async () => { + try { + return ipcSuccess(getInputTelemetryService().getPrivacyReceipt()) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.GET_PHRASES, + async (_event, params?: InputPhrasesParams) => { + try { + return ipcSuccess(getInputTelemetryService().listPhrases(params?.limit ?? 100)) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.DELETE_PHRASE, + async (_event, params: DeletePhraseParams) => { + try { + return ipcSuccess(getInputTelemetryService().deletePhrase(params.id)) + } catch (error) { + return ipcError( + ErrorCode.DBWriteFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle(IPC_CHANNELS.INPUT_TELEMETRY.GET_GRAPH, async () => { + try { + return ipcSuccess(getPersonalGraphService().getStats()) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.INPUT_TELEMETRY.QUERY_GRAPH, + async (_event, params: { text: string; limit?: number }) => { + try { + return ipcSuccess(getPersonalGraphService().query(params.text, params.limit ?? 12)) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle(IPC_CHANNELS.INPUT_TELEMETRY.CLEAR_ALL, async () => { + try { + getInputTelemetryService().clearAll() + return ipcSuccess(undefined) + } catch (error) { + return ipcError( + ErrorCode.DBWriteFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) +} diff --git a/apps/desktop/src/main/ipc/suggestion-handlers.ts b/apps/desktop/src/main/ipc/suggestion-handlers.ts new file mode 100644 index 0000000..9258fea --- /dev/null +++ b/apps/desktop/src/main/ipc/suggestion-handlers.ts @@ -0,0 +1,190 @@ +// src/main/ipc/suggestion-handlers.ts +// 다음 문장 제안(ghost text) 상태/설정/수락 + 오버레이 팝업 내부 채널. + +import { ipcMain } from 'electron' +import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' +import { ipcSuccess, ipcError, ErrorCode } from '@d3ro/core/errors' +import { getSuggestionService } from '../services/SuggestionService' +import { configSet } from '../services/ConfigService' +import { applySuggestionOverlayConfig, hideSuggestionOverlay } from '../windows/WindowManager' + +export interface SetSuggestionConfigParams { + enabled?: boolean + modelId?: string | null + triggerDelayMs?: number + minPrefixChars?: number + maxRequestsPerMinute?: number + dailyBudget?: number + overlayInteractive?: boolean + learnTypedText?: boolean + excludedApps?: string[] + requestTimeoutMs?: number +} + +export interface SuggestionAcceptParams { + index?: number +} + +export interface SuggestionHistoryParams { + limit?: number +} + +export function registerSuggestionHandlers(): void { + ipcMain.handle(IPC_CHANNELS.SUGGESTION.GET_STATE, async () => { + try { + return ipcSuccess(getSuggestionService().getState()) + } catch (error) { + return ipcError( + ErrorCode.SuggestionGenerationFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.SUGGESTION.SET_CONFIG, + async (_event, params: SetSuggestionConfigParams) => { + try { + const service = getSuggestionService() + if (params.enabled !== undefined) service.setEnabled(params.enabled === true) + if (params.modelId !== undefined) configSet('suggestionModelId', params.modelId || null) + if (params.triggerDelayMs !== undefined) { + configSet('suggestionTriggerDelayMs', clamp(params.triggerDelayMs, 100, 2000)) + } + if (params.minPrefixChars !== undefined) { + configSet('suggestionMinPrefixChars', clamp(params.minPrefixChars, 4, 40)) + } + if (params.maxRequestsPerMinute !== undefined) { + configSet('suggestionMaxRequestsPerMinute', clamp(params.maxRequestsPerMinute, 1, 12)) + } + if (params.dailyBudget !== undefined) { + configSet('suggestionDailyBudget', clamp(params.dailyBudget, 1, 100000)) + } + if (params.requestTimeoutMs !== undefined) { + configSet('suggestionRequestTimeoutMs', clamp(params.requestTimeoutMs, 500, 60000)) + } + if (params.overlayInteractive !== undefined) { + configSet('suggestionOverlayInteractive', params.overlayInteractive === true) + applySuggestionOverlayConfig() + } + if (params.learnTypedText !== undefined) { + configSet('inputLearnTypedText', params.learnTypedText === true) + } + if (params.excludedApps !== undefined) { + configSet('inputExcludedApps', sanitizeExcludedApps(params.excludedApps)) + } + + service.applyConfig() + return ipcSuccess(service.getState()) + } catch (error) { + return ipcError( + ErrorCode.ConfigWriteFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle(IPC_CHANNELS.SUGGESTION.REQUEST_NOW, async () => { + try { + const result = await getSuggestionService().requestNow() + return ipcSuccess(result) + } catch (error) { + return ipcError( + ErrorCode.SuggestionGenerationFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.SUGGESTION.ACCEPT, + async (_event, params?: SuggestionAcceptParams) => { + try { + const result = await getSuggestionService().accept(params?.index) + return ipcSuccess(result) + } catch (error) { + return ipcError( + ErrorCode.SuggestionAcceptFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + ipcMain.handle(IPC_CHANNELS.SUGGESTION.NEXT, async () => { + try { + return ipcSuccess(getSuggestionService().next()) + } catch (error) { + return ipcError( + ErrorCode.SuggestionGenerationFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle(IPC_CHANNELS.SUGGESTION.PREV, async () => { + try { + return ipcSuccess(getSuggestionService().previous()) + } catch (error) { + return ipcError( + ErrorCode.SuggestionGenerationFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle(IPC_CHANNELS.SUGGESTION.DISMISS, async () => { + try { + getSuggestionService().dismiss('dismissed') + return ipcSuccess(undefined) + } catch (error) { + return ipcError( + ErrorCode.SuggestionGenerationFailed, + error instanceof Error ? error.message : String(error) + ) + } + }) + + ipcMain.handle( + IPC_CHANNELS.SUGGESTION.GET_HISTORY, + async (_event, params?: SuggestionHistoryParams) => { + try { + return ipcSuccess(getSuggestionService().getHistory(params?.limit ?? 50)) + } catch (error) { + return ipcError( + ErrorCode.DBQueryFailed, + error instanceof Error ? error.message : String(error) + ) + } + } + ) + + // ── 오버레이 팝업 → 메인 (클릭 수락/닫기) ────────────── + ipcMain.on(IPC_CHANNELS.POPUP_SUGGESTION.ACCEPT, (_event, index?: number) => { + void getSuggestionService().accept(index) + }) + + ipcMain.on(IPC_CHANNELS.POPUP_SUGGESTION.DISMISS, () => { + hideSuggestionOverlay() + getSuggestionService().dismiss('dismissed') + }) +} + +function clamp(value: number, min: number, max: number): number { + const numeric = Number(value) + if (!Number.isFinite(numeric)) return min + return Math.max(min, Math.min(Math.round(numeric), max)) +} + +/** 실행 파일명만 남기고 정규화 (최대 100개). */ +function sanitizeExcludedApps(apps: string[]): string[] { + const seen = new Set() + for (const raw of apps) { + const value = String(raw).trim().slice(0, 120) + if (!value) continue + seen.add(value) + if (seen.size >= 100) break + } + return [...seen] +} diff --git a/apps/desktop/src/main/lifecycle.ts b/apps/desktop/src/main/lifecycle.ts index 5486528..f168eb1 100644 --- a/apps/desktop/src/main/lifecycle.ts +++ b/apps/desktop/src/main/lifecycle.ts @@ -3,6 +3,7 @@ import { app } from 'electron' import { getLogger } from './services/LoggerService' import { getConfigService } from './services/ConfigService' +import { disposeLocalLLMService } from './services/LocalLLMService' const logger = getLogger('lifecycle') @@ -24,6 +25,7 @@ export function setupLifecycle(): void { app.on('will-quit', () => { logger.info('[lifecycle] will-quit — flushing config') + disposeLocalLLMService() try { const configService = getConfigService() if (configService) { diff --git a/apps/desktop/src/main/services/AudioCaptureService.ts b/apps/desktop/src/main/services/AudioCaptureService.ts index 5a10c7e..08066f4 100644 --- a/apps/desktop/src/main/services/AudioCaptureService.ts +++ b/apps/desktop/src/main/services/AudioCaptureService.ts @@ -261,7 +261,7 @@ class AudioCaptureService extends EventEmitter { return new Promise((resolve) => { exec( `powershell -NoProfile -Command "${psCommand}"`, - { encoding: 'utf8', timeout: 3000, env: { ...process.env, PYTHONIOENCODING: 'utf-8' } }, + { encoding: 'utf8', timeout: 3000, env: { ...process.env, PYTHONIOENCODING: 'utf-8' }, windowsHide: true }, (err, stdout) => { if (err || !stdout?.trim()) { return resolve([ diff --git a/apps/desktop/src/main/services/ConfigService.ts b/apps/desktop/src/main/services/ConfigService.ts index ba2f88f..e705d93 100644 --- a/apps/desktop/src/main/services/ConfigService.ts +++ b/apps/desktop/src/main/services/ConfigService.ts @@ -3,12 +3,64 @@ import { EventEmitter } from 'events' import type { AppConfig, ConfigChangedEvent, KeyBindingActionId, KeyBindingMap } from '@d3ro/core/types' -import { createDefaultBindingMap, normalizeBinding, parseBindingMap } from '@d3ro/core/keybinding' +import { + createDefaultBindingMap, + findActionSpec, + normalizeBinding, + parseBindingMap +} from '@d3ro/core/keybinding' import { D3ROError, ErrorCode } from '@d3ro/core/errors' import { getLogger } from './LoggerService' const logger = getLogger('ConfigService') +/** 제안/텔레메트리 튜닝 기본값의 현재 개정판. 기본값을 바꾸면 올린다. */ +const SUGGESTION_TUNING_REVISION = 4 + +const INITIAL_SUGGESTION_TUNING = { + suggestionTriggerDelayMs: 300, + suggestionMinPrefixChars: 8, + suggestionMaxRequestsPerMinute: 12, + suggestionDailyBudget: 500 +} as const + +/** 저장된 튜닝 값을 해당 개정판에서만 변경된 항목으로 올린다. */ +function migrateSuggestionTuning(activeStore: ElectronStore): void { + const raw = activeStore.store as unknown as Record + const current = Number(raw.suggestionTuningRevision ?? 0) + if (Number.isFinite(current) && current >= SUGGESTION_TUNING_REVISION) return + + if (current < 1) { + for (const [key, value] of Object.entries(INITIAL_SUGGESTION_TUNING)) { + activeStore.set(key as keyof AppConfig, value as AppConfig[keyof AppConfig]) + } + } + + if (current < 2) { + const rawBindings = raw.keyBindings as unknown + const bindings = parseBindingMap(rawBindings) + const defaults = createDefaultBindingMap() + const suggestionActions = ['suggestion-accept', 'suggestion-next', 'suggestion-prev', 'suggestion-dismiss'] as const + for (const action of suggestionActions) { + const spec = findActionSpec(action) + if (!spec) continue + bindings[action] = defaults[action] ?? spec.defaultBindings.map((binding) => ({ ...binding })) + } + activeStore.set('keyBindings', bindings) + } + + if (current < 3) activeStore.set('suggestionRequestTimeoutMs', 8000) + if (current < 4) { + activeStore.set('suggestionTriggerDelayMs', 600) + activeStore.set('suggestionMaxRequestsPerMinute', 6) + } + + activeStore.set('suggestionTuningRevision', SUGGESTION_TUNING_REVISION) + logger.info( + `Migrated suggestion tuning values to revision ${SUGGESTION_TUNING_REVISION}` + ) +} + // electron-store v10은 ESM 전용이므로 동적 import 필요 interface ElectronStore { get(key: K): T[K] @@ -82,6 +134,21 @@ const CONFIG_DEFAULTS: AppConfig = { updateChannel: 'latest', updateDeviceId: '', skippedUpdateVersion: null, + // 입력 인텔리전스 — 옵트인. 켜기 전까지 어떤 입력도 수집하지 않는다. + inputTelemetryEnabled: false, + inputTelemetryPaused: false, + inputLearnTypedText: false, + inputExcludedApps: [], + suggestionEnabled: false, + suggestionModelId: null, + // 기본값 정본: packages/core/src/input-intelligence.ts SUGGESTION_DEFAULTS + suggestionTriggerDelayMs: 600, + suggestionMinPrefixChars: 8, + suggestionMaxRequestsPerMinute: 6, + suggestionDailyBudget: 500, + suggestionOverlayInteractive: true, + suggestionRequestTimeoutMs: 8000, + suggestionTuningRevision: 4, } let store: ElectronStore | null = null @@ -195,6 +262,7 @@ export async function initConfigService(): Promise { defaults: CONFIG_DEFAULTS }) migrateKeyBindings(store) + migrateSuggestionTuning(store) logger.info('ConfigService initialized') } diff --git a/apps/desktop/src/main/services/FileTranscriptionService.ts b/apps/desktop/src/main/services/FileTranscriptionService.ts index 40fe8ef..93c2d6c 100644 --- a/apps/desktop/src/main/services/FileTranscriptionService.ts +++ b/apps/desktop/src/main/services/FileTranscriptionService.ts @@ -282,7 +282,7 @@ class FileTranscriptionService extends EventEmitter { logger.info(`ffmpeg convert: ${ffmpegPath} ${args.join(' ')}`) - const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'] }) + const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) let stderr = '' proc.stderr?.on('data', (data: Buffer) => { @@ -319,7 +319,7 @@ class FileTranscriptionService extends EventEmitter { '-', ] - const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'] }) + const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) let stderr = '' proc.stderr?.on('data', (data: Buffer) => { @@ -369,7 +369,7 @@ class FileTranscriptionService extends EventEmitter { 'pipe:1', ] - const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'] }) + const proc = spawn(ffmpegPath, args, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) const chunks: Buffer[] = [] proc.stdout?.on('data', (data: Buffer) => { diff --git a/apps/desktop/src/main/services/HistoryService.ts b/apps/desktop/src/main/services/HistoryService.ts index 1ec59ee..b41b6aa 100644 --- a/apps/desktop/src/main/services/HistoryService.ts +++ b/apps/desktop/src/main/services/HistoryService.ts @@ -7,6 +7,7 @@ import { history, stats } from '../db/schema' import type { History, NewHistory } from '../db/schema' import { getLogger } from './LoggerService' import { getCloudSyncService } from './CloudSyncService' +import { getInputTelemetryService } from './InputTelemetryService' import type { HistoryEntry, HistoryQueryParams, @@ -43,6 +44,18 @@ class HistoryService { // 비동기로 LLM 타이틀 자동 생성 (fire-and-forget) this.generateTitle(id).catch(() => { /* ignore */ }) + // 입력 인리전스: 음성 전사도 개인 문구 코퍼스의 표본이다. + // 학습 동의(inputLearnTypedText)가 없으면 서비스 내부에서 무시된다. + try { + getInputTelemetryService().recordExternalText(input.originalText, { + appName: input.focusedAppName ?? null, + windowTitle: input.focusedAppWindowTitle ?? null, + source: 'voice' + }) + } catch { + // 학습 실패가 이력 저장을 막아서는 안 된다. + } + return this._toEntry(entry as History) } diff --git a/apps/desktop/src/main/services/InputTelemetryService.ts b/apps/desktop/src/main/services/InputTelemetryService.ts new file mode 100644 index 0000000..eabc3e2 --- /dev/null +++ b/apps/desktop/src/main/services/InputTelemetryService.ts @@ -0,0 +1,1296 @@ +// src/main/services/InputTelemetryService.ts +// +// 입력 텔레메트리 수집기 — 키보드/마우스 집계 + 포커스 입력창 텍스트 학습. +// +// 설계 근거 (조사 기반): +// - 수집 항목·집계·flush 주기는 ActivityWatch `aw-watcher-input` 따른다: +// 카운트(keystrokes/clicks/scroll) + 마우스 이동량(축별 절대값 합)만 남기고 +// **키 내용은 저장하지 않는다**. 5초 heartbeat 로 집계를 확정한다. +// - 키 이벤트는 uiohook(이미 사용 중) 글로벌 후킹에서 받는다. 후킹은 프로세스 +// 전역 글턴이므로 global-input-hook 의 참조 카운트로 KeyBindingService 와 공존한다. +// - 이핑한 텍스트는 키코드로 복원하지 않는다 (한/일 IME 는 조합 결과가 텍스트라 +// 원리적으로 불가). 대신 UIA 로 커밋된 텍스트를 읽어 스냅샷 diff 로 계산한다. +// - 비밀번호 필드(IsPassword)와 IME 조합 중에는 스트를 쓰지 않는다 (fail-closed). +// +// 동의(consent) 기본값은 모두 OFF 다. `inputTelemetryEnabled` 없이는 아무것도 하지 않는다. + +import { EventEmitter } from 'events' +import { screen } from 'electron' +import { uIOhook } from 'uiohook-napi' +import type { UiohookKeyboardEvent, UiohookMouseEvent, UiohookWheelEvent } from 'uiohook-napi' +import { sql, gte, eq, desc } from 'drizzle-orm' +import { + INPUT_TELEMETRY_DEFAULTS, + SUGGESTION_CONTEXT_MAX_CHARS, + calculateFrictionInsight, + classifyKeyStroke, + computeTypedDelta, + countWords, + emptyActivityBucket, + extractPhrases, + manhattanDistance, + mergeActivityBucket, + rankFlowWindows, + recommendAppExclusion, + summarizeActivity, + textBeforeCaret, + type FocusSnapshot, + type InputActivityBucket, + type InputAppStat, + type InputAppSuggestionStat, + type InputDailyStat, + type InputHourlyStat, + type InputInsightsSummary, + type InputPrivacyReceipt, + type InputTelemetryState, + type InputSnapshotSummary, + type AppReadabilityEvidence, + type PersonalPhrase, + type PhraseSource, + type UiRect +} from '@d3ro/core/input-intelligence' +import { getDatabase } from '../db' +import { inputActivity, personalPhrases, suggestions as suggestionsTable, typingSamples } from '../db/schema' +import { configGet, configSet } from './ConfigService' +import { getLogger } from './LoggerService' +import { acquireGlobalInputHook } from './global-input-hook' +import { uiohookCodeToVk } from './KeyBindingService' +import { getUiaContextService } from './UiaContextService' +import { getForegroundWindowInfo } from '../utils/win32-foreground' +import { getPersonalGraphService } from './PersonalGraphService' + +const logger = getLogger('InputTelemetryService') + +/** 제안 서비스가 소비하는 "지금 치고 있는 것" 스냅샷. */ +export interface TypingContext { + /** 어렛 앞 텍스트 (>제안 프롬프트 입력) */ + prefix: string + fullText: string + caretOffset: number | null + anchor: UiRect | null + isPassword: boolean + isEditable: boolean + isComposing: boolean + hasSelection: boolean + available: boolean + appName: string | null + windowTitle: string | null + idleMs: number + capturedAt: number +} + +interface InputTelemetryEvents { + 'state-changed': (state: InputTelemetryState) => void + 'typed-context': (context: TypingContext) => void + activity: (payload: { totals: InputActivityBucket; appName: string | null }) => void +} + +interface ForegroundInfo { + appName: string | null + windowTitle: string | null + hwnd: number | null +} + +const MAX_SAMPLE_CHARS = 500 +const RETENTION_PRUNE_INTERVAL_MS = 6 * 60 * 60 * 1000 +const MAX_READABILITY_APPS = 64 + +class InputTelemetryService extends EventEmitter { + private _running = false + private _releaseHook: (() => void) | null = null + + /** 아직 DB에 쓰지 않은 버킷 (key = date|hour|appName) */ + private _pending: Map = new Map() + /** 세션 누적 (라이브 UI 용) */ + private _sessionTotals: InputActivityBucket = emptyActivityBucket() + + private _flushTimer: NodeJS.Timeout | null = null + private _textTimer: NodeJS.Timeout | null = null + /** 디바운스 이후의 settle 스냅샷 — 제안 게이트는 더 늦게 열린다 */ + private _settleTimer: NodeJS.Timeout | null = null + /** 백그라운드 주기 샘플러 — 타이핑이 이어져도 계속 관측한다 */ + private _sampleTimer: NodeJS.Timeout | null = null + /** 주기 샘플 스냅샷 진행 중 (중복 방지) */ + private _sampling = false + /** 제안 오버레이가 표시 중일 때만 포커스 이탈/선택을 계속 확인한다. */ + private _suggestionPresentationActive = false + private _presentationValidationTimer: NodeJS.Timeout | null = null + + private _lastEventAt = 0 + /** + * 마지막 키보드 입력 시각. + * + * 제안 문맥은 키보드 기준으로만 판단한다 — 마우스 클릭만으로 스냅샷이 돌면 + * 필드에 이미 있던 텍스트로 제안이 만들어졌다(실측 신고). + */ + private _lastKeyAt = 0 + private _lastActiveTickAt = 0 + private _lastMouse: { x: number; y: number } | null = null + private _lastClickAt = 0 + private _lastClickButton = -1 + private _lastClickPos: { x: number; y: number } | null = null + + private _modifiers = { ctrl: false, alt: false, shift: false, meta: false } + + private _foreground: ForegroundInfo = { appName: null, windowTitle: null, hwnd: null } + private _foregroundSampledAt = 0 + + private _lastSnapshot: FocusSnapshot | null = null + /** 마지막 스냅샷이 caret 폴백(tail 가정)을 썠는지 — 진단용 */ + private _caretFallback = false + private _lastSnapshotKey = '' + private _lastContext: TypingContext | null = null + /** UI 진단용 스냅샷 요약 */ + private _lastSnapshotSummary: InputSnapshotSummary | null = null + /** 원문 없이 앱별 UIA 읽기 가능성만 누적한다. */ + private _appReadability = new Map() + + private _prunedAt = 0 + + private _onKeyDown: ((e: UiohookKeyboardEvent) => void) | null = null + private _onKeyUp: ((e: UiohookKeyboardEvent) => void) | null = null + private _onMouseDown: ((e: UiohookMouseEvent) => void) | null = null + private _onMouseUp: ((e: UiohookMouseEvent) => void) | null = null + private _onMouseMove: ((e: UiohookMouseEvent) => void) | null = null + private _onWheel: ((e: UiohookWheelEvent) => void) | null = null + + // ── 타입 안전한 이벤트 오버로드 ──────────────────────── + override on(event: K, listener: InputTelemetryEvents[K]): this { + return super.on(event, listener as (...args: unknown[]) => void) + } + + override off( + event: K, + listener: InputTelemetryEvents[K] + ): this { + return super.off(event, listener as (...args: unknown[]) => void) + } + + override emit( + event: K, + ...args: Parameters + ): boolean { + return super.emit(event, ...args) + } + + get isRunning(): boolean { + return this._running + } + + get lastContext(): TypingContext | null { + return this._lastContext + } + + setSuggestionPresentationActive(active: boolean): void { + this._suggestionPresentationActive = active + if (!active && this._presentationValidationTimer) { + clearTimeout(this._presentationValidationTimer) + this._presentationValidationTimer = null + } + } + + // ─ 수명 주기 ────────────────────────────────────────── + + /** 설정이 허용하면 수집을 시작한다. 실패해도 앱은 계속 돌아간다. */ + start(): void { + if (!this._isEnabledByConfig()) { + logger.info('입력 텔레메트리 비활성 (설정 또는 일시정지)') + return + } + if (this._running) return + + try { + this._onKeyDown = (e: UiohookKeyboardEvent) => this._handleKeyDown(e) + this._onKeyUp = (e: UiohookKeyboardEvent) => this._handleKeyUp(e) + this._onMouseDown = (e: UiohookMouseEvent) => this._handleMouseDown(e) + this._onMouseUp = (e: UiohookMouseEvent) => this._handleMouseUp(e) + this._onMouseMove = (e: UiohookMouseEvent) => this._handleMouseMove(e) + this._onWheel = (e: UiohookWheelEvent) => this._handleWheel(e) + + uIOhook.on('keydown', this._onKeyDown) + uIOhook.on('keyup', this._onKeyUp) + uIOhook.on('mousedown', this._onMouseDown) + uIOhook.on('mouseup', this._onMouseUp) + uIOhook.on('mousemove', this._onMouseMove) + uIOhook.on('wheel', this._onWheel) + + this._releaseHook = acquireGlobalInputHook() + this._running = true + this._flushTimer = setInterval( + () => this._flush(), + INPUT_TELEMETRY_DEFAULTS.flushIntervalMs + ) + this._flushTimer.unref?.() + + // 주기 샘플러. + // + // 스냅샷이 "타이핑을 멈춘 뒤" 에만 돌면, 계속 치는 동안에는 디바운스 타이머가 + // 계속 리셋돼 샘플이 하나도 만들어지지 않는다(실측: 에이전트 개발처럼 + // 끊김 없이 치는 상황에서 제안이 전혀 안 뜸). 최근 입력이 있었으면 + // 주기적으로 스냅샷을 만들어, 타이핑 중에도 제안이 계속 갱신되게 한다. + this._sampleTimer = setInterval( + () => this._sampleWhileTyping(), + INPUT_TELEMETRY_DEFAULTS.sampleIntervalMs + ) + this._sampleTimer.unref?.() + + this._pruneOldData() + logger.info('입력 텔레메트리 시작 (키/마우스 집계)') + this._emitState() + } catch (error) { + logger.error( + `입력 텔레메트리 시작 실패: ${error instanceof Error ? error.message : String(error)}` + ) + this._detachListeners() + } + } + + stop(): void { + if (!this._running) { + this.setSuggestionPresentationActive(false) + return + } + this._detachListeners() + this._releaseHook?.() + this._releaseHook = null + this._running = false + + if (this._flushTimer) { + clearInterval(this._flushTimer) + this._flushTimer = null + } + if (this._sampleTimer) { + clearInterval(this._sampleTimer) + this._sampleTimer = null + } + if (this._textTimer) { + clearTimeout(this._textTimer) + this._textTimer = null + } + if (this._settleTimer) { + clearTimeout(this._settleTimer) + this._settleTimer = null + } + this.setSuggestionPresentationActive(false) + this._flush() + this._lastSnapshot = null + this._lastSnapshotKey = '' + logger.info('입력 텔레메트리 중지') + this._emitState() + } + + /** 설정 변경 반영 (enable/pause/excludedApps). */ + applyConfig(): void { + const shouldRun = this._isEnabledByConfig() + if (shouldRun && !this._running) this.start() + else if (!shouldRun && this._running) this.stop() + this._emitState() + } + + dispose(): void { + this.stop() + this.removeAllListeners() + } + + private _detachListeners(): void { + if (this._onKeyDown) { + uIOhook.removeListener('keydown', this._onKeyDown) + this._onKeyDown = null + } + if (this._onKeyUp) { + uIOhook.removeListener('keyup', this._onKeyUp) + this._onKeyUp = null + } + if (this._onMouseDown) { + uIOhook.removeListener('mousedown', this._onMouseDown) + this._onMouseDown = null + } + if (this._onMouseUp) { + uIOhook.removeListener('mouseup', this._onMouseUp) + this._onMouseUp = null + } + if (this._onMouseMove) { + uIOhook.removeListener('mousemove', this._onMouseMove) + this._onMouseMove = null + } + if (this._onWheel) { + uIOhook.removeListener('wheel', this._onWheel) + this._onWheel = null + } + } + + private _isEnabledByConfig(): boolean { + return configGet('inputTelemetryEnabled') === true && configGet('inputTelemetryPaused') !== true + } + + // ── 이벤트 수집 ──────────────────────────────────────── + + private _handleKeyDown(event: UiohookKeyboardEvent): void { + this._syncModifiers(event) + this._registerInput(true) + const bucket = this._bucket() + // uiohook 키코드는 VK 와 다른 표계다 — 정본(VK)으로 변환해 분류한다. + const vk = uiohookCodeToVk(event.keycode) + const keyClass = vk === null ? 'other' : classifyKeyStroke(vk, this._modifiers) + + if (keyClass === 'shortcut') bucket.shortcuts += 1 + else bucket.keystrokes += 1 + if (keyClass === 'backspace') bucket.backspaces += 1 + + this._lastKeyAt = Date.now() + + this._scheduleTextSnapshot() + } + + private _handleKeyUp(event: UiohookKeyboardEvent): void { + this._syncModifiers(event) + } + + private _handleMouseDown(event: UiohookMouseEvent): void { + this._registerInput(false) + + const now = Date.now() + const position = { x: event.x, y: event.y } + const button = Number(event.button) + const isDoubleClick = + now - this._lastClickAt < 400 && + this._lastClickButton === button && + this._lastClickPos !== null && + manhattanDistance(this._lastClickPos, position) < 6 + + const bucket = this._bucket() + if (isDoubleClick) bucket.doubleClicks += 1 + else bucket.clicks += 1 + + this._lastClickAt = now + this._lastClickButton = button + this._lastClickPos = position + } + + private _handleMouseUp(event: UiohookMouseEvent): void { + this._lastMouse = { x: event.x, y: event.y } + if (!this._suggestionPresentationActive) return + if (this._presentationValidationTimer) clearTimeout(this._presentationValidationTimer) + this._presentationValidationTimer = setTimeout(() => { + this._presentationValidationTimer = null + void this._captureTextSnapshot() + }, 120) + this._presentationValidationTimer.unref?.() + } + + private _handleMouseMove(event: UiohookMouseEvent): void { + const position = { x: event.x, y: event.y } + if (this._lastMouse) { + const distance = manhattanDistance(this._lastMouse, position) + // 픽셀 단위 미세抖动 제거 (ActivityWatch 는 전량 누적하지만 노이즈가 크다) + if (distance >= 2) { + this._bucket().mouseDistancePx += distance + this._registerInput(false) + } + } + this._lastMouse = position + } + + private _handleWheel(event: UiohookWheelEvent): void { + const bucket = this._bucket() + bucket.scrollTicks += Math.max(1, Math.abs(Number(event.rotation) || 1)) + this._registerInput(false) + } + + /** + * 활성 시간 누적 + 포그라운드 샘플링. + * + * ActivityWatch 처럼 "입력이 있으면 활성" 으로 판단한다. 연속 입력 간격이 + * activeIdleTimeoutMs 를 넘으면 그 구간은 활성 시간에서 제외한다. + */ + private _registerInput(isKeyboard: boolean): void { + const now = Date.now() + const gap = this._lastEventAt === 0 ? 0 : now - this._lastEventAt + + if (gap > 0 && gap <= INPUT_TELEMETRY_DEFAULTS.activeIdleTimeoutMs) { + this._bucket().activeMs += gap + } else if (isKeyboard && this._lastActiveTickAt > 0) { + this._bucket().activeMs += 250 + } + + if (isKeyboard) this._lastActiveTickAt = now + this._lastEventAt = now + this._sampleForeground(now) + } + + /** + * 수정자 상태 — uiohook 이 모든 키 이벤트에 실어 보내는 플래그를 그대로 다. + * (별도 상태 머신을 두면 키 반복/포커스 이동에서 어긋난다) + */ + private _syncModifiers(event: UiohookKeyboardEvent): void { + this._modifiers = { + ctrl: event.ctrlKey === true, + alt: event.altKey === true, + shift: event.shiftKey === true, + meta: event.metaKey === true + } + } + + /** 포그라운드 창 정보 — 1초에 한 번만 갱신 (FFI 호출 비용) */ + private _sampleForeground(now = Date.now()): void { + if (now - this._foregroundSampledAt < INPUT_TELEMETRY_DEFAULTS.activeWindowSampleMinIntervalMs) { + return + } + this._foregroundSampledAt = now + + const info = getForegroundWindowInfo() + if (info) { + this._foreground = { + appName: info.appName || null, + windowTitle: info.title || null, + hwnd: info.hwnd + } + } + } + + /** 이 시각의 (날짜, 시간, 앱) 버킷 */ + private _bucket(now = new Date()): InputActivityBucket { + const date = formatLocalDate(now) + const hour = now.getHours() + const appName = this._foreground.appName ?? '' + const key = `${date}|${hour}|${appName}` + + let bucket = this._pending.get(key) + if (!bucket) { + bucket = emptyActivityBucket() + this._pending.set(key, bucket) + } + return bucket + } + + // ── 텍스트 학습 (UIA 스냅샷 diff) ────────────────────── + + /** + * 최근 입력이 있었으면 주기적으로 스냅샷을 만든다 (백그라운드 관측). + * + * 입력이 끊긴 뒤에는 굳이 돌리지 않는다 — 그 시점은 디바운스/ settle 타이머가 + * 이미 담당한다. + */ + private async _sampleWhileTyping(): Promise { + if (!this._running || this._sampling) return + const hasRecentKeyboardInput = + this._lastKeyAt > 0 && + Date.now() - this._lastKeyAt <= INPUT_TELEMETRY_DEFAULTS.sampleActiveWindowMs + if (!hasRecentKeyboardInput && !this._suggestionPresentationActive) return + + this._sampling = true + try { + await this._captureTextSnapshot() + } finally { + this._sampling = false + } + } + + private _scheduleTextSnapshot(): void { + if (this._textTimer) clearTimeout(this._textTimer) + this._textTimer = setTimeout(() => { + this._textTimer = null + void this._captureTextSnapshot() + }, INPUT_TELEMETRY_DEFAULTS.textSnapshotDebounceMs) + this._textTimer.unref?.() + + // 제안 트리거 지연(기본 1000ms)이 스냅샷 디바운스(700ms)보다 크다. 스냅샷이 + // 디바운스에만 돌면 "타이핑을 멈춘 뒤 N ms" 조건이 결코 충족되지 않아 + // 제안이 영원히 요청되지 않는다(실측: debounce 로그만 반복). 그래서 + // 트리거 지연을 넘긴 시점에 한 번 더 스냅샷을 돌려 게이트를 연다. + if (this._settleTimer) clearTimeout(this._settleTimer) + const settleDelay = Math.max( + INPUT_TELEMETRY_DEFAULTS.textSnapshotDebounceMs + 300, + (configGet('suggestionTriggerDelayMs') || 1000) + 400 + ) + this._settleTimer = setTimeout(() => { + this._settleTimer = null + void this._captureTextSnapshot() + }, settleDelay) + this._settleTimer.unref?.() + } + + /** + * 포커스 입력창 스냅샷을 읽어 (1) 타이핑 통계를 갱신하고 + * (2) 제안 서비스에 컨텍스트 이벤트를 보낸다. + */ + private async _captureTextSnapshot(): Promise { + if (!this._running) return null + + const uia = getUiaContextService() + const snapshot = await uia.getSnapshot() + const now = Date.now() + // 유휴 시간은 키보드 기준이다 (마우스 이동/클릭으로는 제안하지 않는다). + const idleMs = this._lastKeyAt > 0 ? now - this._lastKeyAt : Number.MAX_SAFE_INTEGER + + const context: TypingContext = { + prefix: '', + fullText: '', + caretOffset: null, + anchor: null, + isPassword: snapshot.isPassword, + isEditable: snapshot.isEditable, + isComposing: snapshot.isComposing, + hasSelection: snapshot.hasSelection, + available: snapshot.available, + appName: this._foreground.appName, + windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle, + idleMs, + capturedAt: snapshot.capturedAt + } + + if (snapshot.available) { + context.anchor = snapshot.caretRect ?? snapshot.elementRect + context.caretOffset = snapshot.caretOffset + context.fullText = snapshot.text + // 케어렛 오프셋을 못 얻는 제공자가 많다(실측: Notepad — src=value, caret=null). + // 그때는 "문서 마지막 = 커서 앞" 으로 가정해 tail 만 쓴다. 문서 전체를 접두로 + // 쓰면 프롬프트가 실제 입력 위치와 무관해진다. + const caretKnown = snapshot.caretOffset !== null + // 문서형 컨트롤(터미널 버퍼 등)은 tail 이 직전 출력/이전 줄까지 포함한다. + // "지금 치는 줄" 만 접두로 써야 한다 — 마지막 줄을 취한다. + const documentLike = snapshot.textSource === 'text' + const tail = snapshot.text.slice(-SUGGESTION_CONTEXT_MAX_CHARS) + context.prefix = caretKnown + ? textBeforeCaret(snapshot.text, snapshot.caretOffset) + : documentLike + ? lastLineOf(tail) + : tail + this._caretFallback = !caretKnown + + if (snapshot.isEditable && !snapshot.isPassword && !snapshot.isComposing) { + this._applyTypedDelta(snapshot) + } + } + + this._recordReadabilityEvidence(snapshot) + + // 진단: 왜 제안이 안 뜨는지 로그만으로 판단할 수 있게 스냅샷 요약을 남긴다. + logger.debug( + `입력 스냅샷: avail=${snapshot.available} edit=${snapshot.isEditable} ` + + `pw=${snapshot.isPassword} comp=${snapshot.isComposing} src=${snapshot.textSource} ` + + `len=${snapshot.text.length} caret=${snapshot.caretOffset ?? -1} tail=${this._caretFallback} ` + + `anchor=${context.anchor ? 'yes' : 'no'} app=${this._foreground.appName ?? '-'} ` + + `reason=${snapshot.reason ?? '-'}` + ) + + this._lastSnapshotSummary = { + at: context.capturedAt, + appName: this._foreground.appName, + windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle, + editable: snapshot.isEditable, + isPassword: snapshot.isPassword, + composing: snapshot.isComposing, + textSource: snapshot.textSource, + textLength: snapshot.text.length, + caretFallback: this._caretFallback + } + + this._lastContext = context + this.emit('typed-context', context) + // 진단 줄이 즉시 갱신되도록 상태 변경을 알린다 (키 입력마다가 아니라 스냅샷마다). + this._emitState() + return context + } + + private _recordReadabilityEvidence(snapshot: FocusSnapshot): void { + const appName = this._foreground.appName?.trim() + if (!appName || snapshot.isPassword) return + + let evidence = this._appReadability.get(appName) + if (!evidence) { + if (this._appReadability.size >= MAX_READABILITY_APPS) { + const oldest = this._appReadability.keys().next().value + if (oldest) this._appReadability.delete(oldest) + } + evidence = { appName, samples: 0, readable: 0, unreadable: 0, empty: 0 } + this._appReadability.set(appName, evidence) + } + + evidence.samples += 1 + if (snapshot.isEditable && snapshot.textSource !== 'none' && snapshot.text.length > 0) { + evidence.readable += 1 + } else if (snapshot.isEditable && snapshot.textSource !== 'none' && snapshot.text.length === 0) { + evidence.empty += 1 + } else { + evidence.unreadable += 1 + } + } + + private _applyTypedDelta(snapshot: FocusSnapshot): void { + const key = `${snapshot.windowTitle ?? ''}|${snapshot.controlName ?? ''}|${snapshot.controlType ?? ''}` + const previous = this._lastSnapshot + const focusChanged = key !== this._lastSnapshotKey + + this._lastSnapshot = snapshot + this._lastSnapshotKey = key + + // 필드가 바뀌면 이전 텍스트와 diff 할 수 없다 — 기준선만 저장하고 통계는 건너뛴다. + if (focusChanged || !previous) return + + const delta = computeTypedDelta(previous.text, snapshot.text, { + keepText: configGet('inputLearnTypedText') === true + }) + if (delta.replaced) return + if (delta.insertedChars === 0 && delta.insertedWords === 0) return + + const bucket = this._bucket() + bucket.chars += delta.insertedChars + bucket.words += delta.insertedWords + bucket.sentences += delta.insertedSentences + + if (delta.insertedText) { + this._learnText(delta.insertedText, { + appName: this._foreground.appName, + windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle, + source: 'typed' + }) + } + } + + /** + * 외부(음성 전사 등)에서 들어온 텍스트를 코퍼스에 넣는다. + * + * 음성 전사는 이미 사용자가 말한 문장이므로 제안 품질에 가장 강한 신호다. + * 학습 동의(`inputLearnTypedText`)가 없으면 아무것도 저장하지 않는다. + */ + recordExternalText( + text: string, + meta: { appName?: string | null; windowTitle?: string | null; source: PhraseSource } + ): void { + if (configGet('inputLearnTypedText') !== true) return + this._learnText(text, { + appName: meta.appName ?? null, + windowTitle: meta.windowTitle ?? null, + source: meta.source + }) + } + + private _learnText( + raw: string, + meta: { appName: string | null; windowTitle: string | null; source: PhraseSource } + ): void { + const text = sanitizeSample(raw) + if (!text) return + + try { + const db = getDatabase() + const now = Date.now() + const wordCount = countWords(text) + const charCount = text.length + + db.insert(typingSamples) + .values({ + id: crypto.randomUUID(), + text: text.slice(0, MAX_SAMPLE_CHARS), + wordCount, + charCount, + appName: meta.appName, + windowTitle: meta.windowTitle, + source: meta.source, + createdAt: now + }) + .run() + + // 개인 그래프에 반영 (문장 노드 + follows/용어공유 엣지) + getPersonalGraphService().indexText(text, { + source: meta.source, + appName: meta.appName, + at: now + }) + + for (const phrase of extractPhrases(text)) { + db.insert(personalPhrases) + .values({ + id: crypto.randomUUID(), + phrase, + count: 1, + source: meta.source, + lastUsedAt: now, + createdAt: now, + appName: meta.appName + }) + .onConflictDoUpdate({ + target: personalPhrases.phrase, + set: { + count: sql`${personalPhrases.count} + 1`, + lastUsedAt: now, + appName: meta.appName + } + }) + .run() + } + } catch (error) { + logger.warn( + `텍스트 학습 저장 실패: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + + // ─ 영속화 ───────────────────────────────────────────── + + /** 대기 중인 버킷을 DB로 확정한다 (5초 heartbeat). */ + flushNow(): void { + this._flush() + } + + private _flush(): void { + if (this._pending.size === 0) return + + const entries = [...this._pending.entries()] + this._pending.clear() + const now = Date.now() + + try { + const db = getDatabase() + for (const [key, bucket] of entries) { + const [date, hourRaw, appName] = key.split('|') + const hour = Number(hourRaw) + if (!date || Number.isNaN(hour)) continue + + mergeActivityBucket(this._sessionTotals, bucket) + + db.insert(inputActivity) + .values({ + id: crypto.randomUUID(), + date, + hour, + appName: appName ?? '', + ...bucket, + updatedAt: now + }) + .onConflictDoUpdate({ + target: [inputActivity.date, inputActivity.hour, inputActivity.appName], + set: { + keystrokes: sql`${inputActivity.keystrokes} + ${bucket.keystrokes}`, + shortcuts: sql`${inputActivity.shortcuts} + ${bucket.shortcuts}`, + backspaces: sql`${inputActivity.backspaces} + ${bucket.backspaces}`, + clicks: sql`${inputActivity.clicks} + ${bucket.clicks}`, + doubleClicks: sql`${inputActivity.doubleClicks} + ${bucket.doubleClicks}`, + scrollTicks: sql`${inputActivity.scrollTicks} + ${bucket.scrollTicks}`, + mouseDistancePx: sql`${inputActivity.mouseDistancePx} + ${bucket.mouseDistancePx}`, + chars: sql`${inputActivity.chars} + ${bucket.chars}`, + words: sql`${inputActivity.words} + ${bucket.words}`, + sentences: sql`${inputActivity.sentences} + ${bucket.sentences}`, + activeMs: sql`${inputActivity.activeMs} + ${bucket.activeMs}`, + updatedAt: now + } + }) + .run() + } + this.emit('activity', { + totals: { ...this._sessionTotals }, + appName: this._foreground.appName + }) + } catch (error) { + logger.warn( + `입력 텔레메트리 저장 실패(다음 주기에 재시도): ${ + error instanceof Error ? error.message : String(error) + }` + ) + // 저장 실패 시 버킷을 되돌려 유실을 막는다. + for (const [key, bucket] of entries) { + const existing = this._pending.get(key) + if (existing) mergeActivityBucket(existing, bucket) + else this._pending.set(key, bucket) + } + } + } + + // ── 조회 / 관리 ──────────────────────────────────────── + + getState(): InputTelemetryState { + const uia = getUiaContextService() + const currentApp = this._foreground.appName + const excludedApps = [...configGet('inputExcludedApps')] + const evidence = currentApp ? this._appReadability.get(currentApp) : undefined + return { + enabled: configGet('inputTelemetryEnabled') === true, + paused: configGet('inputTelemetryPaused') === true, + running: this._running, + learnTypedText: configGet('inputLearnTypedText') === true, + excludedApps, + appName: currentApp, + windowTitle: this._foreground.windowTitle, + bridgeReason: uia.lastReason, + bridgeAvailable: uia.isAvailable(), + lastSnapshotAt: uia.lastSuccessAt, + lastSnapshot: this._lastSnapshotSummary, + exclusionRecommendation: + evidence && !excludedApps.some((app) => app.toLowerCase() === currentApp?.toLowerCase()) + ? recommendAppExclusion(evidence) + : null + } + } + + /** 최근 N일 인사이트 (일별/시간대별/앱별). */ + getSummary(days = 7): InputInsightsSummary { + const safeDays = Math.max(1, Math.min(days, 365)) + const since = new Date() + since.setHours(0, 0, 0, 0) + since.setDate(since.getDate() - (safeDays - 1)) + const sinceDate = formatLocalDate(since) + + const totals = emptyActivityBucket() + const daily = new Map() + const hourly = new Map() + const apps = new Map() + + try { + const db = getDatabase() + const rows = db + .select() + .from(inputActivity) + .where(gte(inputActivity.date, sinceDate)) + .all() + + for (const row of rows) { + mergeActivityBucket(totals, { + keystrokes: row.keystrokes, + shortcuts: row.shortcuts, + backspaces: row.backspaces, + clicks: row.clicks, + doubleClicks: row.doubleClicks, + scrollTicks: row.scrollTicks, + mouseDistancePx: row.mouseDistancePx, + chars: row.chars, + words: row.words, + sentences: row.sentences, + activeMs: row.activeMs + }) + + const day = daily.get(row.date) ?? { + date: row.date, + keystrokes: 0, + clicks: 0, + words: 0, + sentences: 0, + mouseDistancePx: 0, + activeMs: 0 + } + day.keystrokes += row.keystrokes + day.clicks += row.clicks + day.words += row.words + day.sentences += row.sentences + day.mouseDistancePx += row.mouseDistancePx + day.activeMs += row.activeMs + daily.set(row.date, day) + + const hour = hourly.get(row.hour) ?? { + hour: row.hour, + keystrokes: 0, + clicks: 0, + chars: 0, + backspaces: 0, + activeMs: 0 + } + hour.keystrokes += row.keystrokes + hour.clicks += row.clicks + hour.chars += row.chars + hour.backspaces += row.backspaces + hour.activeMs += row.activeMs + hourly.set(row.hour, hour) + + if (row.appName) { + const app = apps.get(row.appName) ?? { + appName: row.appName, + keystrokes: 0, + clicks: 0, + activeMs: 0 + } + app.keystrokes += row.keystrokes + app.clicks += row.clicks + app.activeMs += row.activeMs + apps.set(row.appName, app) + } + } + + // 24시간 전체 분포 (topHours 는 상위만 잘라 쓴다) + const hourlyAll: InputHourlyStat[] = Array.from({ length: 24 }, (_, hour) => ({ + hour, + keystrokes: 0, + clicks: 0, + chars: 0, + backspaces: 0, + activeMs: 0 + })) + for (const stat of hourly.values()) { + if (stat.hour >= 0 && stat.hour < 24) hourlyAll[stat.hour] = stat + } + + const dailySorted = [...daily.values()].sort((a, b) => a.date.localeCompare(b.date)) + const activeDays = dailySorted.length + const peakDay = dailySorted.reduce( + (best, day) => (!best || day.keystrokes > best.keystrokes ? day : best), + null + ) + + // 연속 기록 일수 (최장) + let longestStreakDays = 0 + let currentStreak = 0 + let previousDate: string | null = null + for (const day of dailySorted) { + if (previousDate === null) { + currentStreak = 1 + } else { + const prev = new Date(`${previousDate}T00:00:00`) + const gapDays = Math.round( + (new Date(`${day.date}T00:00:00`).getTime() - prev.getTime()) / 86400000 + ) + currentStreak = gapDays === 1 ? currentStreak + 1 : 1 + } + longestStreakDays = Math.max(longestStreakDays, currentStreak) + previousDate = day.date + } + + const suggestionTotals = db + .select({ + total: sql`count(*)`, + accepted: sql`coalesce(sum(case when accepted then 1 else 0 end), 0)`, + avgLatency: sql`avg(latency_ms)` + }) + .from(suggestionsTable) + .where(gte(suggestionsTable.createdAt, Date.now() - safeDays * 86400000)) + .get() + const suggestionStats: InputInsightsSummary['suggestions'] = { + total: Number(suggestionTotals?.total ?? 0), + accepted: Number(suggestionTotals?.accepted ?? 0), + acceptRate: + Number(suggestionTotals?.total ?? 0) > 0 + ? Number(suggestionTotals?.accepted ?? 0) / Number(suggestionTotals?.total ?? 1) + : 0, + avgLatencyMs: + suggestionTotals?.avgLatency === null || suggestionTotals?.avgLatency === undefined + ? null + : Math.round(Number(suggestionTotals.avgLatency)) + } + + const suggestionRows = db + .select({ + appName: suggestionsTable.appName, + accepted: suggestionsTable.accepted, + latencyMs: suggestionsTable.latencyMs + }) + .from(suggestionsTable) + .where(gte(suggestionsTable.createdAt, Date.now() - safeDays * 86400000)) + .all() + const suggestionAppsByName = new Map< + string, + { total: number; accepted: number; latencyTotal: number; latencyCount: number } + >() + for (const row of suggestionRows) { + const appName = row.appName?.trim() + if (!appName) continue + const stat = suggestionAppsByName.get(appName) ?? { + total: 0, + accepted: 0, + latencyTotal: 0, + latencyCount: 0 + } + stat.total += 1 + if (row.accepted) stat.accepted += 1 + if (row.latencyMs !== null && row.latencyMs !== undefined) { + stat.latencyTotal += row.latencyMs + stat.latencyCount += 1 + } + suggestionAppsByName.set(appName, stat) + } + const suggestionApps: InputAppSuggestionStat[] = [...suggestionAppsByName.entries()] + .map(([appName, stat]) => ({ + appName, + total: stat.total, + accepted: stat.accepted, + acceptRate: stat.total > 0 ? stat.accepted / stat.total : 0, + avgLatencyMs: stat.latencyCount > 0 ? Math.round(stat.latencyTotal / stat.latencyCount) : null + })) + .sort((a, b) => b.total - a.total || b.acceptRate - a.acceptRate || a.appName.localeCompare(b.appName)) + .slice(0, 12) + + const phraseCount = db + .select({ value: sql`count(*)` }) + .from(personalPhrases) + .get() + const sampleCount = db + .select({ value: sql`count(*)` }) + .from(typingSamples) + .get() + + return { + days: safeDays, + totals, + daily: dailySorted, + topHours: [...hourly.values()].sort((a, b) => b.keystrokes - a.keystrokes).slice(0, 6), + topApps: [...apps.values()].sort((a, b) => b.keystrokes - a.keystrokes).slice(0, 12), + phraseCount: Number(phraseCount?.value ?? 0), + sampleCount: Number(sampleCount?.value ?? 0), + averages: summarizeActivity(totals, safeDays, getDisplayScaleFactor()), + hourly: hourlyAll, + activeDays, + longestStreakDays, + peakDay, + suggestions: suggestionStats, + friction: calculateFrictionInsight(totals.chars, totals.backspaces), + flowWindows: rankFlowWindows(hourlyAll, activeDays), + suggestionApps + } + } catch (error) { + logger.warn( + `입력 인사이트 조회 실패: ${error instanceof Error ? error.message : String(error)}` + ) + return { + days: safeDays, + totals, + daily: [], + topHours: [], + topApps: [], + phraseCount: 0, + sampleCount: 0, + averages: summarizeActivity(totals, safeDays), + hourly: Array.from({ length: 24 }, (_, hour) => ({ + hour, + keystrokes: 0, + clicks: 0, + chars: 0, + backspaces: 0, + activeMs: 0 + })), + activeDays: 0, + longestStreakDays: 0, + peakDay: null, + suggestions: { total: 0, accepted: 0, acceptRate: 0, avgLatencyMs: null }, + friction: calculateFrictionInsight(0, 0), + flowWindows: [], + suggestionApps: [] + } + } + } + + /** + * 과거에 이 접두 꼬리 뒤에 실제로 이어 쓴 텍스트를 찾는다 (개인 기억). + * + * 자주 쓰는 문구 목록보다 강한 개인화 신호다: 사용자가 그 문맥에서 실제로 + * 어떤 문장을 이어 썼는지가 그대로 들어간다. 임베딩 검색이 아니라 로컬 + * 문자열 관계 조회이므로 추가 의존성이나 네트워크가 없다. + * + * 학습 동의(inputLearnTypedText)가 없으면 저장된 표본이 없어 자연히 빈 배열이다. + */ + findContinuationHints(prefixTail: string, limit = 3): string[] { + const tail = prefixTail.trim() + if (tail.length < 4) return [] + + try { + const db = getDatabase() + const pattern = `%${tail.replace(/[%_\\]/gu, '')}%` + const rows = db + .select({ text: typingSamples.text }) + .from(typingSamples) + .where(sql`${typingSamples.text} LIKE ${pattern}`) + .orderBy(desc(typingSamples.createdAt)) + .limit(40) + .all() + + const out: string[] = [] + for (const row of rows) { + const index = row.text.indexOf(tail) + if (index < 0) continue + const continuation = row.text + .slice(index + tail.length) + .replace(/\s+/gu, ' ') + .trim() + // 바로 뒤가 비어 있으면(문장이 그대로 끝) 참고 가치가 없다 + if (continuation.length < 3) continue + const clipped = continuation.slice(0, 60) + if (out.some((existing) => existing === clipped)) continue + out.push(clipped) + if (out.length >= limit) break + } + return out + } catch (error) { + logger.warn( + `과거 이어쓰기 조회 실패: ${error instanceof Error ? error.message : String(error)}` + ) + return [] + } + } + + getPrivacyReceipt(): InputPrivacyReceipt { + try { + const db = getDatabase() + const countRows = (table: typeof inputActivity | typeof typingSamples | typeof personalPhrases | typeof suggestionsTable): number => + Number(db.select({ value: sql`count(*)` }).from(table).get()?.value ?? 0) + const counts = { + activityBuckets: countRows(inputActivity), + typingSamples: countRows(typingSamples), + personalPhrases: countRows(personalPhrases), + suggestions: countRows(suggestionsTable) + } + return { + localOnly: true, + rawKeyContentStored: false, + retention: { + activityDays: INPUT_TELEMETRY_DEFAULTS.retentionDays, + typingSamplesDays: INPUT_TELEMETRY_DEFAULTS.retentionDays, + suggestionDays: INPUT_TELEMETRY_DEFAULTS.retentionDays, + personalPhrases: 'until-deleted' + }, + counts + } + } catch (error) { + logger.warn(`입력 개인정보 영수증 조회 실패: ${error instanceof Error ? error.message : String(error)}`) + throw error + } + } + + listPhrases(limit = 100): PersonalPhrase[] { + try { + const db = getDatabase() + const rows = db + .select() + .from(personalPhrases) + .orderBy(desc(personalPhrases.count), desc(personalPhrases.lastUsedAt)) + .limit(Math.max(1, Math.min(limit, 500))) + .all() + return rows.map((row) => ({ + id: row.id, + phrase: row.phrase, + count: row.count, + source: row.source, + appName: row.appName, + lastUsedAt: row.lastUsedAt, + createdAt: row.createdAt + })) + } catch (error) { + logger.warn(`개인 문구 조회 실패: ${error instanceof Error ? error.message : String(error)}`) + return [] + } + } + + deletePhrase(id: string): boolean { + try { + const db = getDatabase() + const result = db.delete(personalPhrases).where(eq(personalPhrases.id, id)).run() + return Number(result.changes ?? 0) > 0 + } catch (error) { + logger.warn(`개인 문구 제 실패: ${error instanceof Error ? error.message : String(error)}`) + return false + } + } + + /** 수집된 모든 입력 데이터 삭제 (동의 철회/초기화). */ + clearAll(): void { + try { + const db = getDatabase() + db.delete(inputActivity).run() + db.delete(typingSamples).run() + db.delete(personalPhrases).run() + db.delete(suggestionsTable).run() + getPersonalGraphService().clearAll() + this._pending.clear() + this._appReadability.clear() + this._sessionTotals = emptyActivityBucket() + this._lastSnapshot = null + this._lastSnapshotKey = '' + logger.info('입력 텔레메트리 데이터 전체 삭제') + } catch (error) { + logger.warn(`입력 데이터 삭제 실패: ${error instanceof Error ? error.message : String(error)}`) + throw error + } + } + + /** 보존 기간 초과 데이터 정리 (ActivityWatch 와 동일한 로컬 보존 정책). */ + private _pruneOldData(now = Date.now()): void { + if (now - this._prunedAt < RETENTION_PRUNE_INTERVAL_MS) return + this._prunedAt = now + + // 그래프 유지보수도 같은 주기에 돌린다 (용어 공유 엣지 백필). + getPersonalGraphService().runMaintenance() + + const cutoff = now - INPUT_TELEMETRY_DEFAULTS.retentionDays * 24 * 60 * 60 * 1000 + try { + const db = getDatabase() + db.delete(typingSamples).where(sql`${typingSamples.createdAt} < ${cutoff}`).run() + db.delete(inputActivity).where(sql`${inputActivity.updatedAt} < ${cutoff}`).run() + db.delete(suggestionsTable).where(sql`${suggestionsTable.createdAt} < ${cutoff}`).run() + logger.info(`입력 데이터 보존 정리 완료 (${INPUT_TELEMETRY_DEFAULTS.retentionDays}일 초과)`) + } catch (error) { + logger.warn(`보존 정리 실패: ${error instanceof Error ? error.message : String(error)}`) + } + } + + /** 일시정지 토글 (동의는 유지). */ + setPaused(paused: boolean): void { + configSet('inputTelemetryPaused', paused) + this.applyConfig() + } + + /** 수집 동의 설정. 끄면 즉시 중지하고 수집분도 지운다. */ + setEnabled(enabled: boolean): void { + configSet('inputTelemetryEnabled', enabled) + if (!enabled) { + this.clearAll() + configSet('inputLearnTypedText', false) + } + this.applyConfig() + } + + private _emitState(): void { + this.emit('state-changed', this.getState()) + } +} + +/** + * 마지막 비어 있지 않은 줄을 돌려준다 (터미널/문서형 컨트롤용). + * + * 터미널 버퍼 전체를 프롬프트로 넘기면 모델이 과거 출력을 이어 쓰려 한다. + */ +function lastLineOf(text: string): string { + const lines = text.split(/\r?\n/u) + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i].trim() + if (line.length > 0) return line.slice(-SUGGESTION_CONTEXT_MAX_CHARS) + } + return text.slice(-SUGGESTION_CONTEXT_MAX_CHARS) +} + +function formatLocalDate(date: Date): string { + const year = date.getFullYear() + const month = String(date.getMonth() + 1).padStart(2, '0') + const day = String(date.getDate()).padStart(2, '0') + return `${year}-${month}-${day}` +} + +/** 제어문자 제거 + 공백 정규화. 2자 미만은 버린다. */ +function sanitizeSample(raw: string): string { + let stripped = '' + for (const char of raw) { + const code = char.codePointAt(0) ?? 0 + // 제어문자(개행/탭 포함)는 공백으로 바다 — 학습 표본에 제어문자를 남기지 않는다. + stripped += code < 0x20 || code === 0x7f ? ' ' : char + } + const cleaned = stripped.replace(/\s+/gu, ' ').trim() + if (cleaned.length < 2) return '' + return cleaned.slice(0, MAX_SAMPLE_CHARS) +} + +function getDisplayScaleFactor(): number { + try { + return screen.getPrimaryDisplay().scaleFactor || 1 + } catch { + return 1 + } +} + +let instance: InputTelemetryService | null = null + +export function getInputTelemetryService(): InputTelemetryService { + if (!instance) instance = new InputTelemetryService() + return instance +} + +export function resetInputTelemetryServiceForTests(): void { + instance?.dispose() + instance = null +} + +export { formatLocalDate, sanitizeSample } +export type { InputTelemetryState } diff --git a/apps/desktop/src/main/services/KeyBindingService.ts b/apps/desktop/src/main/services/KeyBindingService.ts index 727ab50..ae78fb2 100644 --- a/apps/desktop/src/main/services/KeyBindingService.ts +++ b/apps/desktop/src/main/services/KeyBindingService.ts @@ -14,6 +14,7 @@ import { globalShortcut } from 'electron' import { uIOhook, UiohookKey } from 'uiohook-napi' import type { UiohookKeyboardEvent, UiohookMouseEvent } from 'uiohook-napi' import { getLogger } from './LoggerService' +import { acquireGlobalInputHook } from './global-input-hook' import { configGet } from './ConfigService' import { D3ROError, ErrorCode } from '@d3ro/core/errors' import { TIMING } from '@d3ro/core/constants' @@ -33,6 +34,37 @@ import type { const logger = getLogger('KeyBindingService') +/** + * AltGr(=Ctrl+Alt) 함정 가드. + * + * 오른쪽 Alt 는 Windows 가 Ctrl+Alt 로 보내므로, Ctrl+Alt+화살표 같은 조합을 누르면 + * Alt 를 누르는 순간 Ctrl+AltRight 바인딩이 먼저 발동한다(실측 신고: 제안 탐색 대신 + * 음성 파이프라인이 돌았다). 수정자만으로 끝나면서 Ctrl+Alt 가 함께 눌린 트리거는 + * 잠깐 보류하고, 그 사이에 다른 키가 눌리면(=조합을 만들려던 것) 취소한다. + * + * 받아쓰기(수정자 단독, Ctrl 없음)와 Alt+Shift 계열은 모양이 달라 영향받지 않는다. + */ +const ALTGR_CHORD_GRACE_MS = 280 + +/** 수정자 키의 VK 코드 (수정자만으로 끝나는 조합 판별). */ +const MODIFIER_VK_CODES: ReadonlySet = new Set([ + 0x10, // Shift + 0x11, // Ctrl + 0x12, // Alt + 0x5b, // LWin + 0x5c, // RWin + 0xa0, + 0xa1, // L/R Shift + 0xa2, + 0xa3, // L/R Ctrl + 0xa4, + 0xa5 // L/R Alt +]) + +let pendingAltGrEmit: NodeJS.Timeout | null = null +/** 보류가 취소된 바인딩 — 대응하는 released 를 내보내지 않기 위해 기억한다. */ +const cancelledAltGrBindings = new Set() + // ============================================================ // 이벤트 페이로드 // ============================================================ @@ -47,6 +79,21 @@ export interface KeyBindingTriggerPayload { timestamp: number /** type === 'released' 일 때의 누름 유지 시간 (pressed 는 0) */ durationMs: number + /** + * 매칭된 바인딩의 주 키 코드와 수정자 상태. + * + * 소비자가 "수정자만으로 끝나는 조합인지" 를 판단해 AltGr 함정을 피하는 데 쓴다. + * (오른쪽 Alt 는 Windows 가 Ctrl+Alt 로 보내므로, Ctrl+Alt+화살표를 누르면 + * Alt 를 누르는 순간 Ctrl+AltRight 바인딩이 먼저 발동한다) + */ + binding: { + device: 'keyboard' | 'mouse' + code: number + ctrl: boolean + alt: boolean + shift: boolean + meta: boolean + } } interface KeyBindingServiceEvents { @@ -200,6 +247,17 @@ const UIOHOOK_TO_VK: ReadonlyMap = new Map( Array.from(VK_TO_UIOHOOK, ([vk, uiohookCode]) => [uiohookCode, vk] as const) ) +/** + * uiohook 이트 키코드 → Windows VK (정본 좌표계). + * + * 같은 변환표를 두 곳에 두지 않기 위해 내보낸다. 입력 텔레메트리가 키 **종류** + * (문자/숫자/백스페이스/단축키 …)를 분류할 때 쓴다 — 키 내용은 저장하지 않는다. + * 매핑이 없는 키는 null. + */ +export function uiohookCodeToVk(code: number): number | null { + return UIOHOOK_TO_VK.get(code) ?? null +} + function isCtrlCode(code: number): boolean { return code === UiohookKey.Ctrl || code === UiohookKey.CtrlRight } @@ -322,12 +380,21 @@ interface ActiveTrigger { actionId: KeyBindingActionId isDoublePress: boolean holdMode: boolean + /** 눌림을 만든 바인딩 (AltGr 형태 판정에 필요) */ + entry: RegisteredBinding } // ============================================================ // KeyBindingService // ============================================================ +/** AltGr(=Ctrl+Alt) 형태의 수정자 전용 조합인가. */ +function isAltGrShaped(binding: KeyBinding): boolean { + if (binding.device !== 'keyboard') return false + if (!binding.ctrl || !binding.alt) return false + return MODIFIER_VK_CODES.has(binding.code) +} + class KeyBindingService extends EventEmitter { private _isRunning = false @@ -354,6 +421,14 @@ class KeyBindingService extends EventEmitter { private _onMouseDown: ((e: UiohookMouseEvent) => void) | null = null private _onMouseUp: ((e: UiohookMouseEvent) => void) | null = null + /** + * 글로벌 후 해제 함수. + * + * 후킹은 프로세스 전역 글턴이고 InputTelemetryService 도 같은 후킹을 쓴다. + * 직접 stop 하면 상대방 수신을 죽이므로 참조 카운트 해제자를 보관한다. + */ + private _releaseHook: (() => void) | null = null + get isRunning(): boolean { return this._isRunning } @@ -386,11 +461,11 @@ class KeyBindingService extends EventEmitter { uIOhook.on('keyup', this._onKeyUp) uIOhook.on('mousedown', this._onMouseDown) uIOhook.on('mouseup', this._onMouseUp) - uIOhook.start() + this._releaseHook = acquireGlobalInputHook() this._isRunning = true this._syncAccelerators() - logger.info('uiohook started, global keyboard/mouse hook active') + logger.info('Key binding listeners attached to global input hook') } catch (error) { const d3roError = new D3ROError( ErrorCode.HotkeyHookInitFailed, @@ -406,7 +481,8 @@ class KeyBindingService extends EventEmitter { if (!this._isRunning) return try { - uIOhook.stop() + this._releaseHook?.() + this._releaseHook = null if (this._onKeyDown) { uIOhook.removeListener('keydown', this._onKeyDown) @@ -684,24 +760,74 @@ class KeyBindingService extends EventEmitter { triggers.push({ actionId: entry.actionId, isDoublePress, - holdMode: entry.spec.holdMode + holdMode: entry.spec.holdMode, + entry }) } this._activeTriggers.set(key, triggers) for (const trigger of triggers) { logger.debug(`Key binding pressed: "${trigger.actionId}" (double=${isDoublePress})`) - this.emit('triggered', { - actionId: trigger.actionId, - type: 'pressed', - isDoublePress: trigger.isDoublePress, - holdMode: trigger.holdMode, - timestamp: now, - durationMs: 0 - }) + this._emitPressed(trigger.entry, trigger, now, key) } } + + /** + * 눌림 이벤트를 내보낸다. + * + * AltGr 형태(수정자 전용 + Ctrl+Alt)는 곧 다른 키가 이어질 수 있으므로 잠깐 + * 보류한다. 그 사이 다른 바인딩의 눌림이 오면 취소한다 — 오른쪽 Alt 는 Windows 가 + * Ctrl+Alt 로 보내므로, Ctrl+Alt+화살표를 누르면 Alt 순간에 Ctrl+AltRight(음성 명령) + * 가 먼저 발동하던 문제를 이 지점에서 막는다. + */ + private _emitPressed( + entry: RegisteredBinding, + trigger: ActiveTrigger, + now: number, + bindingKeyValue: string + ): void { + const payload: KeyBindingTriggerPayload = { + actionId: trigger.actionId, + type: 'pressed', + isDoublePress: trigger.isDoublePress, + holdMode: trigger.holdMode, + timestamp: now, + durationMs: 0, + binding: { + device: entry.binding.device, + code: entry.binding.code, + ctrl: entry.binding.ctrl, + alt: entry.binding.alt, + shift: entry.binding.shift, + meta: entry.binding.meta + } + } + + if (isAltGrShaped(entry.binding)) { + if (pendingAltGrEmit) clearTimeout(pendingAltGrEmit) + const actionId = trigger.actionId + // 취소 표시는 _fireRelease 가 쓰는 것과 같은 키여야 한다 (device:code 조합이 아니다). + cancelledAltGrBindings.add(actionId + ':' + bindingKeyValue) + pendingAltGrEmit = setTimeout(() => { + pendingAltGrEmit = null + logger.debug(`AltGr 형태 트리거 유예 후 발동: "${actionId}"`) + this.emit('triggered', payload) + }, ALTGR_CHORD_GRACE_MS) + pendingAltGrEmit.unref?.() + return + } + + // 다른 키가 이어졌다 = 조합을 만들려던 것 → 보류 중인 AltGr 트리거는 취소. + if (pendingAltGrEmit) { + clearTimeout(pendingAltGrEmit) + pendingAltGrEmit = null + logger.debug('AltGr 형태 보류 트리거 취소 (다른 키가 이어짐)') + } + + this.emit('triggered', payload) + } + private _fireRelease(key: string): void { this._isKeyDown.set(key, false) @@ -717,13 +843,26 @@ class KeyBindingService extends EventEmitter { logger.debug( `Key binding released: "${trigger.actionId}" (duration=${durationMs}ms)` ) + if (cancelledAltGrBindings.delete(trigger.actionId + ':' + key)) { + logger.debug(`AltGr 보류가 취소된 바인딩의 released 는 내보내지 않는다: "${trigger.actionId}"`) + continue + } + this.emit('triggered', { actionId: trigger.actionId, type: 'released', isDoublePress: trigger.isDoublePress, holdMode: trigger.holdMode, timestamp: now, - durationMs + durationMs, + binding: { + device: 'keyboard', + code: 0, + ctrl: false, + alt: false, + shift: false, + meta: false + } }) } } diff --git a/apps/desktop/src/main/services/LocalLLMService.ts b/apps/desktop/src/main/services/LocalLLMService.ts index 64bf06c..fe683c0 100644 --- a/apps/desktop/src/main/services/LocalLLMService.ts +++ b/apps/desktop/src/main/services/LocalLLMService.ts @@ -38,6 +38,22 @@ interface GenerateOptions { maxTokens?: number systemPrompt?: string stream?: boolean + /** + * 호출자 소유의 취소 시그널. + * + * 호출자별 취소를 위해 사용한다. `cancelGeneration()`은 모든 활성 요청을 + * 취소하므로 인라인 제안처럼 자기 요청만 중단해야 하는 쪽은 이 시그널을 쓴다. + */ + signal?: AbortSignal + /** 요청별 생성 제한 시간(ms). */ + timeoutMs?: number + /** + * Ollama keep_alive 값 (예: '30m'). + * + * 제안처럼 반복 호출되는 경로에서 모델이 메모리에서 내려가면 요청마다 + * 콜드 로딩(실측 20초+)을 다시 문다. + */ + keepAlive?: string } interface GenerateResult { @@ -57,6 +73,23 @@ interface OllamaGenerateResponse { eval_count?: number } +interface ChatStreamOptions { + model?: string + temperature?: number + maxTokens?: number + signal?: AbortSignal + timeoutMs?: number + keepAlive?: string +} + +type AbortCause = 'timeout' | 'cancelled' + +interface ActiveRequest { + controller: AbortController + abortCause: AbortCause | null + close: () => void +} + interface OllamaTagsResponse { models: Array<{ name: string @@ -110,7 +143,8 @@ class LocalLLMService extends EventEmitter { private _pollInterval: ReturnType | null = null private _available = false private _serverVersion: string | null = null - private _abortController: AbortController | null = null + private _activeRequests = new Set() + private _ensureRunningPromise: Promise<'running' | 'starting' | 'not-installed' | 'failed'> | null = null private _disposed = false get state(): LLMState { @@ -144,6 +178,22 @@ class LocalLLMService extends EventEmitter { * - 'failed': 스폰 시도 실패 */ async ensureRunning(): Promise<'running' | 'starting' | 'not-installed' | 'failed'> { + if (this._ensureRunningPromise) { + return this._ensureRunningPromise + } + + const task = this._ensureRunning() + this._ensureRunningPromise = task + try { + return await task + } finally { + if (this._ensureRunningPromise === task) { + this._ensureRunningPromise = null + } + } + } + + private async _ensureRunning(): Promise<'running' | 'starting' | 'not-installed' | 'failed'> { if (await this._ping(1500)) { logger.info('Ollama server already running') return 'running' @@ -307,6 +357,7 @@ class LocalLLMService extends EventEmitter { * Ollama 가용성 폴링을 시작한다 (5초 간격). */ startPolling(): void { + if (this._pollInterval) return this._checkAvailability() this._pollInterval = setInterval(() => { if (this._state !== LLMState.Generating) { @@ -333,8 +384,8 @@ class LocalLLMService extends EventEmitter { const serverUrl = getOllamaServerUrl() const model = options?.model ?? configGet('llmModelId') ?? 'gemma4:e4b' - - this._state = LLMState.Generating + const maxTokens = this._resolveMaxTokens(options?.maxTokens, 2048) + const request = this._beginRequest(options?.signal, options?.timeoutMs, 120000) try { const response = await fetch(`${serverUrl}/api/generate`, { @@ -345,15 +396,16 @@ class LocalLLMService extends EventEmitter { prompt, system: options?.systemPrompt, stream: false, + keep_alive: options?.keepAlive, // Ollama v0.20+ think 파라미터: reasoning 모델에서 thinking 토큰 생성 중단. // gemma4/llama3.2 등 non-reasoning 모델에서는 무시됨. think: false, options: { temperature: options?.temperature ?? 0.3, - num_predict: options?.maxTokens ?? 2048 + num_predict: maxTokens } }), - signal: AbortSignal.timeout(120000) + signal: request.controller.signal }) if (!response.ok) { @@ -373,22 +425,18 @@ class LocalLLMService extends EventEmitter { totalDuration: data.total_duration ? data.total_duration / 1e6 : 0 } - this._state = LLMState.Available this.emit('complete', { result }) return result } catch (error) { - this._state = LLMState.Available - if (error instanceof D3ROError) throw error - throw new D3ROError( - ErrorCode.LLMProcessingFailed, - `LLM generation failed: ${error instanceof Error ? error.message : String(error)}` - ) + throw this._toRequestError(error, request, 'LLM generation failed') + } finally { + request.close() } } /** * 스트리밍 텍스트 생성. NDJSON 파싱. - * 반환된 AbortController로 취소 가능. + * 호출자별 AbortSignal 또는 전역 cancelGeneration()으로 취소 가능. */ async *streamGenerate( prompt: string, @@ -400,9 +448,10 @@ class LocalLLMService extends EventEmitter { const serverUrl = getOllamaServerUrl() const model = options?.model ?? configGet('llmModelId') ?? 'gemma4:e4b' - - this._state = LLMState.Generating - this._abortController = new AbortController() + const maxTokens = this._resolveMaxTokens(options?.maxTokens, 2048) + const request = this._beginRequest(options?.signal, options?.timeoutMs, 120000) + let reader: ReadableStreamDefaultReader | null = null + let doneFrame = false try { const response = await fetch(`${serverUrl}/api/generate`, { @@ -413,13 +462,14 @@ class LocalLLMService extends EventEmitter { prompt, system: options?.systemPrompt, stream: true, + keep_alive: options?.keepAlive, think: false, options: { temperature: options?.temperature ?? 0.3, - num_predict: options?.maxTokens ?? 2048 + num_predict: maxTokens } }), - signal: this._abortController.signal + signal: request.controller.signal }) if (!response.ok || !response.body) { @@ -429,11 +479,22 @@ class LocalLLMService extends EventEmitter { ) } - const reader = response.body.getReader() + reader = response.body.getReader() const decoder = new TextDecoder() let buffer = '' let fullText = '' let lastChunk: OllamaGenerateResponse | null = null + const complete = (): GenerateResult => { + const result: GenerateResult = { + text: fullText, + model: lastChunk?.model ?? model, + promptTokens: lastChunk?.prompt_eval_count ?? 0, + completionTokens: lastChunk?.eval_count ?? 0, + totalDuration: lastChunk?.total_duration ? lastChunk.total_duration / 1e6 : 0 + } + this.emit('complete', { result }) + return result + } while (true) { const { done, value } = await reader.read() @@ -448,45 +509,140 @@ class LocalLLMService extends EventEmitter { try { const chunk = JSON.parse(line) as OllamaGenerateResponse fullText += chunk.response - this.emit('token', { token: chunk.response, done: chunk.done }) - yield chunk.response - if (chunk.done) { lastChunk = chunk + doneFrame = true + this.emit('token', { token: chunk.response, done: true }) + if (chunk.response) yield chunk.response + return complete() } + this.emit('token', { token: chunk.response, done: chunk.done }) + yield chunk.response } catch { - logger.warn(`Failed to parse NDJSON line: ${line.substring(0, 100)}`) + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama returned malformed NDJSON') } } } - this._state = LLMState.Available - this._abortController = null - - const result: GenerateResult = { - text: fullText, - model: lastChunk?.model ?? model, - promptTokens: lastChunk?.prompt_eval_count ?? 0, - completionTokens: lastChunk?.eval_count ?? 0, - totalDuration: lastChunk?.total_duration ? lastChunk.total_duration / 1e6 : 0 + buffer += decoder.decode() + const trailing = buffer.trim() + if (trailing) { + try { + const chunk = JSON.parse(trailing) as OllamaGenerateResponse + fullText += chunk.response + if (chunk.done) { + lastChunk = chunk + doneFrame = true + this.emit('token', { token: chunk.response, done: true }) + if (chunk.response) yield chunk.response + return complete() + } + this.emit('token', { token: chunk.response, done: chunk.done }) + yield chunk.response + } catch { + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama returned malformed NDJSON') + } } - this.emit('complete', { result }) - return result + if (!doneFrame) { + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama stream ended before completion') + } + + return complete() } catch (error) { - this._state = LLMState.Available - this._abortController = null - if (error instanceof D3ROError) throw error - if (error instanceof DOMException && error.name === 'AbortError') { - throw new D3ROError(ErrorCode.LLMProcessingCancelled, 'LLM generation cancelled') + throw this._toRequestError(error, request, 'LLM streaming failed') + } finally { + if (reader) { + try { + await reader.cancel() + } catch (error) { + logger.warn(`LLM stream reader cancellation failed: ${error instanceof Error ? error.message : String(error)}`) + } } - throw new D3ROError( - ErrorCode.LLMProcessingFailed, - `LLM streaming failed: ${error instanceof Error ? error.message : String(error)}` - ) + if (!doneFrame) { + request.abortCause ??= 'cancelled' + request.controller.abort() + } + request.close() } } + /** 요청별 취소·기한을 활성 요청 집합에 등록한다. */ + private _beginRequest( + externalSignal: AbortSignal | undefined, + timeoutMs: number | undefined, + defaultTimeoutMs: number + ): ActiveRequest { + const controller = new AbortController() + const resolvedTimeoutMs = this._resolveTimeoutMs(timeoutMs, defaultTimeoutMs) + const abort = (cause: AbortCause): void => { + if (request.abortCause) return + request.abortCause = cause + controller.abort() + } + const onExternalAbort = (): void => abort('cancelled') + const timeout = setTimeout(() => abort('timeout'), resolvedTimeoutMs) + const request: ActiveRequest = { + controller, + abortCause: null, + close: () => { + clearTimeout(timeout) + externalSignal?.removeEventListener('abort', onExternalAbort) + this._activeRequests.delete(request) + this._refreshState() + } + } + this._activeRequests.add(request) + this._refreshState() + + if (externalSignal?.aborted) { + onExternalAbort() + } else { + externalSignal?.addEventListener('abort', onExternalAbort, { once: true }) + } + return request + } + + private _resolveMaxTokens(maxTokens: number | undefined, fallback: number): number { + if (maxTokens === undefined) return fallback + if (!Number.isSafeInteger(maxTokens) || maxTokens <= 0) { + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'maxTokens must be a positive safe integer') + } + return Math.min(maxTokens, 4096) + } + + private _resolveTimeoutMs(timeoutMs: number | undefined, fallback: number): number { + if (timeoutMs === undefined) return fallback + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'timeoutMs must be a positive safe integer') + } + return timeoutMs + } + + private _toRequestError(error: unknown, request: ActiveRequest, prefix: string): D3ROError { + if (request.abortCause === 'timeout') { + return new D3ROError(ErrorCode.LLMProcessingTimeout, 'LLM generation timed out') + } + if (request.abortCause === 'cancelled' || request.controller.signal.aborted) { + return new D3ROError(ErrorCode.LLMProcessingCancelled, 'LLM generation cancelled') + } + if (error instanceof D3ROError) return error + return new D3ROError( + ErrorCode.LLMProcessingFailed, + `${prefix}: ${error instanceof Error ? error.message : String(error)}` + ) + } + + private _refreshState(): void { + this._state = this._disposed + ? LLMState.Unavailable + : this._activeRequests.size > 0 + ? LLMState.Generating + : this._available + ? LLMState.Available + : LLMState.Unavailable + } + /** * 텍스트를 LLM 액션에 따라 처리한다. */ @@ -531,9 +687,11 @@ class LocalLLMService extends EventEmitter { } cancelGeneration(): void { - if (this._abortController) { - this._abortController.abort() - this._abortController = null + if (this._activeRequests.size > 0) { + for (const request of this._activeRequests) { + if (!request.abortCause) request.abortCause = 'cancelled' + request.controller.abort() + } logger.info('LLM generation cancelled') } } @@ -679,7 +837,7 @@ class LocalLLMService extends EventEmitter { */ async *chatStream( messages: Array<{ role: string; content: string }>, - options?: { model?: string; temperature?: number }, + options?: ChatStreamOptions, ): AsyncGenerator { if (!this._available) { throw new D3ROError(ErrorCode.LLMServerUnreachable, 'Ollama server not available') @@ -687,9 +845,10 @@ class LocalLLMService extends EventEmitter { const serverUrl = getOllamaServerUrl() const model = options?.model ?? configGet('llmModelId') ?? 'gemma4:e4b' - - this._abortController = new AbortController() - this._state = LLMState.Generating + const maxTokens = this._resolveMaxTokens(options?.maxTokens, 512) + const request = this._beginRequest(options?.signal, options?.timeoutMs, 60000) + let reader: ReadableStreamDefaultReader | null = null + let doneFrame = false try { const response = await fetch(`${serverUrl}/api/chat`, { @@ -699,19 +858,21 @@ class LocalLLMService extends EventEmitter { model, messages, stream: true, + keep_alive: options?.keepAlive, think: false, options: { temperature: options?.temperature ?? 0.7, + num_predict: maxTokens, }, }), - signal: this._abortController.signal, + signal: request.controller.signal, }) if (!response.ok || !response.body) { throw new D3ROError(ErrorCode.LLMProcessingFailed, `Chat API error: ${response.status}`) } - const reader = response.body.getReader() + reader = response.body.getReader() const decoder = new TextDecoder() let buffer = '' let accumulated = '' @@ -728,6 +889,9 @@ class LocalLLMService extends EventEmitter { if (!line.trim()) continue try { const chunk = JSON.parse(line) as { message?: { content: string }; done: boolean } + if (chunk.done) { + doneFrame = true + } if (chunk.message?.content) { accumulated += chunk.message.content yield chunk.message.content @@ -736,15 +900,47 @@ class LocalLLMService extends EventEmitter { return accumulated } } catch { - // 불완전 JSON 무시 + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama returned malformed NDJSON') } } } - return accumulated + buffer += decoder.decode() + const trailing = buffer.trim() + if (trailing) { + try { + const chunk = JSON.parse(trailing) as { message?: { content: string }; done: boolean } + if (chunk.done) { + doneFrame = true + } + if (chunk.message?.content) { + accumulated += chunk.message.content + yield chunk.message.content + } + if (chunk.done) { + return accumulated + } + } catch { + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama returned malformed NDJSON') + } + } + + throw new D3ROError(ErrorCode.LLMProcessingFailed, 'Ollama chat stream ended before completion') + } catch (error) { + throw this._toRequestError(error, request, 'LLM chat streaming failed') } finally { - this._state = this._available ? LLMState.Available : LLMState.Unavailable - this._abortController = null + if (reader) { + try { + await reader.cancel() + } catch (error) { + logger.warn(`LLM chat reader cancellation failed: ${error instanceof Error ? error.message : String(error)}`) + } + } + if (!doneFrame) { + request.abortCause ??= 'cancelled' + request.controller.abort() + } + request.close() } } @@ -752,6 +948,7 @@ class LocalLLMService extends EventEmitter { this._disposed = true this.stopPolling() this.cancelGeneration() + this._refreshState() this.removeAllListeners() logger.info('LocalLLMService disposed') } @@ -800,11 +997,11 @@ class LocalLLMService extends EventEmitter { if (detectedVersion) this._serverVersion = detectedVersion if (!wasAvailable && this._available) { - this._state = LLMState.Available + this._refreshState() this.emit('availability-changed', { available: true }) logger.info(`Ollama server connected (version: ${this._serverVersion ?? 'active'})`) } else if (wasAvailable && !this._available) { - this._state = LLMState.Unavailable + this._refreshState() this._serverVersion = null this.emit('availability-changed', { available: false }) logger.warn('Ollama server disconnected') @@ -848,6 +1045,10 @@ export async function startLocalLLMAvailability(): Promise { } export function resetLocalLLMServiceForTests(): void { - if (instance) instance.removeAllListeners() + instance?.dispose() instance = null } + +export function disposeLocalLLMService(): void { + instance?.dispose() +} diff --git a/apps/desktop/src/main/services/LocalSTTService.ts b/apps/desktop/src/main/services/LocalSTTService.ts index 3f72185..6de371c 100644 --- a/apps/desktop/src/main/services/LocalSTTService.ts +++ b/apps/desktop/src/main/services/LocalSTTService.ts @@ -354,6 +354,17 @@ class LocalSTTService extends EventEmitter { } } + /** + * 사이드카 프로세스만 보장하고 base URL 을 돌려준다 (모델 로딩 없음). + * + * 입력 인텔리전스(제안/학습)는 UIA 브리지가 사이드카에 있으므로 STT 모델 없이도 + * 사이드카 HTTP 서버가 필요하다. 실패하면 예외를 그대로 올린다. + */ + async ensureSidecar(): Promise { + await this._ensureSidecarRunning() + return this._baseUrl + } + /** * 녹음 중 실시간 미리보기 전사. * 최종 결과와 분리되어 삽입되지 않으며, 실패해도 빈 문자열을 반환한다. diff --git a/apps/desktop/src/main/services/PersonalGraphService.ts b/apps/desktop/src/main/services/PersonalGraphService.ts new file mode 100644 index 0000000..0298479 --- /dev/null +++ b/apps/desktop/src/main/services/PersonalGraphService.ts @@ -0,0 +1,392 @@ +// src/main/services/PersonalGraphService.ts +// +// 개인 그래프 — 사용자의 문장을 노드로, 관계를 엣지로 저장하고 제안 문맥을 끌어온다. +// +// 왜: n-gram 문자열 조회는 "같은 꼬리 뒤" 만 찾는다. 실제 글은 관계로 이어진다 — +// 어떤 문장은 늘 다른 문장 뒤에 오고(follows), 어떤 문장들은 용어를 공유한다 +// (shares_terms). 그 관계를 저장하면 접두가 조금 어긋나도 관련 문맥을 쓸 수 있다. +// +// 전부 로컬 SQLite 다 (임베딩/네트워크 없음). 학습 동의(inputLearnTypedText)가 없으면 +// 표본이 저장되지 않으므로 그래프도 비어 있다. + +import { + SHARES_TERMS_MIN_SIMILARITY, + buildSequenceEdges, + continuationsFrom, + extractTerms, + jaccard, + rankRelated, + splitSentences, + type GraphContext, + type PersonalGraphQuery, + type PersonalGraphStats, + type PhraseSource, + type RelatedCandidate +} from '@d3ro/core/personal-graph' +import { prefixTail } from '@d3ro/core/input-intelligence' +import { and, desc, eq, gte, inArray, sql } from 'drizzle-orm' +import { getDatabase } from '../db' +import { personalPhrases, phraseEdges } from '../db/schema' +import { getLogger } from './LoggerService' + +const logger = getLogger('PersonalGraphService') + +/** 계약 타입은 core 정본을 따른다 (렌더러/프리로드와 공유). */ +export type { PersonalGraphQuery, PersonalGraphStats } + + +/** 용어 공유 엣지 백필 시 한 번에 볼 노드 수. */ +const MAINTENANCE_NODE_LIMIT = 300 +/** 관계 조회에서 최근 노드를 훑는 수. */ +const TERM_SCAN_LIMIT = 400 + +class PersonalGraphService { + /** + * 텍스트를 그래프에 반영한다. + * + * 문장마다 노드를 upsert 하고, 연속한 문장 쌍을 follows 엣지로, 용어를 충분히 + * 공유하는 쌍을 shares_terms 엣지로 남긴다. + */ + indexText( + text: string, + meta: { source: PhraseSource; appName?: string | null; at?: number } + ): void { + const sentences = splitSentences(text) + if (sentences.length === 0) return + + const at = meta.at ?? Date.now() + try { + const db = getDatabase() + + for (const sentence of sentences) { + const terms = extractTerms(sentence) + db.insert(personalPhrases) + .values({ + id: crypto.randomUUID(), + phrase: sentence, + count: 1, + source: meta.source, + lastUsedAt: at, + createdAt: at, + terms: JSON.stringify(terms), + appName: meta.appName ?? null + }) + .onConflictDoUpdate({ + target: personalPhrases.phrase, + set: { + count: sql`${personalPhrases.count} + 1`, + lastUsedAt: at, + terms: JSON.stringify(terms), + appName: meta.appName ?? null + } + }) + .run() + } + + // follows 엣지 + for (const edge of buildSequenceEdges(sentences)) { + this._upsertEdge(edge.from, edge.to, 'follows') + } + + // 같은 텍스트 안에서 용어를 많이 공유하는 쌍 + for (let i = 0; i < sentences.length; i += 1) { + const termsA = extractTerms(sentences[i]) + for (let j = i + 1; j < sentences.length; j += 1) { + const termsB = extractTerms(sentences[j]) + if (jaccard(termsA, termsB) < SHARES_TERMS_MIN_SIMILARITY) continue + this._upsertEdge(sentences[i], sentences[j], 'shares_terms') + } + } + } catch (error) { + logger.warn(`그래프 반영 실패: ${error instanceof Error ? error.message : String(error)}`) + } + } + + private _upsertEdge(from: string, to: string, kind: 'follows' | 'shares_terms'): void { + const db = getDatabase() + const now = Date.now() + db.insert(phraseEdges) + .values({ + id: crypto.randomUUID(), + fromPhrase: from, + toPhrase: to, + kind, + weight: 1, + updatedAt: now + }) + .onConflictDoUpdate({ + target: [phraseEdges.fromPhrase, phraseEdges.toPhrase, phraseEdges.kind], + set: { + weight: sql`${phraseEdges.weight} + 1`, + updatedAt: now + } + }) + .run() + } + + /** + * 제안 문맥을 그래프에서 끌어온다. + * + * 1) 접두 꼬리를 포함하는 노드를 찾고 그 뒤에 실제로 이어 쓴 텍스트를 뽑는다. + * 2) 그 노드들의 follows 이웃과 용어를 공유하는 노드를 순위화해 관련 문장으로 준다. + */ + retrieveContext(prefix: string, limit = 4): GraphContext { + const tail = prefixTail(prefix) + if (tail.length < 4) return { continuations: [], related: [] } + + try { + const db = getDatabase() + const pattern = `%${tail.replace(/[%_\\]/gu, '')}%` + + const anchorRows = db + .select() + .from(personalPhrases) + .where(sql`${personalPhrases.phrase} LIKE ${pattern}`) + .orderBy(desc(personalPhrases.lastUsedAt)) + .limit(20) + .all() + + if (anchorRows.length === 0) return { continuations: [], related: [] } + + const anchorTexts = anchorRows.map((row) => row.phrase) + const continuations = continuationsFrom(anchorTexts, tail, 3) + + const anchorTerms = new Set() + for (const row of anchorRows) { + for (const term of parseTerms(row.terms)) anchorTerms.add(term) + } + + const candidates: RelatedCandidate[] = [] + const seen = new Set(anchorTexts) + + // (a) follows 이웃 — 가장 강한 관계 + const edgeRows = db + .select() + .from(phraseEdges) + .where( + and(inArray(phraseEdges.fromPhrase, anchorTexts), eq(phraseEdges.kind, 'follows')) + ) + .orderBy(desc(phraseEdges.weight)) + .limit(40) + .all() + + for (const edge of edgeRows) { + if (seen.has(edge.toPhrase)) continue + seen.add(edge.toPhrase) + candidates.push({ + text: edge.toPhrase, + terms: [], + weight: edge.weight * 2, + lastUsedAt: null + }) + } + + // (b) 용어 공유 — 최근 노드를 훑어 유사도가 높은 것 + const recentNodes = db + .select() + .from(personalPhrases) + .orderBy(desc(personalPhrases.lastUsedAt)) + .limit(TERM_SCAN_LIMIT) + .all() + + for (const node of recentNodes) { + if (seen.has(node.phrase)) continue + const terms = parseTerms(node.terms) + if (terms.length === 0) continue + const similarity = jaccard([...anchorTerms], terms) + if (similarity < SHARES_TERMS_MIN_SIMILARITY) continue + seen.add(node.phrase) + candidates.push({ + text: node.phrase, + terms, + weight: 0, + lastUsedAt: node.lastUsedAt + }) + } + + const related = rankRelated([...anchorTerms], candidates, limit) + return { continuations, related } + } catch (error) { + logger.warn( + `그래프 문맥 조회 실패: ${error instanceof Error ? error.message : String(error)}` + ) + return { continuations: [], related: [] } + } + } + + /** 용어 공유 엣지를 백필한다 (기존 노드들 사이 관계 보강). */ + runMaintenance(limit = MAINTENANCE_NODE_LIMIT): number { + try { + const db = getDatabase() + const nodes = db + .select() + .from(personalPhrases) + .orderBy(desc(personalPhrases.lastUsedAt)) + .limit(limit) + .all() + + let created = 0 + for (let i = 0; i < nodes.length; i += 1) { + const termsA = parseTerms(nodes[i].terms) + if (termsA.length === 0) continue + for (let j = i + 1; j < nodes.length; j += 1) { + const termsB = parseTerms(nodes[j].terms) + if (termsB.length === 0) continue + if (jaccard(termsA, termsB) < SHARES_TERMS_MIN_SIMILARITY) continue + this._upsertEdge(nodes[i].phrase, nodes[j].phrase, 'shares_terms') + created += 1 + if (created >= 200) return created + } + } + if (created > 0) logger.info(`그래프 유지보수: shares_terms 엣지 ${created}개 생성`) + return created + } catch (error) { + logger.warn(`그래프 유지보수 실패: ${error instanceof Error ? error.message : String(error)}`) + return 0 + } + } + + getStats(days = 30): PersonalGraphStats { + const empty: PersonalGraphStats = { + nodes: 0, + followsEdges: 0, + sharesTermsEdges: 0, + topEdges: [], + recentNodes: [] + } + + try { + const db = getDatabase() + const since = Date.now() - days * 86400000 + + const nodeCount = db + .select({ value: sql`count(*)` }) + .from(personalPhrases) + .get() + const follows = db + .select({ value: sql`count(*)` }) + .from(phraseEdges) + .where(eq(phraseEdges.kind, 'follows')) + .get() + const shares = db + .select({ value: sql`count(*)` }) + .from(phraseEdges) + .where(eq(phraseEdges.kind, 'shares_terms')) + .get() + + const topEdges = db + .select({ + from: phraseEdges.fromPhrase, + to: phraseEdges.toPhrase, + kind: phraseEdges.kind, + weight: phraseEdges.weight + }) + .from(phraseEdges) + .orderBy(desc(phraseEdges.weight)) + .limit(12) + .all() + + const recentNodes = db + .select() + .from(personalPhrases) + .where(gte(personalPhrases.createdAt, since)) + .orderBy(desc(personalPhrases.lastUsedAt)) + .limit(20) + .all() + + return { + nodes: Number(nodeCount?.value ?? 0), + followsEdges: Number(follows?.value ?? 0), + sharesTermsEdges: Number(shares?.value ?? 0), + topEdges, + recentNodes: recentNodes.map((row) => ({ + text: row.phrase, + terms: parseTerms(row.terms), + count: row.count, + appName: row.appName + })) + } + } catch (error) { + logger.warn(`그래프 통계 실패: ${error instanceof Error ? error.message : String(error)}`) + return empty + } + } + + /** 특정 텍스트 주변의 그래프를 조회한다 (설정/지식베이스 UI). */ + query(text: string, limit = 12): PersonalGraphQuery { + try { + const db = getDatabase() + const needle = text.trim() + if (needle.length < 2) return { anchors: [], neighbors: [] } + + const pattern = `%${needle.replace(/[%_\\]/gu, '')}%` + const anchors = db + .select() + .from(personalPhrases) + .where(sql`${personalPhrases.phrase} LIKE ${pattern}`) + .orderBy(desc(personalPhrases.lastUsedAt)) + .limit(6) + .all() + + if (anchors.length === 0) return { anchors: [], neighbors: [] } + + const anchorTexts = anchors.map((row) => row.phrase) + const neighbors = db + .select({ + text: phraseEdges.toPhrase, + kind: phraseEdges.kind, + weight: phraseEdges.weight + }) + .from(phraseEdges) + .where(inArray(phraseEdges.fromPhrase, anchorTexts)) + .orderBy(desc(phraseEdges.weight)) + .limit(limit) + .all() + + return { + anchors: anchors.map((row) => ({ + text: row.phrase, + terms: parseTerms(row.terms), + count: row.count + })), + neighbors + } + } catch (error) { + logger.warn(`그래프 조회 실패: ${error instanceof Error ? error.message : String(error)}`) + return { anchors: [], neighbors: [] } + } + } + + clearAll(): void { + try { + const db = getDatabase() + db.delete(phraseEdges).run() + logger.info('개인 그래프 엣지 전체 삭제') + } catch (error) { + logger.warn(`그래프 삭제 실패: ${error instanceof Error ? error.message : String(error)}`) + } + } +} + +/** DB 의 terms 컬럼(JSON)을 안전하게 파싱한다. */ +function parseTerms(raw: string | null): string[] { + if (!raw) return [] + try { + const parsed = JSON.parse(raw) as unknown + if (!Array.isArray(parsed)) return [] + return parsed.filter((value): value is string => typeof value === 'string') + } catch { + return [] + } +} + +let instance: PersonalGraphService | null = null + +export function getPersonalGraphService(): PersonalGraphService { + if (!instance) instance = new PersonalGraphService() + return instance +} + +export function resetPersonalGraphServiceForTests(): void { + instance = null +} \ No newline at end of file diff --git a/apps/desktop/src/main/services/ScreenContextService.ts b/apps/desktop/src/main/services/ScreenContextService.ts index 412ad01..5b05cb6 100644 --- a/apps/desktop/src/main/services/ScreenContextService.ts +++ b/apps/desktop/src/main/services/ScreenContextService.ts @@ -241,7 +241,7 @@ $title = $sb.ToString() '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', psScript, - ], { timeout: 3000 }) + ], { timeout: 3000, windowsHide: true }) const lines = stdout.trim().split('\n') const appName = lines[0]?.trim() || null diff --git a/apps/desktop/src/main/services/SuggestionService.ts b/apps/desktop/src/main/services/SuggestionService.ts new file mode 100644 index 0000000..da2d4cc --- /dev/null +++ b/apps/desktop/src/main/services/SuggestionService.ts @@ -0,0 +1,1199 @@ +// src/main/services/SuggestionService.ts +// +// 다음 문장 제안(ghost text) — 입력 맥락을 받아 gemma(Ollama)로 후보를 만들고, +// 수락 시 대상 앱에 삽입한다. +// +// 설계 근거 (2025~2026 인라인 컴플리션 실측): +// - 디바운스는 타이핑 정지 후 600ms(기본)로, 모델 호출 빈도를 보수적으로 제한한다. +// - 출력 토큰은 극단적으로 작게(기본 64, Tabby 64 / KeyType 4~16) — +// 인라인 제안은 스트리밍으로 빨리 보여주고 짧게 끊는 것이 정석이다. +// - 문맥이 stale 이 된 진행 중 생성만 호출자 시그널로 취소하고, 다음 입력 문맥에서 재평가한다. +// - 후보가 나온 접두와 현재 접두가 달라지면 그 즉시 오버레이를 지운다 (stale). + +import { EventEmitter } from 'events' +import { desc, eq } from 'drizzle-orm' +import { + SUGGESTION_CONTEXT_MAX_CHARS, + SUGGESTION_DEFAULTS, + SUGGESTION_MAX_OUTPUT_CHARS, + buildLocalSuggestionCandidates, + decideSuggestion, + decideSuggestionRefresh, + isAppExcluded, + parseSuggestionCandidates, + sanitizeSuggestionLine, + selectPhraseHints, + type SuggestionCandidate, + type SuggestionProvenance, + type SuggestionSkipReason, + type SuggestionState, + type UiRect +} from '@d3ro/core/input-intelligence' +import { getDatabase } from '../db' +import { suggestions } from '../db/schema' +import { configGet, configSet } from './ConfigService' +import { getLogger } from './LoggerService' +import { getLocalLLMService } from './LocalLLMService' +import { buildSuggestionPrompt } from './llm-prompts' +import { getInputTelemetryService, type TypingContext } from './InputTelemetryService' +import { getPersonalGraphService } from './PersonalGraphService' +import { getTextInsertService } from './TextInsertService' + +const logger = getLogger('SuggestionService') + +export interface SuggestionHistoryEntry { + id: string + appName: string | null + prefixText: string + suggestionText: string + model: string | null + latencyMs: number | null + accepted: boolean + createdAt: number +} + +export interface SuggestionRequestResult { + ok: boolean + reason?: SuggestionSkipReason +} + +interface SuggestionEvents { + updated: (state: SuggestionState) => void + cleared: (payload: { reason: SuggestionSkipReason }) => void + 'state-changed': (state: SuggestionState) => void +} + +interface LocalMemoryContext { + continuationHints: string[] + relatedHints: string[] + phraseHints: string[] + appPhraseCount: number +} + +/** 이 길이를 넘는 모델 출력은 버린다 (스트리밍 폭주 방지) */ +const MAX_RAW_OUTPUT_CHARS = 1200 + +/** + * 모델을 메모리에 유지하는 시간. + * + * 자동 입력 제안은 반복 호출 경로이므로, 짧은 유휴 구간만 모델을 유지한다. + */ +const SUGGESTION_KEEP_ALIVE = '2m' + +class SuggestionService extends EventEmitter { + private _candidates: SuggestionCandidate[] = [] + private _activeIndex = 0 + private _anchor: UiRect | null = null + private _appName: string | null = null + private _windowTitle: string | null = null + /** 후보를 만든 시점의 접두 — 접두가 달라지면 stale */ + private _generatedForPrefix = '' + private _lastSkipReason: SuggestionSkipReason | null = null + private _lastDecisionSignature = '' + /** 생성 중 — 후보 도착 전에도 오버레이를 띄운다 */ + private _generating = false + /** 모델 적재 중 — UI 가 "준비 중" 을 표시한다 */ + private _warmingUp = false + /** + * 생성 세대 토큰. + * + * 사용자가 제안을 닫거나 새로 시작하면 증가한다. 진행 중이던 요청이 나중에 + * 끝나도 토큰이 달라졌으면 결과를 버린다 — "X 로 닫았는데 잠시 뒤 다시 뜨던" + * 문제를 막는다. + */ + private _generationToken = 0 + /** 마지막 문맥 수신 시각 — 늦게 도착한 결과를 판단한다 */ + private _lastContextAt = 0 + /** 표시 수명 타이머 (TTL) */ + private _ttlTimer: NodeJS.Timeout | null = null + /** 마지막으로 실제 요청한 접두 (같은 텍스트 반복 요청 방지) */ + private _lastRequestedPrefix = '' + /** 연속 실패 횟수 — 임계 도달 시 쿨다운 */ + private _consecutiveFailures = 0 + /** 쿨다운 종료 시각 (ms) — 이 동안은 요청하지 않는다 */ + private _cooldownUntil = 0 + /** 사용자가 X 로 닫은 뒤 재표시를 막는 시각 (ms) */ + private _userDismissedUntil = 0 + /** 스트리밍 누적 텍스트 */ + private _partialText = '' + /** 마지막 스트리밍 방출 시각 (IPC 과다 방출 방지) */ + private _lastPartialEmitAt = 0 + private _lastContext: TypingContext | null = null + private _provenance: SuggestionProvenance | null = null + + private _lastRequestAt = 0 + private _minuteWindowStart = 0 + private _minuteCount = 0 + private _dayKey = '' + private _dayCount = 0 + + private _inFlight = false + private _abort: AbortController | null = null + private _timeoutTimer: NodeJS.Timeout | null = null + private _lastLatencyMs: number | null = null + private _warmUpPromise: Promise | null = null + private _warmUpAbort: AbortController | null = null + private _warmUpRetryTimer: NodeJS.Timeout | null = null + + + override on(event: K, listener: SuggestionEvents[K]): this { + return super.on(event, listener as (...args: unknown[]) => void) + } + + override off(event: K, listener: SuggestionEvents[K]): this { + return super.off(event, listener as (...args: unknown[]) => void) + } + + override emit( + event: K, + ...args: Parameters + ): boolean { + return super.emit(event, ...args) + } + + // ── 상태 ────────────────────────────────────────────── + + isEnabled(): boolean { + return configGet('suggestionEnabled') === true + } + + get isVisible(): boolean { + return this._candidates.length > 0 + } + + get isPresentationActive(): boolean { + return this.isVisible || this._generating || this._warmingUp || this._partialText.length > 0 + } + + get activeText(): string | null { + return this._candidates[this._activeIndex]?.text ?? null + } + + getState(): SuggestionState { + const now = Date.now() + this._rollCounters(now) + return { + enabled: this.isEnabled(), + modelId: this.resolveModel(), + modelAvailable: getLocalLLMService().isAvailable(), + visible: this.isVisible, + generating: this._generating, + warmingUp: this._warmingUp, + partialText: this._partialText || null, + candidates: [...this._candidates], + activeIndex: this._activeIndex, + anchor: this._anchor, + appName: this._appName, + updatedAt: now, + lastSkipReason: this._lastSkipReason, + requestsToday: this._dayCount, + coolingDown: Date.now() < this._cooldownUntil, + dailyBudget: configGet('suggestionDailyBudget') ?? SUGGESTION_DEFAULTS.dailyBudget, + lastLatencyMs: this._lastLatencyMs, + triggerDelayMs: this.readPolicyConfig().triggerDelayMs, + minPrefixChars: this.readPolicyConfig().minPrefixChars, + requestTimeoutMs: this.readTimeoutMs(), + learnTypedText: configGet('inputLearnTypedText') === true, + telemetryEnabled: configGet('inputTelemetryEnabled') === true, + overlayInteractive: configGet('suggestionOverlayInteractive') !== false, + provenance: this._provenance + } + } + + /** 제안 전용 모델 (설정 없으면 기본 LLM 모델). */ + resolveModel(): string | null { + const dedicated = configGet('suggestionModelId') + if (dedicated) return dedicated + return configGet('llmModelId') ?? null + } + + /** 설정 변경 반영 — 꺼지면 즉시 오버레이를 내린다. */ + applyConfig(): void { + if (!this.isEnabled()) { + this.dismiss('disabled') + } + this.emit('state-changed', this.getState()) + } + + setEnabled(enabled: boolean): void { + configSet('suggestionEnabled', enabled) + if (!enabled) this.dismiss('disabled') + else void this.warmUp() + this.emit('state-changed', this.getState()) + } + + /** 설정에서 명시적으로 제안을 켤 때만 모델을 짧게 준비한다. */ + warmUp(): Promise { + if (!this.isEnabled()) return Promise.resolve() + if (this._warmUpPromise) return this._warmUpPromise + + const abort = new AbortController() + this._warmUpAbort = abort + this._warmingUp = true + this.emit('state-changed', this.getState()) + const warmUp = this._warmUpUntilReady(abort) + this._warmUpPromise = warmUp + void warmUp.finally(() => { + if (this._warmUpPromise === warmUp) this._warmUpPromise = null + if (this._warmUpAbort === abort) this._warmUpAbort = null + }) + return warmUp + } + + private async _warmUpUntilReady(abort: AbortController): Promise { + try { + for (let attempt = 0; attempt <= 15; attempt += 1) { + if (abort.signal.aborted || !this.isEnabled()) return + if (!getLocalLLMService().isAvailable()) { + if (attempt === 15) return + await this._waitForWarmUpRetry(abort.signal) + continue + } + + const model = this.resolveModel() + if (!model) return + + try { + const stream = getLocalLLMService().streamGenerate('hi', { + model, + maxTokens: 1, + temperature: 0, + signal: abort.signal, + keepAlive: SUGGESTION_KEEP_ALIVE + }) + for await (const chunk of stream) void chunk + if (abort.signal.aborted) return + logger.info(`제안 모델 워밍업 완료 (model=${model})`) + } catch (error) { + if (!abort.signal.aborted) { + logger.warn( + `제안 모델 워밍업 실패: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + return + } + } finally { + this._warmingUp = false + this.emit('state-changed', this.getState()) + } + } + + private _waitForWarmUpRetry(signal: AbortSignal): Promise { + return new Promise((resolve) => { + const finish = (): void => { + signal.removeEventListener('abort', finish) + if (this._warmUpRetryTimer) { + clearTimeout(this._warmUpRetryTimer) + this._warmUpRetryTimer = null + } + resolve() + } + this._warmUpRetryTimer = setTimeout(finish, 2000) + this._warmUpRetryTimer.unref?.() + signal.addEventListener('abort', finish, { once: true }) + }) + } + + private _cancelWarmUp(): void { + if (this._warmUpRetryTimer) { + clearTimeout(this._warmUpRetryTimer) + this._warmUpRetryTimer = null + } + this._warmUpAbort?.abort() + this._warmingUp = false + } + + // ── 입력 텍스트 처리 ──────────────────────────────── + + /** + * 입력 텔레메트리가 보내는 "지금 치고 있는 것" 이벤트. + * + * 정책 판단 → (stale 정리) → 필요하면 생성. + */ + handleTypingContext(context: TypingContext): void { + this._lastContext = context + const now = Date.now() + this._lastContextAt = now + this._watchdog() + this._rollCounters(now) + + const config = this.readPolicyConfig() + this._logDecisionInputs(context) + this._abortStaleGeneration(context.prefix) + + // 연속 실패 쿨다운. 모델이 다른 작업으로 바쁘면(로컬 에이전트 동시 사용 등) + // 요청이 계속 타임아웃되어 스피너만 깜빡인다 — 잠시 요청을 멈춘다. + if (Date.now() < this._cooldownUntil) { + this._lastSkipReason = 'cooldown' + if ( + this._canUseLocalMemory(context, config, now) && + this._publishLocalMemory( + this._currentPrefix(context), + context, + config.maxCandidates, + config.maxChars, + 0, + null + ) + ) { + return + } + if (this.isPresentationActive) this.dismiss('cooldown') + this.emit('state-changed', this.getState()) + return + } + + // 사용자가 X 로 닫은 직후에는 다시 띄우지 않는다 (닫기가 확실히 먹도록). + if (Date.now() < this._userDismissedUntil) { + this._lastSkipReason = 'cooldown' + this.emit('state-changed', this.getState()) + return + } + const decision = decideSuggestion({ + enabled: this.isEnabled(), + // 접두가 충분히 자랐으면 "이미 떠 있음" 으로 막지 않는다 (지속 갱신). + // (decideSuggestion 의 overlayVisible 인자는 아래에서 계산한다) + modelAvailable: getLocalLLMService().isAvailable(), + // 생성 중에도 "이미 진행 중" 으로 취급해 중복 요청과 깜빡임을 막는다. + overlayVisible: this.isPresentationActive && !this._shouldRegenerate(context.prefix), + composing: context.isComposing, + hasSelection: context.hasSelection, + isPassword: context.isPassword, + isEditable: context.isEditable, + appName: context.appName, + excludedApps: config.excludedApps, + prefix: context.prefix, + idleMs: context.idleMs, + triggerDelayMs: config.triggerDelayMs, + minPrefixChars: config.minPrefixChars, + sinceLastRequestMs: now - this._lastRequestAt, + minIntervalMs: SUGGESTION_DEFAULTS.minIntervalMs, + requestsThisMinute: this._minuteCount, + maxRequestsPerMinute: config.maxRequestsPerMinute, + requestsToday: this._dayCount, + dailyBudget: config.dailyBudget + }) + + if (decision.action === 'clear') { + logger.debug(`제안 삭제: ${decision.reason}`) + if (this.isPresentationActive) this.dismiss(decision.reason) + else this._lastSkipReason = decision.reason + this.emit('state-changed', this.getState()) + return + } + + // 이미 보여준 제안이 현재 접두와 어긋나면 즉시 지운다 (tab-completion 표준 동작). + if (this.isVisible && !this._matchesGeneratedPrefix(context.prefix)) { + this.dismiss('stale') + this.emit('state-changed', this.getState()) + return + } + + if (decision.action === 'skip') { + logger.debug(`제안 건너: ${decision.reason}`) + this._lastSkipReason = decision.reason + + // 같은 텍스트로 이미 요청했다면 다시 만들지 않는다. + // (필드를 클릭만 했거나, 엔터로 보낸 뒤 UIA 가 옛 텍스트를 돌려줄 때 + // 제안이 반복 생성되던 문제) + if (this._currentPrefix(context) === this._lastRequestedPrefix && this._lastRequestedPrefix) { + this._lastSkipReason = 'unchanged' + this.emit('state-changed', this.getState()) + return + } + + + // 모델이 아직 안 떠 있으면 "준비 중" 을 보여준다 — 아무 반응이 없으면 + // 사용자는 기능이 죽었다고 판단한다(첫 실행에서 실제로 그랬다). + if (decision.reason === 'model-unavailable') { + if (this._canUseLocalMemory(context, config, now)) { + const shown = this._publishLocalMemory( + this._currentPrefix(context), + context, + config.maxCandidates, + config.maxChars, + 0, + null + ) + if (shown) return + } + this._warmingUp = true + this._anchor = context.anchor + this._appName = context.appName + // 워밍업 안내도 수명이 있다 — 모델이 뜬 뒤 갱신이 없으면 스스로 사라진다. + this._armVisibleTtl() + this.emit('updated', this.getState()) + } + + this.emit('state-changed', this.getState()) + return + } + + // 비동기 실패가 조용히 사라지면 기능이 죽은 이유를 알 수 없다. + void this._generate(decision.prefix, context, config.maxCandidates, config.maxChars).catch( + (error: unknown) => { + logger.warn( + `제안 생성 파이프라인 예외: ${error instanceof Error ? error.message : String(error)}` + ) + this._releaseGeneration() + } + ) + } + + /** 수동 요청 (설정/단축키 경로). 디바운스를 건너뛴다. */ + async requestNow(): Promise { + if (!this.isEnabled()) return { ok: false, reason: 'disabled' } + const context = this._lastContext + if (!context?.available || !context.isEditable || context.isPassword || context.hasSelection) { + const reason = context?.isPassword + ? 'password-field' + : context?.hasSelection + ? 'selection-active' + : 'not-editable' + if (this.isPresentationActive) this.dismiss(reason) + return { ok: false, reason } + } + const prefix = context.prefix.replace(/\s+$/u, '') + if (prefix.length < 1) return { ok: false, reason: 'empty-prefix' } + + const config = this.readPolicyConfig() + await this._generate(prefix, context, config.maxCandidates, config.maxChars).catch((error: unknown) => { + logger.warn(`수동 제안 생성 예외: ${error instanceof Error ? error.message : String(error)}`) + this._releaseGeneration() + }) + return this.isVisible ? { ok: true } : { ok: false, reason: this._lastSkipReason ?? 'generation-failed' } + } + + private _canUseLocalMemory( + context: TypingContext, + config: ReturnType, + now: number + ): boolean { + const decision = decideSuggestion({ + enabled: this.isEnabled(), + modelAvailable: true, + overlayVisible: false, + composing: context.isComposing, + hasSelection: context.hasSelection, + isPassword: context.isPassword, + isEditable: context.isEditable, + appName: context.appName, + excludedApps: config.excludedApps, + prefix: context.prefix, + idleMs: context.idleMs, + triggerDelayMs: config.triggerDelayMs, + minPrefixChars: config.minPrefixChars, + sinceLastRequestMs: now - this._lastRequestAt, + minIntervalMs: SUGGESTION_DEFAULTS.minIntervalMs, + requestsThisMinute: this._minuteCount, + maxRequestsPerMinute: config.maxRequestsPerMinute, + requestsToday: this._dayCount, + dailyBudget: config.dailyBudget + }) + return decision.action === 'request' + } + + private _collectMemoryHints(prefix: string, appName: string | null): LocalMemoryContext { + const telemetry = getInputTelemetryService() + const graphContext = getPersonalGraphService().retrieveContext(prefix, 4) + const phrases = telemetry.listPhrases(60) + const phraseHints = selectPhraseHints(phrases, prefix, 5, { appName }) + const requestedApp = appName?.trim().toLowerCase() + const appPhraseCount = requestedApp + ? phrases.filter( + (phrase) => + phrase.appName?.trim().toLowerCase() === requestedApp && phraseHints.includes(phrase.phrase) + ).length + : 0 + return { + continuationHints: graphContext.continuations, + relatedHints: graphContext.related, + phraseHints, + appPhraseCount + } + } + + private _isCurrentFallbackContext( + prefix: string, + context: TypingContext, + token: number | null + ): boolean { + if (token !== null && token !== this._generationToken) return false + const current = this._lastContext + if ( + !current || + !current.available || + !current.isEditable || + current.isPassword || + current.isComposing || + current.hasSelection + ) { + return false + } + return ( + current.prefix.replace(/\s+$/u, '').slice(-SUGGESTION_CONTEXT_MAX_CHARS) === + prefix.replace(/\s+$/u, '').slice(-SUGGESTION_CONTEXT_MAX_CHARS) && + current.appName === context.appName && + current.windowTitle === context.windowTitle + ) + } + + private _publishLocalMemory( + prefix: string, + context: TypingContext, + maxCandidates: number, + maxChars: number, + latencyMs: number, + token: number | null, + memory: LocalMemoryContext | null = null + ): boolean { + if (!this._isCurrentFallbackContext(prefix, context, token)) return false + const trimmedPrefix = prefix.slice(-SUGGESTION_CONTEXT_MAX_CHARS) + const localMemory = memory ?? this._collectMemoryHints(trimmedPrefix, context.appName) + const candidates = buildLocalSuggestionCandidates(trimmedPrefix, localMemory, maxCandidates, maxChars) + if (candidates.length === 0) return false + + this._partialText = '' + this._generating = false + this._warmingUp = false + this._candidates = candidates.map((text, index) => ({ text, rank: index })) + this._activeIndex = 0 + this._anchor = context.anchor + this._appName = context.appName + this._windowTitle = context.windowTitle + this._generatedForPrefix = trimmedPrefix + this._lastSkipReason = null + this._lastLatencyMs = latencyMs + this._provenance = { + mode: 'local-memory', + continuationCount: localMemory.continuationHints.length, + relatedCount: localMemory.relatedHints.length, + phraseCount: localMemory.phraseHints.length, + appPhraseCount: localMemory.appPhraseCount + } + this._record({ + prefix: trimmedPrefix, + text: candidates[0], + model: 'local-memory', + latencyMs, + candidateCount: candidates.length + }) + this._armVisibleTtl() + this.emit('updated', this.getState()) + this.emit('state-changed', this.getState()) + logger.info(`로컬 기억 제안 ${candidates.length}개 생성`) + return true + } + + // ── 생성 ────────────────────────────────────────────── + + private async _generate( + prefix: string, + context: TypingContext, + maxCandidates: number, + maxChars: number + ): Promise { + if (this._inFlight) { + // 침묵하면 "왜 안 뜨는지" 를 알 수 없다 — 진단을 남긴다. + logger.debug(`제안 건너뜀: 이미 생성 중 (${Date.now() - this._lastRequestAt}ms 경과)`) + return + } + + // 어떤 경로로 끝나든 진행 플래그는 반드시 해제한다. + // (emit/showOverlay 쪽 예외가 여기로 새면 _inFlight 가 true 로 굳어 + // 이후 모든 제안이 조용히 막힌다 — "한 번 나오고 안 나옴" 의 원인 후보) + try { + + const startedAt = Date.now() + const trimmedPrefix = prefix.slice(-SUGGESTION_CONTEXT_MAX_CHARS) + const model = this.resolveModel() + if (!model) { + const config = this.readPolicyConfig() + if ( + this._canUseLocalMemory(context, config, startedAt) && + this._publishLocalMemory( + trimmedPrefix, + context, + maxCandidates, + maxChars, + Date.now() - startedAt, + null + ) + ) { + return + } + this._lastSkipReason = 'model-unavailable' + this.emit('state-changed', this.getState()) + return + } + + this._abort?.abort() + const abort = new AbortController() + this._abort = abort + const token = (this._generationToken += 1) + + this._inFlight = true + this._generating = true + this._lastRequestedPrefix = prefix.replace(/\s+$/u, '') + this._warmingUp = false + this._provenance = null + this._lastRequestAt = Date.now() + this._minuteCount += 1 + this._dayCount += 1 + + // 후보가 오기 전에 빈 화면으로 기다리게 하지 않는다 — 즉시 자리를 잡고 + // "생성 중" 을 보여준 뒤 내용으로 채운다. 실측 5초 지연에서 특히 중요하다. + this._candidates = [] + this._activeIndex = 0 + this._anchor = context.anchor + this._appName = context.appName + this._windowTitle = context.windowTitle + this._armVisibleTtl() + this.emit('updated', this.getState()) + + const timeoutMs = this.readTimeoutMs() + let timedOut = false + if (this._timeoutTimer) clearTimeout(this._timeoutTimer) + this._timeoutTimer = setTimeout(() => { + // 모델이 하드웨어보다 느릴 때 UI 를 붙잡지 않는다 (다음 타이핑 주기에 재시도). + if (!abort.signal.aborted) { + timedOut = true + abort.abort() + } + }, timeoutMs) + this._timeoutTimer.unref?.() + + // 개인 그래프 문맥: (1) 과거에 그 꼬리 뒤에 실제로 이어 쓴 문장, + // (2) follows/용어공유 관계로 끌어온 관련 문장. + const memory = this._collectMemoryHints(trimmedPrefix, context.appName) + const continuationHints = memory.continuationHints + const hints = [...memory.relatedHints, ...memory.phraseHints] + .filter((value, index, list) => list.indexOf(value) === index) + .slice(0, 5) + const { systemPrompt, text } = buildSuggestionPrompt({ + prefix: trimmedPrefix, + appName: context.appName, + windowTitle: context.windowTitle, + phraseHints: hints, + continuationHints, + candidates: maxCandidates, + maxChars + }) + + let raw = '' + try { + this._partialText = '' + + const stream = getLocalLLMService().streamGenerate(text, { + model, + systemPrompt, + temperature: SUGGESTION_DEFAULTS.temperature, + maxTokens: SUGGESTION_DEFAULTS.maxOutputTokens, + signal: abort.signal, + keepAlive: SUGGESTION_KEEP_ALIVE + }) + + for await (const chunk of stream) { + if (abort.signal.aborted) break + raw += chunk + + // 도착하는 대로 오버레이에 흘려보낸다 — 사용자는 "계속 생성되는" 것을 본다. + // IPC 과다 방출을 막기 위해 120ms 간격으로만 보낸다. + this._partialText = sanitizePartial(raw) + const now = Date.now() + if (now - this._lastPartialEmitAt >= 120) { + this._lastPartialEmitAt = now + this.emit('updated', this.getState()) + } + + if (raw.length >= MAX_RAW_OUTPUT_CHARS) break + } + } catch (error) { + if (!abort.signal.aborted) { + this._lastSkipReason = 'generation-failed' + this._noteFailure(error instanceof Error ? error.message : String(error)) + if ( + this._publishLocalMemory( + trimmedPrefix, + context, + maxCandidates, + maxChars, + Date.now() - startedAt, + token, + memory + ) + ) { + logger.warn(`모델 제안 실패 후 로컬 기억 제안으로 전환: ${error instanceof Error ? error.message : String(error)}`) + return + } + this.dismiss('generation-failed') + this.emit('state-changed', this.getState()) + return + } + } finally { + if (this._abort === abort) this._abort = null + if (this._timeoutTimer) { + clearTimeout(this._timeoutTimer) + this._timeoutTimer = null + } + } + + if (abort.signal.aborted) { + if (token !== this._generationToken) { + logger.debug(`중단된 제안 결과 폐기 (세대 ${token} ≠ ${this._generationToken})`) + return + } + + if (!timedOut) return + + this._lastSkipReason = 'generation-failed' + this._noteFailure(`timeout ${timeoutMs}ms`) + if ( + this._publishLocalMemory( + trimmedPrefix, + context, + maxCandidates, + maxChars, + Date.now() - startedAt, + token, + memory + ) + ) { + logger.warn(`모델 제안 시간 초과 후 로컬 기억 제안으로 전환 (${timeoutMs}ms)`) + return + } + this.dismiss('generation-failed') + this.emit('state-changed', this.getState()) + return + } + + // 사용자가 그사이 닫았거나 새 요청이 시작됐다면 이 결과는 버린다. + if (token !== this._generationToken) { + logger.debug(`제안 결과 폐기 (세대 ${token} ≠ ${this._generationToken})`) + return + } + + // 타이핑을 멈추고 자리를 뜬 뒤 도착한 결과는 화면에 띄우지 않는다. + // (아무것도 안 치는데 제안창이 뜨던 신고의 원인 중 하나) + const staleness = Date.now() - this._lastContextAt + if (this._lastContextAt > 0 && staleness > SUGGESTION_DEFAULTS.resultMaxStalenessMs) { + logger.debug(`제안 결과 폐기 (문맥이 ${staleness}ms 지났다)`) + this.dismiss('stale') + return + } + + const candidates = parseSuggestionCandidates(raw, trimmedPrefix, maxCandidates, maxChars) + if (candidates.length === 0) { + this._lastSkipReason = 'generation-failed' + logger.info('제안 후보가 비어 있음 (모델 출력 정제 후)') + if ( + this._publishLocalMemory( + trimmedPrefix, + context, + maxCandidates, + maxChars, + Date.now() - startedAt, + token, + memory + ) + ) { + return + } + this.dismiss('generation-failed') + this.emit('state-changed', this.getState()) + return + } + + const latencyMs = Date.now() - startedAt + this._lastLatencyMs = latencyMs + this._consecutiveFailures = 0 + this._cooldownUntil = 0 + this._generating = false + this._partialText = '' + this._candidates = candidates.map((candidate, index) => ({ text: candidate, rank: index })) + this._activeIndex = 0 + this._anchor = context.anchor + this._appName = context.appName + this._windowTitle = context.windowTitle + this._generatedForPrefix = trimmedPrefix + this._lastSkipReason = null + this._provenance = { + mode: 'local-model', + continuationCount: continuationHints.length, + relatedCount: memory.relatedHints.length, + phraseCount: memory.phraseHints.length, + appPhraseCount: memory.appPhraseCount + } + + this._record({ prefix: trimmedPrefix, text: candidates[0], model, latencyMs, candidateCount: candidates.length }) + logger.info(`제안 ${candidates.length}개 생성 (${latencyMs}ms, model=${model})`) + this._armVisibleTtl() + this.emit('updated', this.getState()) + } finally { + // 어떤 경로로 끝나든 진행 플래그를 해제한다 — + // 예외가 어디로 새든 다음 제안이 조용히 막히지 않는다. + this._releaseGeneration() + } + } + + /** + * 생성 플래그를 무조건 해제한다 (누수 감시 포함). + * + * 응답 제한 + 여유를 넘겨 진행 중이면 비정상 상태로 보고 스스로 복구한다. + */ + private _releaseGeneration(): void { + this._inFlight = false + this._generating = false + this._partialText = '' + } + + /** 마지막 생성 이후 흐른 시간 (ms). */ + get inFlightAgeMs(): number { + return this._inFlight ? Date.now() - this._lastRequestAt : 0 + } + + /** 응답 제한 (설정 → SSOT 기본값). */ + private readTimeoutMs(): number { + const configured = configGet('suggestionRequestTimeoutMs') + return configured && configured > 0 ? configured : SUGGESTION_DEFAULTS.requestTimeoutMs + } + + /** 현재 정책 기준의 접두. */ + private _currentPrefix(context: TypingContext): string { + return context.prefix.replace(/\s+$/u, '') + } + + /** 판단 입력을 서명이 바뀔 때만 남긴다. */ + private _logDecisionInputs(context: TypingContext): void { + const signature = [ + context.available, + context.isEditable, + context.isPassword, + context.isComposing, + context.anchor ? 'A' : '-', + getLocalLLMService().isAvailable(), + this.isEnabled() + ].join('|') + + if (signature === this._lastDecisionSignature) return + this._lastDecisionSignature = signature + logger.info( + `제안 판단 입력: avail=${context.available} edit=${context.isEditable} pw=${context.isPassword} ` + + `comp=${context.isComposing} prefixLen=${context.prefix.length} idle=${context.idleMs}ms ` + + `app=${context.appName ?? '-'} enabled=${this.isEnabled()} ` + + `model=${getLocalLLMService().isAvailable()} anchor=${context.anchor ? 'yes' : 'no'}` + ) + } + + /** + * 생성 실패를 기록하고, 연속 실패면 잠시 쉰다. + * + * 실패할 때마다 새 요청이 스피너를 다시 걸면 "끝나지도 꺼지지도 않는" 것처럼 보인다 + * (실측 신고). 임계 도달 시 쿨다운 동안은 요청 자체를 하지 않는다. + */ + private _noteFailure(reason: string): void { + this._consecutiveFailures += 1 + if (this._consecutiveFailures < SUGGESTION_DEFAULTS.failureCooldownThreshold) return + + this._cooldownUntil = Date.now() + SUGGESTION_DEFAULTS.failureCooldownMs + this._consecutiveFailures = 0 + logger.warn( + `제안 생성 연속 실패 (${reason}) — ${Math.round( + SUGGESTION_DEFAULTS.failureCooldownMs / 1000 + )}초 동안 요청을 쉰다 (모델이 다른 작업으로 바쁠 수 있음)` + ) + } + + private _armVisibleTtl(): void { + if (this._ttlTimer) clearTimeout(this._ttlTimer) + this._ttlTimer = setTimeout(() => { + this._ttlTimer = null + logger.debug('제안 표시 수명 만료 — 자동으로 닫는다') + this.dismiss('stale') + }, SUGGESTION_DEFAULTS.visibleTtlMs) + this._ttlTimer.unref?.() + } + + /** 표시 수명 타이머 해제. */ + private _clearVisibleTtl(): void { + if (this._ttlTimer) { + clearTimeout(this._ttlTimer) + this._ttlTimer = null + } + } + + private _shouldRegenerate(currentPrefix: string): boolean { + return decideSuggestionRefresh(this._generatedForPrefix, currentPrefix) === 'regenerate' + } + + private _abortStaleGeneration(currentPrefix: string): void { + if (!this._inFlight || !this._abort || this._abort.signal.aborted) return + const refresh = decideSuggestionRefresh(this._lastRequestedPrefix, currentPrefix) + if (refresh === 'keep') return + + this._generationToken += 1 + this._abort.abort() + logger.debug(`제안 생성 취소 (${refresh}) — 다음 정상 입력 문맥에서만 재평가`) + } + + /** + * 진행 플래그가 비정상적으로 오래 남았으면 강제 해제한다. + * + * 응답 제한(기본 8초) + 5초를 넘긴 in-flight 는 죽은 것으로 본다 — + * 어떤 예외 경로로든 플래그가 누수되면 제안이 영구히 멈추기 때문이다. + */ + private _watchdog(): void { + if (!this._inFlight) return + const age = Date.now() - this._lastRequestAt + if (age < this.readTimeoutMs() + 5000) return + logger.warn(`제안 생성 플래그 누수 감지 (${age}ms) — 강제 해제`) + this._generationToken += 1 + this._abort?.abort() + this._abort = null + if (this._timeoutTimer) { + clearTimeout(this._timeoutTimer) + this._timeoutTimer = null + } + this._releaseGeneration() + this._lastSkipReason = 'stale' + this.emit('updated', this.getState()) + } + + private _matchesGeneratedPrefix(currentPrefix: string): boolean { + const current = currentPrefix.replace(/\s+$/u, '') + if (!this._generatedForPrefix) return false + return current === this._generatedForPrefix || current.startsWith(this._generatedForPrefix) + } + + // ── 사용자 동작 ─────────────────────────────────────── + + /** 이전 후보로 순환. */ + previous(): SuggestionState { + if (this._candidates.length > 1) { + this._activeIndex = + (this._activeIndex - 1 + this._candidates.length) % this._candidates.length + this.emit('updated', this.getState()) + } + return this.getState() + } + + next(): SuggestionState { + if (this._candidates.length > 1) { + this._activeIndex = (this._activeIndex + 1) % this._candidates.length + this.emit('updated', this.getState()) + } + return this.getState() + } + + dismiss(reason: SuggestionSkipReason = 'dismissed'): void { + if (reason === 'dismissed') { + // 명시적 닫기: 잠깐 조용히 있고, 진행 중 생성은 무효화한다. + this._userDismissedUntil = Date.now() + SUGGESTION_DEFAULTS.userDismissQuietMs + logger.info(`사용자가 제안을 닫음 — ${SUGGESTION_DEFAULTS.userDismissQuietMs}ms 동안 재표시하지 않는다`) + } + const wasVisible = this.isPresentationActive + // 진행 중이던 생성을 무효화한다 (닫았는데 잠시 뒤 결과가 다시 뜨는 것을 막는다). + this._generationToken += 1 + this._clearVisibleTtl() + this._candidates = [] + this._generating = false + this._warmingUp = false + this._partialText = '' + this._activeIndex = 0 + this._anchor = null + this._generatedForPrefix = '' + this._provenance = null + this._lastSkipReason = reason + this._abort?.abort() + this._abort = null + if (this._timeoutTimer) { + clearTimeout(this._timeoutTimer) + this._timeoutTimer = null + } + this._cancelWarmUp() + if (wasVisible || reason === 'dismissed') { + this.emit('cleared', { reason }) + } + } + + /** 활성 후보를 대상 앱에 삽입하고 수락으로 기록한다. */ + async accept(index?: number): Promise { + if (index !== undefined && index >= 0 && index < this._candidates.length) { + this._activeIndex = index + } + const candidate = this._candidates[this._activeIndex] + if (!candidate) return { ok: false, reason: 'already-visible' } + + const method = configGet('insertMethod') + try { + const result = await getTextInsertService().insertText( + candidate.text, + method === 'keyboard' ? 'keyboard' : 'clipboard' + ) + if (!result.success) { + logger.warn(`제안 삽입 실패 (method=${result.method}, len=${result.textLength})`) + return { ok: false, reason: 'generation-failed' } + } + } catch (error) { + logger.warn(`제안 삽입 예외: ${error instanceof Error ? error.message : String(error)}`) + return { ok: false, reason: 'generation-failed' } + } + + this._markAccepted(candidate.text) + // 수락한 문장은 사용자 문체의 확실한 표본이다 (학습 동의 시에만 저장됨). + getInputTelemetryService().recordExternalText(candidate.text, { + appName: this._appName, + windowTitle: this._windowTitle, + source: 'suggestion' + }) + + this.dismiss('accepted') + this.emit('state-changed', this.getState()) + return { ok: true } + } + + getHistory(limit = 50): SuggestionHistoryEntry[] { + try { + const db = getDatabase() + return db + .select() + .from(suggestions) + .orderBy(desc(suggestions.createdAt)) + .limit(Math.max(1, Math.min(limit, 200))) + .all() + .map((row) => ({ + id: row.id, + appName: row.appName, + prefixText: row.prefixText, + suggestionText: row.suggestionText, + model: row.model, + latencyMs: row.latencyMs, + accepted: row.accepted, + createdAt: row.createdAt + })) + } catch (error) { + logger.warn(`제안 이력 조회 실패: ${error instanceof Error ? error.message : String(error)}`) + return [] + } + } + + // ── 내부 ────────────────────────────────────────────── + + private readPolicyConfig(): { + triggerDelayMs: number + minPrefixChars: number + maxRequestsPerMinute: number + dailyBudget: number + maxCandidates: number + maxChars: number + excludedApps: string[] + } { + return { + triggerDelayMs: configGet('suggestionTriggerDelayMs') || SUGGESTION_DEFAULTS.triggerDelayMs, + minPrefixChars: configGet('suggestionMinPrefixChars') || SUGGESTION_DEFAULTS.minPrefixChars, + maxRequestsPerMinute: + Math.min( + configGet('suggestionMaxRequestsPerMinute') || SUGGESTION_DEFAULTS.maxRequestsPerMinute, + 12 + ), + dailyBudget: configGet('suggestionDailyBudget') || SUGGESTION_DEFAULTS.dailyBudget, + maxCandidates: SUGGESTION_DEFAULTS.maxCandidates, + maxChars: SUGGESTION_MAX_OUTPUT_CHARS, + excludedApps: [...configGet('inputExcludedApps')] + } + } + + /** 분/일 카운터 롤링. */ + private _rollCounters(now: number): void { + if (now - this._minuteWindowStart >= 60000) { + this._minuteWindowStart = now + this._minuteCount = 0 + } + const dayKey = new Date(now).toDateString() + if (dayKey !== this._dayKey) { + this._dayKey = dayKey + this._dayCount = 0 + } + } + + private _record(input: { + prefix: string + text: string + model: string + latencyMs: number + candidateCount: number + }): void { + try { + const db = getDatabase() + db.insert(suggestions) + .values({ + id: crypto.randomUUID(), + appName: this._appName, + prefixText: input.prefix.slice(-SUGGESTION_CONTEXT_MAX_CHARS), + suggestionText: input.text, + candidateCount: input.candidateCount, + model: input.model, + latencyMs: input.latencyMs, + accepted: false, + createdAt: Date.now() + }) + .run() + } catch (error) { + logger.warn(`제안 기록 실패: ${error instanceof Error ? error.message : String(error)}`) + } + } + + private _markAccepted(text: string): void { + try { + const db = getDatabase() + const row = db + .select({ id: suggestions.id }) + .from(suggestions) + .where(eq(suggestions.suggestionText, text)) + .orderBy(desc(suggestions.createdAt)) + .limit(1) + .get() + if (!row) return + db.update(suggestions).set({ accepted: true }).where(eq(suggestions.id, row.id)).run() + } catch (error) { + logger.warn(`수락 기록 실패: ${error instanceof Error ? error.message : String(error)}`) + } + } + + /** 테스트/진단 — 현재 앱이 제외 대상인지. */ + isAppAllowed(appName: string | null): boolean { + if (!appName) return true + return !isAppExcluded(appName, configGet('inputExcludedApps')) + } + + dispose(): void { + this.dismiss('disabled') + this._cancelWarmUp() + this.removeAllListeners() + } +} + +/** + * 스트리밍 중간 텍스트 정제 — 번호/불릿/따옴표와 지시문 누출을 화면에 잠깐 + * 보여주지 않도록 줄 단위로 다듬는다. + */ +function sanitizePartial(raw: string): string { + const lines = raw + .split(/\r?\n/u) + .map((line) => sanitizeSuggestionLine(line, SUGGESTION_MAX_OUTPUT_CHARS) ?? line.trim()) + .filter((line) => line.length > 0) + return lines.slice(0, 2).join(' ').slice(0, SUGGESTION_MAX_OUTPUT_CHARS) +} + +let instance: SuggestionService | null = null + +export function getSuggestionService(): SuggestionService { + if (!instance) instance = new SuggestionService() + return instance +} + +export function resetSuggestionServiceForTests(): void { + instance?.dispose() + instance = null +} diff --git a/apps/desktop/src/main/services/TTSPlaybackService.ts b/apps/desktop/src/main/services/TTSPlaybackService.ts index 545c302..c470198 100644 --- a/apps/desktop/src/main/services/TTSPlaybackService.ts +++ b/apps/desktop/src/main/services/TTSPlaybackService.ts @@ -146,7 +146,7 @@ class TTSPlaybackService extends EventEmitter { '-NonInteractive', '-Command', script, - ], { stdio: 'pipe' }) + ], { stdio: 'pipe', windowsHide: true }) this._bindProcessHandlers(resolve, reject) }) diff --git a/apps/desktop/src/main/services/UiaContextService.ts b/apps/desktop/src/main/services/UiaContextService.ts new file mode 100644 index 0000000..c22afd3 --- /dev/null +++ b/apps/desktop/src/main/services/UiaContextService.ts @@ -0,0 +1,214 @@ +// src/main/services/UiaContextService.ts +// +// 사이드카의 UIA 브리지(`GET /uia/focus`)를 감싸는 유일한 창구. +// +// 사이드카인가: +// Win32 케어렛 API(GetGUIThreadInfo)는 Chromium/Electron 에서 동작하지 않고, +// Node UIA 바인딩(selection-hook / xa11y)은 케어 rect 나 IsPassword 를 노출하지 +// 않는다. 이미 배포 중인 Python 사이드카에 UIA 브리지를 두는 것이 검증된 유일한 경로다. +// +// 안전 규칙: +// - 비밀번호 필드는 브리지가 fail-closed 로 차단하지만, 여기서도 한 번 더 막는다. +// - 읽기 실패/타임아웃은 텍스트를 쓰지 않는다 (available=false). +// - 실패 후에는 백오프를 걸어 매 키입력마다 사이드카를 두드리지 않는다. + +import { D3ROError, ErrorCode } from '@d3ro/core/errors' +import { + emptyFocusSnapshot, + type FocusSnapshot, + type FocusTextSource, + type UiRect +} from '@d3ro/core/input-intelligence' +import { getLogger } from './LoggerService' +import { getLocalSTTService } from './LocalSTTService' + +const logger = getLogger('UiaContextService') + +const REQUEST_TIMEOUT_MS = 1500 +/** + * 실패 후 재시도 대기 (ms). + * + * 최초 실패는 대부분 "사이드카가 아직 포트를 열지 않음" 이다 — 실측: 부팅 0.87초 후 첫 실패, + * 사이드카 준비는 1.1초 후였다. 20초를 통째로 백오프하면 앱을 켠 직후의 입력이 전부 죽으므로 + * 일시 실패는 짧게 시작해 배수로 늘리고, 브리지 자체가 없는 경우(미지원 플랫폼)만 길게 잡는다. + */ +const BACKOFF_STEPS_MS = [1500, 3000, 6000, 12000, 30000] as const +/** 영구 실패(브리지 없음) 백오프 */ +const BACKOFF_PERMANENT_MS = 5 * 60 * 1000 + +interface UiaBridgePayload { + available?: boolean + reason?: string + isPassword?: boolean + isEditable?: boolean + isComposing?: boolean + hasSelection?: boolean + controlType?: string | null + controlName?: string | null + className?: string | null + textSource?: string + text?: string + caretOffset?: number | null + caretRect?: UiRect | null + elementRect?: UiRect | null + windowTitle?: string | null + processId?: number | null + capturedAt?: number +} + +class UiaContextService { + private _backoffUntil = 0 + /** 연속 실패 수 — 백오프 단계 계산용 */ + private _failureStreak = 0 + private _lastReason = '' + private _inFlight: Promise | null = null + private _last: FocusSnapshot | null = null + private _lastSuccessAt = 0 + + get lastSnapshot(): FocusSnapshot | null { + return this._last + } + + get lastSuccessAt(): number { + return this._lastSuccessAt + } + + /** 브리지 사용 가능 여부 (백오프 상태면 false). */ + isAvailable(now = Date.now()): boolean { + return now >= this._backoffUntil + } + + get lastReason(): string { + return this._lastReason + } + + /** + * 현재 포커스 입력 요소 스냅샷. + * + * 동시 호출은 하나의 요청으로 합친다 — 키 입력마다 부르므로 중복 방지가 중요하다. + */ + async getSnapshot(now = Date.now()): Promise { + if (!this.isAvailable(now)) { + return emptyFocusSnapshot(this._lastReason || 'backoff', now) + } + if (this._inFlight) return this._inFlight + + this._inFlight = this._fetch(now) + try { + return await this._inFlight + } finally { + this._inFlight = null + } + } + + /** 테스트/설정 변경 시 상태 초기화. */ + reset(): void { + this._backoffUntil = 0 + this._failureStreak = 0 + this._lastReason = '' + } + + private async _fetch(now: number): Promise { + let baseUrl: string + try { + baseUrl = await getLocalSTTService().ensureSidecar() + } catch (error) { + return this._fail( + `sidecar-unavailable:${error instanceof Error ? error.message : String(error)}`, + now + ) + } + + let payload: UiaBridgePayload + try { + const response = await fetch(`${baseUrl}/uia/focus`, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }) + if (!response.ok) { + return this._fail(`http-${response.status}`, now) + } + payload = (await response.json()) as UiaBridgePayload + } catch (error) { + return this._fail( + `request-failed:${error instanceof Error ? error.message : String(error)}`, + now + ) + } + + if (!payload.available) { + // 브리지가 없는 환경(미지원 플랫폼/미설치)은 길게 백오프한다. + return this._fail(payload.reason ?? 'unavailable', now) + } + + const snapshot: FocusSnapshot = { + available: true, + isPassword: payload.isPassword === true, + isEditable: payload.isEditable === true, + isComposing: payload.isComposing === true, + hasSelection: payload.hasSelection === true, + controlType: payload.controlType ?? undefined, + controlName: payload.controlName ?? undefined, + className: payload.className ?? undefined, + textSource: normalizeTextSource(payload.textSource), + text: typeof payload.text === 'string' ? payload.text : '', + caretOffset: typeof payload.caretOffset === 'number' ? payload.caretOffset : null, + caretRect: payload.caretRect ?? null, + elementRect: payload.elementRect ?? null, + windowTitle: payload.windowTitle ?? null, + appName: null, + processId: typeof payload.processId === 'number' ? payload.processId : null, + capturedAt: typeof payload.capturedAt === 'number' ? payload.capturedAt : now + } + + // 2차 방어: 비밀번호로 판정되면 텍스트를 즉시 버린다 (fail-closed). + if (snapshot.isPassword) { + snapshot.text = '' + snapshot.textSource = 'none' + snapshot.caretOffset = null + } + + this._last = snapshot + this._lastSuccessAt = now + this._backoffUntil = 0 + this._lastReason = '' + return snapshot + } + + private _fail(reason: string, now: number): FocusSnapshot { + this._lastReason = reason + + // 영구적 사용 불가(미지원 플랫폼/미설치)는 길게, 일시 오류는 단계적으로. + const permanent = reason === 'uiautomation-missing' || reason.startsWith('bridge-import') + const waitMs = permanent + ? BACKOFF_PERMANENT_MS + : BACKOFF_STEPS_MS[Math.min(this._failureStreak, BACKOFF_STEPS_MS.length - 1)] + + this._failureStreak += 1 + this._backoffUntil = now + waitMs + logger.warn( + `UIA 스냅샷 사용 불가 (${reason}) — ${waitMs}ms 백오프 (연속 ${this._failureStreak}회)` + ) + return emptyFocusSnapshot(reason, now) + } +} + +function normalizeTextSource(value: string | undefined): FocusTextSource { + if (value === 'value' || value === 'text' || value === 'legacy') return value + return 'none' +} + +let instance: UiaContextService | null = null + +export function getUiaContextService(): UiaContextService { + if (!instance) instance = new UiaContextService() + return instance +} + +export function resetUiaContextServiceForTests(): void { + instance = null +} + +/** 진단용 — D3ROError 로 감싼 브리지 오류 생성. */ +export function createUiaBridgeError(reason: string): D3ROError { + return new D3ROError(ErrorCode.UiaBridgeUnavailable, `UIA bridge unavailable: ${reason}`) +} diff --git a/apps/desktop/src/main/services/VoiceActionService.ts b/apps/desktop/src/main/services/VoiceActionService.ts index d1a140a..c5f01c1 100644 --- a/apps/desktop/src/main/services/VoiceActionService.ts +++ b/apps/desktop/src/main/services/VoiceActionService.ts @@ -258,7 +258,7 @@ class VoiceActionService extends EventEmitter { private _openApp(appName: string): Promise { return new Promise((resolve, reject) => { const cmd = `start "" "${appName}"` - exec(cmd, { shell: 'cmd.exe' }, (err) => { + exec(cmd, { shell: 'cmd.exe', windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) @@ -277,7 +277,7 @@ class VoiceActionService extends EventEmitter { $wshell = New-Object -ComObject WScript.Shell $wshell.SendKeys([char]175) ` - exec(`powershell -NoProfile -Command "${script}"`, (err) => { + exec(`powershell -NoProfile -Command "${script}"`, { windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) @@ -288,7 +288,7 @@ class VoiceActionService extends EventEmitter { $wshell = New-Object -ComObject WScript.Shell $wshell.SendKeys([char]174) ` - exec(`powershell -NoProfile -Command "${script}"`, (err) => { + exec(`powershell -NoProfile -Command "${script}"`, { windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) @@ -299,7 +299,7 @@ class VoiceActionService extends EventEmitter { $wshell = New-Object -ComObject WScript.Shell $wshell.SendKeys([char]173) ` - exec(`powershell -NoProfile -Command "${script}"`, (err) => { + exec(`powershell -NoProfile -Command "${script}"`, { windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) @@ -318,7 +318,7 @@ class VoiceActionService extends EventEmitter { $wshell = New-Object -ComObject WScript.Shell $wshell.SendKeys('${sendKeysStr}') ` - exec(`powershell -NoProfile -Command "${script}"`, (err) => { + exec(`powershell -NoProfile -Command "${script}"`, { windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) @@ -336,7 +336,7 @@ class VoiceActionService extends EventEmitter { private _runCommand(command: string): Promise { return new Promise((resolve, reject) => { - exec(command, { timeout: 10000 }, (err) => { + exec(command, { timeout: 10000, windowsHide: true }, (err) => { if (err) reject(err) else resolve() }) diff --git a/apps/desktop/src/main/services/VoiceConversationService.ts b/apps/desktop/src/main/services/VoiceConversationService.ts index 1978897..45b8751 100644 --- a/apps/desktop/src/main/services/VoiceConversationService.ts +++ b/apps/desktop/src/main/services/VoiceConversationService.ts @@ -37,6 +37,7 @@ class VoiceConversationService extends EventEmitter { private _audioBuffers: Buffer[] = [] private _audioListenerBound = false private _audioLevelListenerBound = false + private _responseAbortController: AbortController | null = null get state(): ConversationState { return this._state @@ -105,6 +106,7 @@ class VoiceConversationService extends EventEmitter { if (!this._isActive) return this._stopListening() + this._responseAbortController?.abort() getTTSPlaybackService().stop() getPremiumLLMService().cancelGeneration() @@ -128,9 +130,10 @@ class VoiceConversationService extends EventEmitter { * 현재 LLM 응답 또는 TTS 재생을 취소. */ cancelResponse(): void { + this._responseAbortController?.abort() getPremiumLLMService().cancelGeneration() getTTSPlaybackService().stop() - if (this._isActive) { + if (this._isActive && this._state !== 'listening') { this._setState('listening') this._startListening() } @@ -238,6 +241,8 @@ class VoiceConversationService extends EventEmitter { const language = (configGet('sttLanguage') as string | undefined) ?? 'auto' const result = await sttService.transcribe(audioBuffer, { language, vadFilter: true }) + if (!this._isActive) return + if (!result.text || result.text.trim().length === 0) { // Bug 13: VAD가 전체 오디오를 무음 판정한 경우에도 사용자 피드백. this._emitError('stt', 'No speech detected. Check microphone and try again.') @@ -248,6 +253,7 @@ class VoiceConversationService extends EventEmitter { await this._processUserMessage(result.text.trim()) } catch (err) { + if (!this._isActive) return logger.error('STT failed in conversation:', err) this._emitError('stt', err instanceof Error ? err.message : 'STT failed') this._setState('listening') @@ -256,6 +262,13 @@ class VoiceConversationService extends EventEmitter { } private async _processUserMessage(text: string): Promise { + if (this._responseAbortController) { + throw new D3ROError(ErrorCode.ConversationLLMFailed, 'Conversation response is already in progress') + } + + const responseAbortController = new AbortController() + this._responseAbortController = responseAbortController + // 사용자 메시지 추가 const userMsg: ConversationMessage = { id: crypto.randomUUID(), @@ -271,7 +284,7 @@ class VoiceConversationService extends EventEmitter { try { // LLM 백엔드 선택: premium 설정 + 사용 가능 → PremiumLLM, 아니면 LocalLLM - const { generator: chatGenerator, backend } = await this._createChatStream() + const { generator: chatGenerator, backend } = await this._createChatStream(responseAbortController.signal) logger.info(`Conversation LLM backend: ${backend}`) const assistantMsgId = crypto.randomUUID() @@ -280,6 +293,8 @@ class VoiceConversationService extends EventEmitter { let sentenceBuffer = '' for await (const token of chatGenerator) { + if (!this._isCurrentResponse(responseAbortController)) return + accumulated += token // 렌더러에 델타 전송 @@ -308,6 +323,8 @@ class VoiceConversationService extends EventEmitter { } } + if (!this._isCurrentResponse(responseAbortController)) return + // 남은 텍스트도 TTS 큐에 추가 if (sentenceBuffer.trim()) { ttsSentences.push(sentenceBuffer.trim()) @@ -331,12 +348,16 @@ class VoiceConversationService extends EventEmitter { // TTS 재생 if (ttsSentences.length > 0) { + if (!this._isCurrentResponse(responseAbortController)) return + this._setState('speaking') this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.TTS_STARTED, {}) const ttsService = getTTSPlaybackService() await ttsService.speakSentences(ttsSentences) + if (!this._isCurrentResponse(responseAbortController)) return + this._sendToRenderer(IPC_CHANNELS.VOICE_CONVERSATION.TTS_FINISHED, {}) // 응답 완료 chime (자동 listening 재진입 직전) @@ -349,12 +370,18 @@ class VoiceConversationService extends EventEmitter { this._startListening() } } catch (err) { + if (responseAbortController.signal.aborted) return + logger.error('LLM chat failed in conversation:', err) this._emitError('llm', err instanceof Error ? err.message : 'LLM failed') if (this._isActive) { this._setState('listening') this._startListening() } + } finally { + if (this._responseAbortController === responseAbortController) { + this._responseAbortController = null + } } } @@ -362,7 +389,7 @@ class VoiceConversationService extends EventEmitter { * LLM 백엔드 선택 + chatStream 생성. * premium 설정 + 사용 가능 → PremiumLLM, 실패 시 LocalLLM fallback. */ - private async _createChatStream(): Promise<{ + private async _createChatStream(signal: AbortSignal): Promise<{ generator: AsyncGenerator backend: 'local' | 'premium' }> { @@ -389,7 +416,21 @@ class VoiceConversationService extends EventEmitter { 'Local LLM (Ollama) is not available', ) } - return { generator: local.chatStream(chatMessages), backend: 'local' } + return { + generator: local.chatStream(chatMessages, { + signal, + maxTokens: 512, + timeoutMs: 60_000, + keepAlive: '2m', + }), + backend: 'local', + } + } + + private _isCurrentResponse(responseAbortController: AbortController): boolean { + return this._isActive + && this._responseAbortController === responseAbortController + && !responseAbortController.signal.aborted } private _buildChatMessages(): Array<{ role: string; content: string }> { diff --git a/apps/desktop/src/main/services/global-input-hook.ts b/apps/desktop/src/main/services/global-input-hook.ts new file mode 100644 index 0000000..c8684c6 --- /dev/null +++ b/apps/desktop/src/main/services/global-input-hook.ts @@ -0,0 +1,69 @@ +// src/main/services/global-input-hook.ts +// uiohook 글로벌 후킹의 소유권을 참조 카운트로 관리한다. +// +// uiohook 은 프로세스 전역 싱글턴이다. KeyBindingService InputTelemetryService 가 +// 각자 start()/stop() 을 부르면 두 번째 start() 는 무시되거나, 먼저 끝난 쪽의 stop() 이 +// 상대방 수신까지 죽인다. 그래서 "후킹이 켜져 있어야 하는 소비자 수"를 세고 +// 0 이 될 때만 실제로 춘다. + +import { uIOhook } from 'uiohook-napi' +import { getLogger } from './LoggerService' + +const logger = getLogger('GlobalInputHook') + +let refCount = 0 +let running = false + +/** + * 글로벌 후킹을 (필요하면) 시작하고 해제 함수를 돌려준다. + * + * 반드시 반환된 함수를 호출해야 한다. 시작에 실패하면 예외를 던지고 카운트를 되돌린다. + */ +export function acquireGlobalInputHook(): () => void { + refCount += 1 + + if (!running) { + try { + uIOhook.start() + running = true + logger.info('uiohook started (global input hook active)') + } catch (error) { + refCount = Math.max(0, refCount - 1) + throw error + } + } + + let released = false + return () => { + if (released) return + released = true + refCount = Math.max(0, refCount - 1) + + if (refCount === 0 && running) { + try { + uIOhook.stop() + logger.info('uiohook stopped (no consumer left)') + } catch (error) { + logger.error( + `Failed to stop uiohook: ${error instanceof Error ? error.message : String(error)}` + ) + } finally { + running = false + } + } + } +} + +export function isGlobalInputHookRunning(): boolean { + return running +} + +export function getGlobalInputHookRefCount(): number { + return refCount +} + +/** 테스트 전용 — 모듈 상태 초기화. */ +export function resetGlobalInputHookForTests(): void { + refCount = 0 + running = false +} \ No newline at end of file diff --git a/apps/desktop/src/main/services/llm-prompts.ts b/apps/desktop/src/main/services/llm-prompts.ts index 5e42300..b8a9324 100644 --- a/apps/desktop/src/main/services/llm-prompts.ts +++ b/apps/desktop/src/main/services/llm-prompts.ts @@ -143,3 +143,98 @@ export function resolveSystemPrompt( } export { BASE_SYSTEM_PROMPTS } + +// ============================================================ +// 다음 문장 제안 (ghost text) — 프롬프트 정본 +// ============================================================ + +/** + * 추론 모델이 `thinking` 블록을 지 않게 하는 prefix. + * + * `LocalLLMService.processText` 가 같은 목적으로 쓰는 값과 같다. 제안 경로는 + * `streamGenerate` 를 직접 부르므로 여기서 직접 붙인다. + */ +export const SUGGESTION_NO_THINK_PREFIX = '/no_think' + +/** + * 제안 시스템 프롬프트. + * + * 지시문은 **시스템 프롬프트에만** 둔다. 지시문이 처리 대상 텍스트 자리에 들어가 + * 지시문 자체를 다듬어 돌려주던 회귀(밋 9c2b4d4)와 같은 부류의 사고를 막는다. + */ +const SUGGESTION_SYSTEM_PROMPT = `사용자가 지금 입력창에 글을 쓰는 중입니다. 사용자가 마지막으로 쓴 글 뒤에 이어질 다음 문장을 제안하세요. + +규칙: +- 사용자가 언어와 같은 언어로 쓰세요. +- 사용자의 말투와 문체를 유지하세요. 존댓말/반말, 격식/비격식을 바꾸지 마세요. +- 이미 나온 단어를 되풀이하지 말고 이어지는 내용만 쓰세요. +- 후보를 한 줄에 하나씩, 번호나 따옴표, 설명 없이 출력하세요. +- 각 후보는 한 문장, 최대 {{maxChars}}자입니다. +- 확신이 없으면 짧게 쓰세요. 리말, 요약, 빈 줄을 출력하지 마세요.` + +export interface SuggestionPromptInput { + /** 어렛 앞까지의 텍스트 */ + prefix: string + /** 활성 앱/창 (맥락 트) */ + appName?: string | null + windowTitle?: string | null + /** 사용자가 자주 쓰는 표현 (개인화 힌트) */ + phraseHints?: readonly string[] + /** + * 과거에 같은 꼬리 뒤에 실제로 이어 쓴 문장 (개인 기억). + * + * 자주 쓰는 표현보다 훨씬 강한 문맥 신호다 — 사용자 자신의 실제 이어쓰기다. + */ + continuationHints?: readonly string[] + /** 후보 개수 */ + candidates?: number + /** 후보당 최대 길이 */ + maxChars?: number +} + +/** + * 제안 요청용 (systemPrompt, text) 를 만든다. + * + * 시스템 프롬프트에 지시문, 사용자 메시지에는 **지금까지 쓴 텍스트와 맥락**만 는다. + */ +export function buildSuggestionPrompt(input: SuggestionPromptInput): { + systemPrompt: string + text: string +} { + const candidates = Math.max(1, Math.min(input.candidates ?? 3, 5)) + const maxChars = Math.max(20, Math.min(input.maxChars ?? 160, 400)) + + const sections: string[] = ['[지금까지 텍스트]', input.prefix] + + if (input.phraseHints && input.phraseHints.length > 0) { + sections.push('', '[자주 쓰는 표현]') + for (const hint of input.phraseHints.slice(0, 5)) { + sections.push(`- ${hint}`) + } + } + + if (input.appName || input.windowTitle) { + sections.push('', '[현재 앱]') + const app = [input.appName, input.windowTitle].filter(Boolean).join(' — ') + sections.push(app) + } + + if (input.continuationHints && input.continuationHints.length > 0) { + sections.push('', '[과거에 사용자가 비슷한 문장 뒤에 실제로 이어 쓴 내용]') + for (const hint of input.continuationHints.slice(0, 3)) { + sections.push(`- ${hint}`) + } + sections.push('위 내용은 문체와 맥락 참고용입니다. 그대로 복사하지 말고 이어질 문장을 새로 쓰세요.') + } + + sections.push('', `이어질 다음 문장 ${candidates}개를 한 줄씩 출력하세요.`) + + const systemPrompt = `${SUGGESTION_NO_THINK_PREFIX}\n${SUGGESTION_SYSTEM_PROMPT.replace( + '{{maxChars}}', + String(maxChars) + )}` + + return { systemPrompt, text: sections.join('\n') } +} + +export { SUGGESTION_SYSTEM_PROMPT } diff --git a/apps/desktop/src/main/utils/win32-foreground.ts b/apps/desktop/src/main/utils/win32-foreground.ts new file mode 100644 index 0000000..6d1bb34 --- /dev/null +++ b/apps/desktop/src/main/utils/win32-foreground.ts @@ -0,0 +1,180 @@ +// src/main/utils/win32-foreground.ts +// +// koffi(FFI)로 포그라운드 창 정보를 읽는다 — 창 제목 · 프로세스 · 실행 파일 · 창 rect. +// +// 왜 koffi 인가 (조사 결과): +// - `uiohook-napi` 는 키/마우스만 주고 포그라운드 창 정보는 없다. +// - `get-windows`(active-win 후속)는 ESM 전용 + 설치 시 node-pre-gyp 다운로드가 +// 필요한데 이 저장소는 install script 를 허용하지 않는다 (빌드가 깨진다). +// - `koffi` 3.3.1 은 N-API 8 prebuild 를 optional dep 로 배포해 install script 없이 +// 동작한다. User32/Kernel32 호출만 쓰므로 COM vtable 을 다 필요도 없다. +// +// 실패(비 Windows, 모듈 로드 실패)는 예외 대신 null 로 돌려준다 — 호출자는 +// 앱 이름 없이도 텔레메트리를 계속 수집할 수 있어야 한다. + +import { getLogger } from '../services/LoggerService' +import type { UiRect } from '@d3ro/core/input-intelligence' + +const logger = getLogger('win32-foreground') + +export interface ForegroundWindowInfo { + /** HWND (숫자로 정규화) */ + hwnd: number + title: string + processId: number + /** 실행 파일명 (예: chrome.exe). 알 수 없으면 '' */ + appName: string + /** 실행 파일 전체 경로 */ + appPath: string | null + /** 창 rect (스크린 좌표) */ + bounds: UiRect | null +} + +interface KoffiRect { + left: number + top: number + right: number + bottom: number +} + +type AnyFunc = (...args: unknown[]) => unknown + +interface Win32Api { + koffi: { decode: (buffer: Buffer, type: string, length: number) => string } + GetForegroundWindow: AnyFunc + GetWindowTextW: AnyFunc + GetWindowThreadProcessId: AnyFunc + GetWindowRect: AnyFunc + OpenProcess: AnyFunc + QueryFullProcessImageNameW: AnyFunc + CloseHandle: AnyFunc +} + +const PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 +const TITLE_BUFFER_CHARS = 512 +const PATH_BUFFER_CHARS = 1024 + +let api: Win32Api | null = null +let loadFailed = false + +function loadApi(): Win32Api | null { + if (api) return api + if (loadFailed) return null + if (process.platform !== 'win32') { + loadFailed = true + return null + } + + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const koffi = require('koffi') as { + load: (lib: string) => { func: (signature: string) => AnyFunc } + struct: (name: string, fields: Record) => unknown + decode: (buffer: Buffer, type: string, length: number) => string + } + const user32 = koffi.load('user32.dll') + const kernel32 = koffi.load('kernel32.dll') + koffi.struct('RECT', { left: 'long', top: 'long', right: 'long', bottom: 'long' }) + + api = { + koffi: { decode: koffi.decode }, + GetForegroundWindow: user32.func('void *GetForegroundWindow()'), + GetWindowTextW: user32.func( + 'int GetWindowTextW(void *hWnd, _Out_ uint16_t *lpString, int nMaxCount)' + ), + GetWindowThreadProcessId: user32.func( + 'uint32_t GetWindowThreadProcessId(void *hWnd, _Out_ uint32_t *lpdwProcessId)' + ), + GetWindowRect: user32.func('bool GetWindowRect(void *hWnd, _Out_ RECT *rect)'), + OpenProcess: kernel32.func('void *OpenProcess(uint32_t access, bool inherit, uint32_t pid)'), + QueryFullProcessImageNameW: kernel32.func( + 'bool QueryFullProcessImageNameW(void *hProcess, uint32_t flags, _Out_ uint16_t *lpExeName, _Inout_ uint32_t *size)' + ), + CloseHandle: kernel32.func('bool CloseHandle(void *handle)') + } + return api + } catch (error) { + loadFailed = true + logger.warn( + `koffi/Win32 FFI 사용 불가 — 포그라운드 정보 없이 계속 (${ + error instanceof Error ? error.message : String(error) + })` + ) + return null + } +} + +/** 포그라운드 창 정보. Windows 가 아니거나 FFI 로드 실패면 null. */ +export function getForegroundWindowInfo(): ForegroundWindowInfo | null { + const a = loadApi() + if (!a) return null + + try { + const hwndRaw = a.GetForegroundWindow() as bigint | number | null + if (hwndRaw === null) return null + const hwnd = Number(hwndRaw) + if (!Number.isFinite(hwnd) || hwnd === 0) return null + + const titleBuffer = Buffer.alloc(TITLE_BUFFER_CHARS * 2) + const titleLength = Number(a.GetWindowTextW(hwnd, titleBuffer, TITLE_BUFFER_CHARS) as number) + const title = + titleLength > 0 + ? a.koffi.decode(titleBuffer, 'char16_t', titleLength).replace(/\0.*$/su, '') + : '' + + const pidBuffer = Buffer.alloc(4) + a.GetWindowThreadProcessId(hwnd, pidBuffer) + const processId = pidBuffer.readUInt32LE(0) + + const rect: KoffiRect = { left: 0, top: 0, right: 0, bottom: 0 } + const rectOk = a.GetWindowRect(hwnd, rect) as boolean + const bounds: UiRect | null = rectOk + ? { + x: Number(rect.left), + y: Number(rect.top), + width: Number(rect.right) - Number(rect.left), + height: Number(rect.bottom) - Number(rect.top) + } + : null + + let appPath: string | null = null + const processHandle = a.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, processId) as + | bigint + | number + | null + if (processHandle) { + try { + const sizeBuffer = Buffer.alloc(4) + sizeBuffer.writeUInt32LE(PATH_BUFFER_CHARS, 0) + const pathBuffer = Buffer.alloc(PATH_BUFFER_CHARS * 2) + const pathOk = a.QueryFullProcessImageNameW( + processHandle, + 0, + pathBuffer, + sizeBuffer + ) as boolean + if (pathOk) { + const length = sizeBuffer.readUInt32LE(0) + appPath = a.koffi.decode(pathBuffer, 'char16_t', length).replace(/\0.*$/su, '') + } + } finally { + a.CloseHandle(processHandle) + } + } + + const appName = appPath ? (appPath.split(/[\\/]/u).pop() ?? '') : '' + + return { hwnd, title, processId, appName, appPath, bounds } + } catch (error) { + logger.warn( + `포그라운드 창 조회 실패: ${error instanceof Error ? error.message : String(error)}` + ) + return null + } +} + +/** 테스트 전용 — 지연 로드 상태 초기화. */ +export function resetWin32ForegroundForTests(): void { + api = null + loadFailed = false +} \ No newline at end of file diff --git a/apps/desktop/src/main/windows/WindowManager.ts b/apps/desktop/src/main/windows/WindowManager.ts index 2d0d6f8..479b4cd 100644 --- a/apps/desktop/src/main/windows/WindowManager.ts +++ b/apps/desktop/src/main/windows/WindowManager.ts @@ -5,6 +5,7 @@ import { BrowserWindow, shell, screen, ipcMain, Menu, clipboard } from 'electron import { join } from 'path' import { is } from '@electron-toolkit/utils' import { WINDOW_SIZE } from '@d3ro/core/constants' +import { anchorFloatingPanel } from '@d3ro/core/input-intelligence' import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' import { getLogger } from '../services/LoggerService' import { getIsQuitting } from '../lifecycle' @@ -59,6 +60,19 @@ function getPopupI18nStrings(): Record { errorDefault: t('popup.error.default'), // caption-overlay captionLoading: t('popup.caption.loading'), + // suggestion-overlay + suggestionHintAccept: t('popup.suggestion.hintAccept'), + suggestionHintNext: t('popup.suggestion.hintNext'), + suggestionHintDismiss: t('popup.suggestion.hintDismiss'), + suggestionWarming: t('popup.suggestion.warming'), + suggestionHintGenerating: t('popup.suggestion.hintGenerating'), + suggestionLoading: t('popup.suggestion.loading'), + suggestionSourceModel: t('popup.suggestion.sourceModel'), + suggestionSourceMemory: t('popup.suggestion.sourceMemory'), + suggestionContinuations: t('popup.suggestion.continuations'), + suggestionRelated: t('popup.suggestion.related'), + suggestionPhrases: t('popup.suggestion.phrases'), + suggestionAppPhrases: t('popup.suggestion.appPhrases'), } } @@ -163,6 +177,7 @@ let resultPopupWindow: BrowserWindow | null = null let historyPopupWindow: BrowserWindow | null = null let commandPopupWindow: BrowserWindow | null = null let captionOverlayWindow: BrowserWindow | null = null +let suggestionOverlayWindow: BrowserWindow | null = null // ── 메인 윈도우 ─────────────────────────────────────── @@ -706,7 +721,177 @@ export function sendToCaptionOverlay(channel: string, data: unknown): void { } } -// ── 프리로딩 ────────────────────────────────────────── +// ── SuggestionOverlay 업 (입력 인텔리전스) ────────── + +/** + * 다음 문장 제안 ghost text 오버레이. + * + * IME 후보창과 같은 관례를 따른다: 케어/포커스 요소 바로 아래에 붙고, + * 포커스를 막지 않으며(focusable:false) 기본은 클릭 통과다. + * (KeyType.Windows 의 WS_EX_TRANSPARENT|WS_EX_NOACTIVATE 오버레이와 동일한 전략) + */ +const SUGGESTION_OVERLAY_WIDTH = 460 +/** 후보 5개 + 스크롤을 담을 높이 (목록은 내부 스크롤) */ +const SUGGESTION_OVERLAY_HEIGHT = 258 + +function applySuggestionOverlayMouseMode(win: BrowserWindow): void { + const interactive = configGet('suggestionOverlayInteractive') !== false + try { + if (interactive) { + // 클릭을 받아 수락할 수 있게 한다. focusable:false 라 대상 앱 포커스는 유지된다. + win.setIgnoreMouseEvents(false) + } else { + win.setIgnoreMouseEvents(true, { forward: true }) + } + } catch (err) { + logger.warn( + `SuggestionOverlay 마우스 모드 적용 실패: ${err instanceof Error ? err.message : String(err)}` + ) + } +} + +function createSuggestionOverlayWindow(): BrowserWindow { + const win = new BrowserWindow({ + width: SUGGESTION_OVERLAY_WIDTH, + height: SUGGESTION_OVERLAY_HEIGHT, + show: false, + frame: false, + transparent: true, + resizable: false, + alwaysOnTop: true, + skipTaskbar: true, + focusable: false, + webPreferences: { + preload: join(__dirname, '../preload/popup.js'), + sandbox: false, + contextIsolation: true, + nodeIntegration: false, + backgroundThrottling: false + } + }) + + applySuggestionOverlayMouseMode(win) + + if (is.dev && process.env['ELECTRON_RENDERER_URL']) { + win.loadURL(`${process.env['ELECTRON_RENDERER_URL']}/popups/suggestion-overlay/index.html`) + } else { + win.loadFile(join(__dirname, '../renderer/popups/suggestion-overlay/index.html')) + } + + attachPopupLifecycle(win, 'suggestion-overlay') + + win.on('closed', () => { + suggestionOverlayWindow = null + }) + + return win +} + +export function getSuggestionOverlayWindow(): BrowserWindow { + if (!suggestionOverlayWindow || suggestionOverlayWindow.isDestroyed()) { + suggestionOverlayWindow = createSuggestionOverlayWindow() + logger.info('SuggestionOverlay window created') + } + return suggestionOverlayWindow +} + +/** 제안 오버레이 표시 — 커(케어 → 요소 → 커서) 기준 배치. */ +export function showSuggestionOverlay(payload: { + candidates: Array<{ text: string; rank: number }> + activeIndex: number + /** 후보 도착 전(생성 중) — 팝업이 로딩 행을 보여준다 */ + generating?: boolean + /** 모델 적재 중 — 팝업이 "준비 중" 을 보여준다 */ + warmingUp?: boolean + /** 스트리밍 중 부분 텍스트 */ + partialText?: string | null + provenance?: { + mode: 'local-model' | 'local-memory' + continuationCount: number + relatedCount: number + phraseCount: number + appPhraseCount: number + } | null + anchor: { x: number; y: number; width: number; height: number } | null + appName: string | null +}): void { + const win = getSuggestionOverlayWindow() + const cursor = screen.getCursorScreenPoint() + const anchorPoint = payload.anchor + ? { x: payload.anchor.x, y: payload.anchor.y + payload.anchor.height } + : cursor + const display = screen.getDisplayNearestPoint(anchorPoint) + + const position = anchorFloatingPanel( + payload.anchor, + cursor, + { width: SUGGESTION_OVERLAY_WIDTH, height: SUGGESTION_OVERLAY_HEIGHT }, + display.workArea + ) + + win.setBounds({ + x: position.x, + y: position.y, + width: SUGGESTION_OVERLAY_WIDTH, + height: SUGGESTION_OVERLAY_HEIGHT + }) + + sendToPopupWindow(win, IPC_CHANNELS.POPUP_SUGGESTION.SHOW, { + candidates: payload.candidates, + activeIndex: payload.activeIndex, + generating: payload.generating === true, + warmingUp: payload.warmingUp === true, + partialText: payload.partialText ?? null, + appName: payload.appName, + provenance: payload.provenance ?? null, + _i18n: getPopupI18nStrings() + }) + + presentPopup(win, 'screen-saver') +} + +export function updateSuggestionOverlay(payload: { + candidates: Array<{ text: string; rank: number }> + activeIndex: number + generating?: boolean + warmingUp?: boolean + partialText?: string | null + provenance?: { + mode: 'local-model' | 'local-memory' + continuationCount: number + relatedCount: number + phraseCount: number + appPhraseCount: number + } | null +}): void { + if (suggestionOverlayWindow && !suggestionOverlayWindow.isDestroyed()) { + sendToPopupWindow(suggestionOverlayWindow, IPC_CHANNELS.POPUP_SUGGESTION.UPDATE, payload) + } +} + +export function hideSuggestionOverlay(): void { + if (suggestionOverlayWindow && !suggestionOverlayWindow.isDestroyed()) { + sendToPopupWindow(suggestionOverlayWindow, IPC_CHANNELS.POPUP_SUGGESTION.HIDE, {}) + suggestionOverlayWindow.hide() + } +} + +export function isSuggestionOverlayVisible(): boolean { + return !!( + suggestionOverlayWindow && + !suggestionOverlayWindow.isDestroyed() && + suggestionOverlayWindow.isVisible() + ) +} + +/** 설정(클릭 허용) 변경 시 즉시 반영. */ +export function applySuggestionOverlayConfig(): void { + if (suggestionOverlayWindow && !suggestionOverlayWindow.isDestroyed()) { + applySuggestionOverlayMouseMode(suggestionOverlayWindow) + } +} + +// ─ 프리로딩 ────────────────────────────────────────── export function preloadPopupWindows(): void { getRecordingTipWindow() @@ -716,7 +901,7 @@ export function preloadPopupWindows(): void { logger.info('Popup windows preloaded') } -// ── 테마 재주입 (설정에서 테마 변경 시 호출) ────────── +// ── 테마 재주입 (설정에서 테마 변경 시 호출) ─────────── /** * 현재 살아있는 팝업 윈도우에 테마 CSS를 재주입. @@ -729,6 +914,7 @@ export function reapplyThemeToAllPopups(): void { historyPopupWindow, commandPopupWindow, captionOverlayWindow, + suggestionOverlayWindow, ] for (const win of popupWindows) { if (win && !win.isDestroyed()) { diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index c269e97..d69816b 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -882,6 +882,114 @@ const electronAPI = { getSubscriptionStatus: () => invoke(IPC_CHANNELS.PAYMENT.GET_SUBSCRIPTION_STATUS), }, + + // ── Input telemetry (입력 수집 동의 · 리포트) ─────────── + inputTelemetry: { + getState: () => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.GET_STATE + ), + setEnabled: (params: { enabled: boolean }) => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.SET_ENABLED, + params + ), + setPaused: (params: { paused: boolean }) => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.SET_PAUSED, + params + ), + getSummary: (params?: { days?: number }) => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.GET_SUMMARY, + params + ), + getPrivacyReceipt: () => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.GET_PRIVACY_RECEIPT + ), + getPhrases: (params?: { limit?: number }) => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.GET_PHRASES, + params + ), + deletePhrase: (params: { id: string }) => + invoke(IPC_CHANNELS.INPUT_TELEMETRY.DELETE_PHRASE, params), + clearAll: () => invoke(IPC_CHANNELS.INPUT_TELEMETRY.CLEAR_ALL), + getGraph: () => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.GET_GRAPH + ), + queryGraph: (params: { text: string; limit?: number }) => + invoke( + IPC_CHANNELS.INPUT_TELEMETRY.QUERY_GRAPH, + params + ), + onStateChanged: ( + cb: (state: import('@d3ro/core/input-intelligence').InputTelemetryState) => void + ): Unsubscribe => on(IPC_CHANNELS.INPUT_TELEMETRY.STATE_CHANGED, cb), + onActivity: ( + cb: (payload: { + totals: import('@d3ro/core/input-intelligence').InputActivityBucket + appName: string | null + }) => void + ): Unsubscribe => on(IPC_CHANNELS.INPUT_TELEMETRY.ACTIVITY, cb), + }, + + // ── 다음 문장 제안 (ghost text) ──────────────────────── + suggestion: { + getState: () => + invoke( + IPC_CHANNELS.SUGGESTION.GET_STATE + ), + setConfig: (params: { + enabled?: boolean + modelId?: string | null + triggerDelayMs?: number + minPrefixChars?: number + maxRequestsPerMinute?: number + dailyBudget?: number + overlayInteractive?: boolean + learnTypedText?: boolean + excludedApps?: string[] + requestTimeoutMs?: number + }) => + invoke( + IPC_CHANNELS.SUGGESTION.SET_CONFIG, + params + ), + requestNow: () => invoke<{ ok: boolean; reason?: string }>(IPC_CHANNELS.SUGGESTION.REQUEST_NOW), + accept: (params?: { index?: number }) => + invoke<{ ok: boolean; reason?: string }>(IPC_CHANNELS.SUGGESTION.ACCEPT, params), + next: () => + invoke( + IPC_CHANNELS.SUGGESTION.NEXT + ), + prev: () => + invoke( + IPC_CHANNELS.SUGGESTION.PREV + ), + dismiss: () => invoke(IPC_CHANNELS.SUGGESTION.DISMISS), + getHistory: (params?: { limit?: number }) => + invoke< + Array<{ + id: string + appName: string | null + prefixText: string + suggestionText: string + model: string | null + latencyMs: number | null + accepted: boolean + createdAt: number + }> + >(IPC_CHANNELS.SUGGESTION.GET_HISTORY, params), + onUpdated: (cb: (state: import('@d3ro/core/input-intelligence').SuggestionState) => void): Unsubscribe => + on(IPC_CHANNELS.SUGGESTION.UPDATED, cb), + onCleared: (cb: (payload: { reason: string }) => void): Unsubscribe => + on(IPC_CHANNELS.SUGGESTION.CLEARED, cb), + onStateChanged: (cb: (state: import('@d3ro/core/input-intelligence').SuggestionState) => void): Unsubscribe => + on(IPC_CHANNELS.SUGGESTION.STATE_CHANGED, cb), + }, } as const contextBridge.exposeInMainWorld('electronAPI', electronAPI) diff --git a/apps/desktop/src/renderer/components/SettingsModal.tsx b/apps/desktop/src/renderer/components/SettingsModal.tsx index bd7472e..49f95c1 100644 --- a/apps/desktop/src/renderer/components/SettingsModal.tsx +++ b/apps/desktop/src/renderer/components/SettingsModal.tsx @@ -1,7 +1,7 @@ // src/renderer/components/SettingsModal.tsx // 설계서 03: Settings React Modal — General(음성 모드+핫키)/Audio/STT/LLM 탭 -import { useState, useEffect, useCallback } from 'react' +import { useState, useEffect, useCallback, useMemo } from 'react' import { Dialog, DialogTitle, @@ -22,19 +22,21 @@ import { Button, Paper, } from '@mui/material' -import { X, Keyboard, Mic, Lock, Cloud, RefreshCw, Play, Cpu, HelpCircle } from 'lucide-react' +import { X, Keyboard, Mic, Lock, Cloud, RefreshCw, Play, Cpu, HelpCircle, TextCursorInput } from 'lucide-react' import { d3roPalette, d3roFontMono, d3roShadow, d3roRadius, d3roTypo } from '@d3ro/ui/theme' import { Led } from '@d3ro/ui/components/ds' import { LicenseTab } from './LicenseTab' import { CloudSyncSection } from './CloudSyncSection' import { STTTab } from './STTTab' +import { InputConsentPanel } from './input-insights/InputConsentPanel' import { KeyBindingField } from './keybinding/KeyBindingField' import { asTranslationKey } from './keybinding/translation-key' import { useI18n, LOCALE_META } from '@d3ro/i18n' import type { Locale } from '@d3ro/i18n' import type { KeyBindingActionGroup } from '@d3ro/core/keybinding' -import { KEYBINDING_ACTIONS } from '@d3ro/core/keybinding' +import { auditKeyBindingMap, KEYBINDING_ACTIONS } from '@d3ro/core/keybinding' import type { ThemeMode, AppConfig, AudioDevice, LLMModel, LLMStatus } from '@d3ro/core/types' +import { useKeyBindingMap } from '../hooks/useKeyBindingMap' interface SettingsModalProps { open: boolean @@ -57,9 +59,10 @@ function TabPanel({ children, value, index }: TabPanelProps): React.ReactElement const ACTION_GROUP_LABEL_KEYS: Readonly> = { voice: 'keybinding.ui.sectionVoice', window: 'keybinding.ui.sectionWindow', + input: 'keybinding.ui.sectionInput', } -const ACTION_GROUP_ORDER: readonly KeyBindingActionGroup[] = ['voice', 'window'] +const ACTION_GROUP_ORDER: readonly KeyBindingActionGroup[] = ['voice', 'window', 'input'] export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalProps): React.ReactElement { const { t, locale, setLocale } = useI18n() @@ -67,6 +70,11 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr const [config, setConfig] = useState>({}) const [loading, setLoading] = useState(true) const [appVersion, setAppVersion] = useState(null) + const keybindingMap = useKeyBindingMap() + const keybindingIssues = useMemo( + () => (keybindingMap === null ? [] : auditKeyBindingMap(keybindingMap)), + [keybindingMap] + ) useEffect(() => { if (open && initialTab !== undefined) { @@ -229,6 +237,7 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr } iconPosition="start" /> + } iconPosition="start" /> } iconPosition="start" /> } iconPosition="start" /> @@ -259,6 +268,45 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr /> + + + {keybindingIssues.length === 0 + ? t('keybinding.ui.auditClear') + : t('keybinding.ui.auditIssues', { count: keybindingIssues.length })} + + {keybindingIssues.slice(0, 3).map((issue) => { + const action = KEYBINDING_ACTIONS.find((entry) => entry.id === issue.actionId) + const actionLabel = action ? t(asTranslationKey(action.labelKey)) : issue.actionId + const conflictLabels = issue.conflictActionIds + .map((actionId) => { + const conflict = KEYBINDING_ACTIONS.find((entry) => entry.id === actionId) + return conflict ? t(asTranslationKey(conflict.labelKey)) : actionId + }) + .join(', ') + return ( + + {issue.kind === 'invalid' + ? t('keybinding.ui.auditInvalid', { + action: actionLabel, + reason: issue.reasonKey ? t(asTranslationKey(issue.reasonKey)) : t('keybinding.ui.auditUnknown') + }) + : t('keybinding.ui.auditConflict', { action: actionLabel, conflicts: conflictLabels })} + + ) + })} + + {ACTION_GROUP_ORDER.map((group) => { const actions = KEYBINDING_ACTIONS.filter((action) => action.group === group) if (actions.length === 0) return null @@ -718,16 +766,20 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr {/* ── 라이선스 탭 ────────────────────────────── */} + + + + {/* ── Cloud Sync 탭 ──────────────────────────── */} - + {/* ── 정보 탭 ──────────────────────────────── */} - + D3RO-VOICE diff --git a/apps/desktop/src/renderer/components/input-insights/BarChart.tsx b/apps/desktop/src/renderer/components/input-insights/BarChart.tsx new file mode 100644 index 0000000..1b4804f --- /dev/null +++ b/apps/desktop/src/renderer/components/input-insights/BarChart.tsx @@ -0,0 +1,163 @@ +// src/renderer/components/input-insights/BarChart.tsx +// 입력 통계용 막대 그래프. +// +// 이전 구현은 flex 비율 + % 높이만 써서, 데이터가 1~2일치일 때 막대 하나가 화면을 +// 가득 채우는 사각형이 됐고 축/라벨도 없어 "그래프"로 읽을 수 없었다. 그래서 +// - 막대 폭을 고정 상한(24px)으로 두어 데이터가 적어도 거대해지지 않게 하고 +// - 값 축(최대값)과 라벨 행을 항상 함께 그리며 +// - 값이 0 뿐이면 그래프 대신 빈 상태 문구를 보여준다. + +import { Box, Typography } from '@mui/material' +import { d3roPalette, d3roRadius, typoSx } from '@d3ro/ui/theme' +import { useI18n } from '@d3ro/i18n' + +export interface BarDatum { + /** x축 라벨 */ + label: string + value: number + /** 툴팁/접근성 텍스트 */ + hint?: string +} + +interface BarChartProps { + data: readonly BarDatum[] + /** 그래프 높이 (px) */ + height?: number + /** 값 포맷 (기본: 정수) */ + formatValue?: (value: number) => string + /** 라벨을 몇 개마다 표시할지 */ + labelEvery?: number + accent?: string +} + +export function BarChart({ + data, + height = 110, + formatValue = (value) => value.toLocaleString(), + labelEvery = 1, + accent = d3roPalette.accent.main +}: BarChartProps): React.ReactElement { + const { t } = useI18n() + const max = Math.max(1, ...data.map((item) => item.value)) + const hasData = data.some((item) => item.value > 0) + + if (data.length === 0 || !hasData) { + return ( + + {t('input.chart.noData')} + + ) + } + + return ( + + + + {t('input.chart.max', { value: formatValue(max) })} + + + + + {data.map((item, index) => ( + 0 ? '5px' : '2px', + height: `${Math.max(2, (item.value / max) * 100)}%`, + borderRadius: `${d3roRadius.xs} ${d3roRadius.xs} 0 0`, + bgcolor: item.value > 0 ? accent : d3roPalette.border.subtle, + opacity: item.value > 0 ? 0.85 : 1, + transition: 'opacity 120ms ease-out', + '&:hover': { opacity: 1 } + }} + /> + ))} + + + + {data.map((item, index) => ( + + {index % labelEvery === 0 ? ( + + {item.label} + + ) : null} + + ))} + + + ) +} + +/** 값 비중을 가로 막대로 보여주는 목록 행 (앱 비중 등). */ +export function ShareBar({ + label, + value, + total, + right +}: { + label: string + value: number + total: number + right?: string +}): React.ReactElement { + const percent = total > 0 ? value / total : 0 + return ( + + + + {label} + + + {right ?? `${Math.round(percent * 100)}%`} + + + + + + + ) +} \ No newline at end of file diff --git a/apps/desktop/src/renderer/components/input-insights/InputConsentPanel.tsx b/apps/desktop/src/renderer/components/input-insights/InputConsentPanel.tsx new file mode 100644 index 0000000..0fce438 --- /dev/null +++ b/apps/desktop/src/renderer/components/input-insights/InputConsentPanel.tsx @@ -0,0 +1,555 @@ +// src/renderer/components/input-insights/InputConsentPanel.tsx +// 입력 인텔리전스 — 동의 · 정책 · 실시간 진단. +// +// 상세 통계(요약/키보드/마우스/앱/문구)는 지식 베이스 화면의 "입력 인사이트" 탭에 있다. +// 이 패널은 "무엇을 수집하고, 언제 멈추고, 어떻게 지우는가" 만 다룬다. + +import { useCallback, useEffect, useState } from 'react' +import { + Box, + Button, + Divider, + FormControlLabel, + MenuItem, + Select, + Switch, + TextField, + Typography +} from '@mui/material' +import { Trash2 } from 'lucide-react' +import { d3roPalette, d3roRadius, d3roTypo, typoSx } from '@d3ro/ui/theme' +import { useI18n } from '@d3ro/i18n' +import type { LLMModel } from '@d3ro/core/types' +import { useInputInsights } from '../../hooks/useInputInsights' + +const OVERLINE_SX = { color: d3roPalette.text.label, letterSpacing: '1.5px' } as const + +const HINT_SX = { + color: d3roPalette.text.secondary, + fontSize: d3roTypo.small.size, + lineHeight: d3roTypo.small.line +} as const + +export function InputConsentPanel(): React.ReactElement { + const { t } = useI18n() + const { telemetry, suggestion, receipt, receiptUnavailable, refresh } = useInputInsights(7) + + const [excludedApps, setExcludedApps] = useState('') + const [triggerDelayMs, setTriggerDelayMs] = useState('') + const [minPrefixChars, setMinPrefixChars] = useState('') + const [requestTimeoutMs, setRequestTimeoutMs] = useState('') + const [models, setModels] = useState([]) + const [feedback, setFeedback] = useState<{ message: string; tone: 'success' | 'error' } | null>(null) + + const enabled = telemetry?.enabled ?? false + const paused = telemetry?.paused ?? false + const suggestionEnabled = suggestion?.enabled ?? false + + useEffect(() => { + if (!telemetry) return + setExcludedApps(telemetry.excludedApps.join(', ')) + }, [telemetry]) + + // 설정 파일에 굳은 실제 값을 그대로 보여준다 (하드코딩 기본값을 표시하면 + // 사용자가 보는 값과 동작이 어긋난다 — 실측: 트리거가 3008ms 로 저장돼 있었다). + useEffect(() => { + if (!suggestion) return + setTriggerDelayMs(String(suggestion.triggerDelayMs)) + setMinPrefixChars(String(suggestion.minPrefixChars)) + setRequestTimeoutMs(String(suggestion.requestTimeoutMs)) + }, [suggestion]) + + useEffect(() => { + let cancelled = false + void window.electronAPI.llm.getModels().then((result) => { + if (!cancelled && result.success) setModels(result.data) + }) + return () => { + cancelled = true + } + }, []) + + const flash = useCallback((message: string, tone: 'success' | 'error' = 'success'): void => { + setFeedback({ message, tone }) + setTimeout(() => setFeedback(null), 2500) + }, []) + + const handleConsent = useCallback( + async (next: boolean): Promise => { + await window.electronAPI.inputTelemetry.setEnabled({ enabled: next }) + await refresh() + }, + [refresh] + ) + + const handlePause = useCallback( + async (next: boolean): Promise => { + await window.electronAPI.inputTelemetry.setPaused({ paused: next }) + await refresh() + }, + [refresh] + ) + + const handleLearn = useCallback( + async (next: boolean): Promise => { + await window.electronAPI.suggestion.setConfig({ learnTypedText: next }) + await refresh() + }, + [refresh] + ) + + const handleSuggestionToggle = useCallback( + async (next: boolean): Promise => { + await window.electronAPI.suggestion.setConfig({ enabled: next }) + await refresh() + }, + [refresh] + ) + + const handleOverlayInteractive = useCallback( + async (next: boolean): Promise => { + await window.electronAPI.suggestion.setConfig({ overlayInteractive: next }) + await refresh() + }, + [refresh] + ) + + const handleModel = useCallback( + async (modelId: string): Promise => { + await window.electronAPI.suggestion.setConfig({ modelId: modelId || null }) + await refresh() + }, + [refresh] + ) + + const handleExcludedAppsBlur = useCallback(async (): Promise => { + const apps = excludedApps + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + const result = await window.electronAPI.suggestion.setConfig({ excludedApps: apps }) + if (result.success) { + await refresh() + flash(t('input.feedback.excludedSaved')) + } else { + flash(t('input.feedback.excludedFailed'), 'error') + } + }, [excludedApps, refresh, t, flash]) + + const handleAddRecommendedApp = useCallback( + async (appName: string): Promise => { + const candidates = excludedApps + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + .concat(appName) + const nextApps = candidates.filter( + (entry, index) => candidates.findIndex((candidate) => candidate.toLowerCase() === entry.toLowerCase()) === index + ) + const result = await window.electronAPI.suggestion.setConfig({ excludedApps: nextApps }) + if (result.success) { + setExcludedApps(nextApps.join(', ')) + await refresh() + flash(t('input.feedback.recommendationSaved', { app: appName })) + } else { + flash(t('input.feedback.recommendationFailed', { app: appName }), 'error') + } + }, + [excludedApps, flash, refresh, t] + ) + + const handleClearAll = useCallback(async (): Promise => { + const result = await window.electronAPI.inputTelemetry.clearAll() + if (result.success) { + await refresh() + flash(t('input.feedback.cleared')) + } else { + flash(t('input.feedback.clearFailed'), 'error') + } + }, [refresh, t, flash]) + + const snapshot = telemetry?.lastSnapshot ?? null + const exclusionRecommendation = telemetry?.exclusionRecommendation ?? null + + return ( + + {/* ── 동의 ─────────────────────────────────────── */} + + + {t('input.consent.title')} + + {t('input.consent.description')} + + + + {t('input.privacy.title')} + + {t('input.privacy.localOnly')} + {t('input.privacy.rawKeys')} + {receipt ? ( + + {t('input.privacy.activityRetention')} + {t('input.privacy.days', { days: receipt.retention.activityDays })} + {t('input.privacy.typingSamplesRetention')} + {t('input.privacy.days', { days: receipt.retention.typingSamplesDays })} + {t('input.privacy.suggestionRetention')} + {t('input.privacy.days', { days: receipt.retention.suggestionDays })} + {t('input.privacy.learnedRetention')} + {t('input.privacy.untilDeleted')} + {t('input.privacy.activityBuckets')} + {receipt.counts.activityBuckets.toLocaleString()} + {t('input.privacy.typingSamples')} + {receipt.counts.typingSamples.toLocaleString()} + {t('input.privacy.personalPhrases')} + {receipt.counts.personalPhrases.toLocaleString()} + {t('input.privacy.suggestions')} + {receipt.counts.suggestions.toLocaleString()} + + ) : receiptUnavailable ? ( + {t('input.privacy.unavailable')} + ) : null} + + + void handleConsent(e.target.checked)} /> + } + label={ + + {t('input.consent.collect')} + + } + /> + + {telemetry?.running ? t('input.consent.running') : t('input.consent.stopped')} + + + {/* 실시간 진단 — "왜 제안이 안 뜨는지" 를 사용자가 직접 볼 수 있게 한다. */} + + + {t('input.diagnostics.title')} + + + {snapshot + ? t('input.diagnostics.app', { app: snapshot.appName ?? t('input.diagnostics.unknownApp') }) + : t('input.diagnostics.noSnapshot')} + + + {snapshot?.isPassword + ? t('input.diagnostics.password') + : snapshot?.editable + ? t('input.diagnostics.readable', { + source: snapshot.textSource, + length: snapshot.textLength + }) + : t('input.diagnostics.notReadable')} + + {snapshot?.composing ? ( + + {t('input.diagnostics.composing')} + + ) : null} + {snapshot?.caretFallback && snapshot.editable ? ( + + {t('input.diagnostics.caretFallback')} + + ) : null} + + + {exclusionRecommendation ? ( + + + {t('input.exclusion.recommendation', { + app: exclusionRecommendation.appName, + samples: exclusionRecommendation.samples, + reason: + exclusionRecommendation.reason === 'repeated-empty' + ? t('input.exclusion.reason.repeated-empty') + : t('input.exclusion.reason.repeated-unreadable') + })} + + + + ) : null} + + void handlePause(e.target.checked)} />} + label={ + + {t('input.consent.pause')} + + } + /> + + void handleLearn(e.target.checked)} + /> + } + label={ + + {t('input.consent.learnText')} + + } + /> + {t('input.consent.learnTextHint')} + + setExcludedApps(e.target.value)} + onBlur={() => void handleExcludedAppsBlur()} + helperText={t('input.consent.excludedAppsHint')} + InputProps={{ sx: { fontSize: d3roTypo.compact.size } }} + /> + + + + {feedback ? ( + + {feedback.message} + + ) : null} + + + + {t('input.insights.statsMovedHint')} + + + + + + {/* ── 제안 정책 ─────────────────────────────────── */} + + + {t('input.suggestion.title')} + + {t('input.suggestion.description')} + + void handleSuggestionToggle(e.target.checked)} + /> + } + label={ + + {t('input.suggestion.enabled')} + + } + /> + + {suggestion?.modelAvailable ? t('input.suggestion.modelReady') : t('input.suggestion.modelMissing')} + + + + setTriggerDelayMs(e.target.value)} + onBlur={() => + void window.electronAPI.suggestion + .setConfig({ triggerDelayMs: Number(triggerDelayMs) }) + .then(() => refresh()) + } + sx={{ width: 150 }} + /> + setMinPrefixChars(e.target.value)} + onBlur={() => + void window.electronAPI.suggestion + .setConfig({ minPrefixChars: Number(minPrefixChars) }) + .then(() => refresh()) + } + sx={{ width: 150 }} + /> + setRequestTimeoutMs(e.target.value)} + onBlur={() => + void window.electronAPI.suggestion + .setConfig({ requestTimeoutMs: Number(requestTimeoutMs) }) + .then(() => refresh()) + } + sx={{ width: 150 }} + /> + + + + void handleOverlayInteractive(e.target.checked)} + /> + } + label={ + + {t('input.suggestion.overlayInteractive')} + + } + /> + + {/* 상태 표시 — 스위치로 두면 설정처럼 보여서 "왜 못 켜지?" 가 된다. */} + + + {t('input.suggestion.onScreenLabel')} + + + + {suggestion?.visible ? t('input.suggestion.onScreen') : t('input.suggestion.offScreen')} + + + {suggestion?.generating ? ( + + {t('input.suggestion.generating')} + + ) : null} + + + {t('input.suggestion.keyHint')} + + + + + {t('input.suggestion.usage', { + requests: suggestion?.requestsToday ?? 0, + budget: suggestion?.dailyBudget ?? 0 + })} + + {suggestion?.lastLatencyMs !== null && suggestion?.lastLatencyMs !== undefined ? ( + + {t('input.suggestion.latency', { ms: suggestion.lastLatencyMs })} + + ) : null} + + + {suggestion?.lastSkipReason ? ( + + {t('input.suggestion.lastSkip', { reason: suggestion.lastSkipReason })} + + ) : null} + + + ) +} diff --git a/apps/desktop/src/renderer/components/input-insights/InputInsightsView.tsx b/apps/desktop/src/renderer/components/input-insights/InputInsightsView.tsx new file mode 100644 index 0000000..32f03ee --- /dev/null +++ b/apps/desktop/src/renderer/components/input-insights/InputInsightsView.tsx @@ -0,0 +1,701 @@ +// src/renderer/components/input-insights/InputInsightsView.tsx +// 입력 인사이트 — 지식 베이스 안에 들어가는 상세 통계 화면. +// +// 종류별 inner tab 으로 나눈다: 요약 / 키보드 / 마우스 / 앱 / 문구·제안. +// 모든 수치는 수집된 로컬 집계(input_activity)와 제안 이력(suggestions)에서 나온다. + +import { useCallback, useEffect, useState } from 'react' +import { + Box, + Button, + Divider, + IconButton, + Tab, + Tabs, + TextField, + Typography +} from '@mui/material' +import { Trash2 } from 'lucide-react' +import { d3roPalette, d3roRadius, d3roTypo, typoSx } from '@d3ro/ui/theme' +import { useI18n } from '@d3ro/i18n' +import { BarChart, ShareBar, type BarDatum } from './BarChart' +import { useInputInsights } from '../../hooks/useInputInsights' +import type { + PersonalGraphQuery, + PersonalGraphStats +} from '@d3ro/core/personal-graph' + +const OVERLINE_SX = { color: d3roPalette.text.label, letterSpacing: '1.5px' } as const + +const HINT_SX = { + color: d3roPalette.text.secondary, + fontSize: d3roTypo.small.size, + lineHeight: d3roTypo.small.line +} as const + +interface TileProps { + label: string + value: string + unit?: string + emphasis?: boolean +} + +function StatTile({ label, value, unit, emphasis = false }: TileProps): React.ReactElement { + return ( + + {label} + + {value} + {unit ? ( + + {unit} + + ) : null} + + + ) +} + +function TileRow({ children }: { children: React.ReactNode }): React.ReactElement { + return {children} +} + +function Section({ title, children }: { title: string; children: React.ReactNode }): React.ReactElement { + return ( + + + {title} + + {children} + + ) +} + +interface SuggestionHistoryRow { + id: string + appName: string | null + prefixText: string + suggestionText: string + model: string | null + latencyMs: number | null + accepted: boolean + createdAt: number +} + +export function InputInsightsView(): React.ReactElement { + const { t, formatRelativeDate } = useI18n() + const [days, setDays] = useState(7) + const [tab, setTab] = useState(0) + const [history, setHistory] = useState([]) + const [graph, setGraph] = useState(null) + const [graphQuery, setGraphQuery] = useState('') + const [graphResult, setGraphResult] = useState(null) + const { telemetry, summary, phrases, refresh } = useInputInsights(days) + + const loadHistory = useCallback(async (): Promise => { + const result = await window.electronAPI.suggestion.getHistory({ limit: 20 }) + if (result.success) setHistory(result.data) + }, []) + + useEffect(() => { + void loadHistory() + }, [loadHistory]) + + useEffect(() => { + let cancelled = false + void window.electronAPI.inputTelemetry.getGraph().then((result) => { + if (!cancelled && result.success) setGraph(result.data) + }) + return () => { + cancelled = true + } + }, [tab, phrases.length]) + + const runGraphQuery = useCallback(async (): Promise => { + const result = await window.electronAPI.inputTelemetry.queryGraph({ text: graphQuery }) + if (result.success) setGraphResult(result.data) + }, [graphQuery]) + + const handleDeletePhrase = useCallback( + async (id: string): Promise => { + await window.electronAPI.inputTelemetry.deletePhrase({ id }) + await refresh() + }, + [refresh] + ) + + const dailyKeys: BarDatum[] = (summary?.daily ?? []).map((day) => ({ + label: day.date.slice(5), + value: day.keystrokes, + hint: `${day.date} · ${day.keystrokes.toLocaleString()}` + })) + + const dailyWords: BarDatum[] = (summary?.daily ?? []).map((day) => ({ + label: day.date.slice(5), + value: day.words, + hint: `${day.date} · ${day.words.toLocaleString()}` + })) + + const dailyDistance: BarDatum[] = (summary?.daily ?? []).map((day) => ({ + label: day.date.slice(5), + value: Math.round(day.mouseDistancePx / 96 / 0.0254), + hint: `${day.date} · ${Math.round(day.mouseDistancePx)}px` + })) + + const hourlyKeys: BarDatum[] = (summary?.hourly ?? []).map((hour) => ({ + label: String(hour.hour), + value: hour.keystrokes, + hint: `${hour.hour}:00 · ${hour.keystrokes.toLocaleString()}` + })) + + const labelEvery = days > 14 ? 3 : days > 7 ? 2 : 1 + const totals = summary?.totals + const averages = summary?.averages + const suggestions = summary?.suggestions + const friction = summary?.friction + const flowWindows = summary?.flowWindows ?? [] + const suggestionApps = summary?.suggestionApps ?? [] + const frictionBandLabel = + friction?.band === 'high' + ? t('input.friction.band.high') + : friction?.band === 'watch' + ? t('input.friction.band.watch') + : t('input.friction.band.steady') + const topAppTotal = (summary?.topApps ?? []).reduce((sum, app) => sum + app.keystrokes, 0) + + if (!telemetry?.enabled) { + return ( + + {t('input.insights.disabledHint')} + + ) + } + + return ( + + + + {t('input.insights.rangeLabel')} + + {[7, 14, 30].map((value) => ( + setDays(value)} + sx={{ + cursor: 'pointer', + px: 1, + py: 0.25, + borderRadius: d3roRadius.xs, + border: `1px solid ${value === days ? d3roPalette.accent.main : d3roPalette.border.subtle}`, + bgcolor: value === days ? d3roPalette.accent.dim : 'transparent', + color: value === days ? d3roPalette.accent.main : d3roPalette.text.inactive, + font: 'inherit', + ...typoSx('micro') + }} + > + {t('input.insights.rangeDays', { days: value })} + + ))} + + {t('input.insights.headerSummary', { + days: summary?.days ?? days, + keys: totals?.keystrokes ?? 0, + apps: summary?.topApps.length ?? 0 + })} + + + + setTab(next)} + variant="scrollable" + scrollButtons={false} + sx={{ + minHeight: 36, + borderBottom: `1px solid ${d3roPalette.border.subtle}`, + '& .MuiTab-root': { + minHeight: 36, + py: 0.5, + textTransform: 'none', + color: d3roPalette.text.inactive, + fontSize: d3roTypo.label.size, + '&.Mui-selected': { color: d3roPalette.accent.main } as const + }, + '& .MuiTabs-indicator': { bgcolor: d3roPalette.accent.main, height: 2 } + }} + > + + + + + + + + + {/* ── 요약 ─────────────────────────────────────── */} + {tab === 0 ? ( + + + + + + + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+ + ) : null} + + {/* ── 키보드 ───────────────────────────────────── */} + {tab === 1 ? ( + + + + + + + + + + +
+ +
+ +
+ + + + {flowWindows.length === 0 ? ( + {t('input.flow.empty')} + ) : ( + flowWindows.map((window) => ( + + + {t('input.flow.hour', { hour: window.hour })} + + + {t('input.flow.window', { + score: window.score, + minutes: Math.round(window.activeMinutes), + friction: Math.round(window.frictionRate * 100) + })} + + + )) + )} +
+ +
+ +
+ + + {t('input.insights.topHoursHint', { + hours: (summary?.topHours ?? []) + .slice(0, 3) + .map((hour) => `${hour.hour}시`) + .join(', ') + })} + + + ) : null} + + {/* ── 마우스 ───────────────────────────────────── */} + {tab === 2 ? ( + + + + + + + + +
+ +
+ + + {t('input.insights.mouseNote')} + + + ) : null} + + {/* ── 앱 ──────────────────────────────────────── */} + {tab === 3 ? ( + + {t('input.insights.appsHint')} + {(summary?.topApps ?? []).length === 0 ? ( + {t('input.insights.empty')} + ) : ( + (summary?.topApps ?? []).map((app) => ( + + )) + )} + +
+ {suggestionApps.length === 0 ? ( + {t('input.appQuality.empty')} + ) : ( + suggestionApps.map((app) => ( + + + {app.appName} + + + {t('input.appQuality.row', { + accepted: app.accepted, + total: app.total, + rate: Math.round(app.acceptRate * 100), + latency: app.avgLatencyMs ?? '—' + })} + + + )) + )} +
+ + ) : null} + + {/* ── 개인 그래프 ──────────────────────────────── */} + {tab === 5 ? ( + + {t('input.graph.description')} + + + + + + + + + ) => setGraphQuery(event.target.value)} + onKeyDown={(event: React.KeyboardEvent) => { + if (event.key === 'Enter') void runGraphQuery() + }} + placeholder={t('input.graph.searchPlaceholder')} + label={t('input.graph.searchLabel')} + /> + + + + {graphResult && graphResult.anchors.length > 0 ? ( +
+ {graphResult.anchors.map((anchor) => ( + + + {anchor.text} + + {graphResult.neighbors + .slice(0, 4) + .map((neighbor) => ( + + → {neighbor.text} ({neighbor.kind === 'follows' ? t('input.graph.kindFollows') : t('input.graph.kindShares')} · {neighbor.weight}) + + ))} + + ))} +
+ ) : null} + +
+ {(graph?.topEdges ?? []).length === 0 ? ( + {t('input.graph.empty')} + ) : ( + (graph?.topEdges ?? []).map((edge) => ( + + + {edge.kind === 'follows' ? t('input.graph.kindFollows') : t('input.graph.kindShares')} + + + {edge.from} → {edge.to} + + + {edge.weight}× + + + )) + )} +
+ +
+ {(graph?.recentNodes ?? []).slice(0, 8).map((node) => ( + + + {node.text} + + + {node.count}× · {node.appName ?? '—'} + + + ))} +
+ + ) : null} + + {/* ── 문구 · 제안 ─────────────────────────────── */} + {tab === 4 ? ( + +
+ + + + + + +
+ +
+ {history.length === 0 ? ( + {t('input.insights.noSuggestions')} + ) : ( + history.map((entry) => ( + + + + {entry.accepted ? t('input.insights.accepted') : t('input.insights.notAccepted')} + + + {entry.appName ?? '—'} + + + {entry.latencyMs === null ? '' : `${entry.latencyMs}ms`} + + + + {entry.prefixText.slice(-40)} → {entry.suggestionText} + + + )) + )} +
+ + + +
+ {t('input.phrases.description')} + {phrases.length === 0 ? ( + {t('input.phrases.empty')} + ) : ( + phrases.slice(0, 40).map((phrase) => ( + + + {phrase.phrase} + + + {t('input.phrases.metadata', { + source: phrase.source, + count: phrase.count, + app: phrase.appName ?? t('input.phrases.appUnknown') + })} + + void handleDeletePhrase(phrase.id)} + sx={{ flexShrink: 0 }} + > + + + + )) + )} +
+ + + {t('input.insights.samplesNote', { count: summary?.sampleCount ?? 0 })}{' '} + {t('input.insights.collectedAt', { at: formatRelativeDate(telemetry.lastSnapshotAt || Date.now()) })} + + + ) : null} + + ) +} diff --git a/apps/desktop/src/renderer/hooks/useInputInsights.ts b/apps/desktop/src/renderer/hooks/useInputInsights.ts new file mode 100644 index 0000000..22f7eea --- /dev/null +++ b/apps/desktop/src/renderer/hooks/useInputInsights.ts @@ -0,0 +1,83 @@ +// src/renderer/hooks/useInputInsights.ts +// 입력 텔레메트리/제안 상태 + 주간 인사이트 로딩 훅. +// +// 동의(수집 여부)와는 무관하게 **설정값과 집계만** 읽는다. 수집이 꺼져 있으면 +// 집계는 비어 있고 UI 는 "꺼" 상태를 보여준다. + +import { useCallback, useEffect, useState } from 'react' +import type { + InputInsightsSummary, + InputPrivacyReceipt, + InputTelemetryState, + PersonalPhrase, + SuggestionState +} from '@d3ro/core/input-intelligence' + +export interface UseInputInsightsResult { + telemetry: InputTelemetryState | null + suggestion: SuggestionState | null + summary: InputInsightsSummary | null + receipt: InputPrivacyReceipt | null + receiptUnavailable: boolean + phrases: PersonalPhrase[] + loading: boolean + refresh: (days?: number) => Promise +} + +export function useInputInsights(days = 7): UseInputInsightsResult { + const [telemetry, setTelemetry] = useState(null) + const [suggestion, setSuggestion] = useState(null) + const [summary, setSummary] = useState(null) + const [receipt, setReceipt] = useState(null) + const [receiptUnavailable, setReceiptUnavailable] = useState(false) + const [phrases, setPhrases] = useState([]) + const [loading, setLoading] = useState(true) + + const refresh = useCallback( + async (rangeDays = days): Promise => { + setLoading(true) + try { + const [stateResult, suggestionResult, summaryResult, phrasesResult, receiptResult] = await Promise.allSettled([ + window.electronAPI.inputTelemetry.getState(), + window.electronAPI.suggestion.getState(), + window.electronAPI.inputTelemetry.getSummary({ days: rangeDays }), + window.electronAPI.inputTelemetry.getPhrases({ limit: 50 }), + window.electronAPI.inputTelemetry.getPrivacyReceipt() + ]) + if (stateResult.status === 'fulfilled' && stateResult.value.success) setTelemetry(stateResult.value.data) + if (suggestionResult.status === 'fulfilled' && suggestionResult.value.success) setSuggestion(suggestionResult.value.data) + if (summaryResult.status === 'fulfilled' && summaryResult.value.success) setSummary(summaryResult.value.data) + if (phrasesResult.status === 'fulfilled' && phrasesResult.value.success) setPhrases(phrasesResult.value.data) + if (receiptResult.status === 'fulfilled' && receiptResult.value.success) { + setReceipt(receiptResult.value.data) + setReceiptUnavailable(false) + } else { + setReceipt(null) + setReceiptUnavailable(true) + } + } finally { + setLoading(false) + } + }, + [days] + ) + + useEffect(() => { + void refresh(days) + }, [refresh, days]) + + useEffect(() => { + const unsubTelemetry = window.electronAPI.inputTelemetry.onStateChanged((state) => { + setTelemetry(state) + }) + const unsubSuggestion = window.electronAPI.suggestion.onStateChanged((state) => { + setSuggestion(state) + }) + return () => { + unsubTelemetry() + unsubSuggestion() + } + }, []) + + return { telemetry, suggestion, summary, receipt, receiptUnavailable, phrases, loading, refresh } +} diff --git a/apps/desktop/src/renderer/pages/DashboardPage.tsx b/apps/desktop/src/renderer/pages/DashboardPage.tsx index 46d454f..805d799 100644 --- a/apps/desktop/src/renderer/pages/DashboardPage.tsx +++ b/apps/desktop/src/renderer/pages/DashboardPage.tsx @@ -39,6 +39,7 @@ import { formatRecordingTime, formatRecordingTimeUnit, formatNumber, getDateKey import { BindingKeycaps } from '../components/keybinding/Keycap' import { useKeyBindingMap } from '../hooks/useKeyBindingMap' import { FileDropZone } from '../components/FileDropZone' +import { useInputInsights } from '../hooks/useInputInsights' import type { StatsSummary, HistoryEntry, CaptionState, LicenseTier, UsageQuota } from '@d3ro/core/types' /** 카드 섹션 헤더 (제목 + 우측 액세서리) — 헤어라인 하단 구분 */ @@ -103,6 +104,7 @@ export function DashboardPage(): React.ReactElement { const [llmModel, setLlmModel] = useState(null) const bindingMap = useKeyBindingMap() const dictationBinding = bindingMap?.dictation[0] ?? null + const inputInsights = useInputInsights(7) const [captionState, setCaptionState] = useState('inactive') const [audioLevel, setAudioLevel] = useState(0) const audioDecayRef = useRef | null>(null) @@ -642,6 +644,57 @@ export function DashboardPage(): React.ReactElement { + {/* ── 5b. 입력 인사이트 (주간) ──────────────────── */} + {inputInsights.telemetry?.enabled ? ( + + } + right={ + + {t('input.insights.keystrokes')} {formatNumber(inputInsights.summary?.totals.keystrokes ?? 0)} + + } + /> + + {[ + { + label: t('input.insights.clicks'), + value: formatNumber(inputInsights.summary?.totals.clicks ?? 0), + }, + { + label: t('input.insights.words'), + value: formatNumber(inputInsights.summary?.totals.words ?? 0), + }, + { + label: t('input.insights.sentences'), + value: formatNumber(inputInsights.summary?.totals.sentences ?? 0), + }, + { + label: t('input.insights.mouseDistance'), + value: `${inputInsights.summary?.averages.mouseDistanceMeters ?? 0}${t('input.insights.perDayMeters')}`, + }, + { + label: t('input.insights.phrases'), + value: formatNumber(inputInsights.summary?.phraseCount ?? 0), + }, + ].map((item) => ( + + + {item.label} + + + {item.value} + + + ))} + + + ) : null} + {/* ── 6. 파일 전사 드롭존 ──────────────────────────── */} diff --git a/apps/desktop/src/renderer/pages/KnowledgeBasePage.tsx b/apps/desktop/src/renderer/pages/KnowledgeBasePage.tsx index 1bc4639..295779a 100644 --- a/apps/desktop/src/renderer/pages/KnowledgeBasePage.tsx +++ b/apps/desktop/src/renderer/pages/KnowledgeBasePage.tsx @@ -2,16 +2,21 @@ // Local RAG Knowledge Base & Semantic Memory Intelligence import React, { useState, useEffect, useCallback, useRef } from 'react' -import { Box, TextField, IconButton, Tooltip, LinearProgress } from '@mui/material' +import { Box, TextField, IconButton, Tooltip, LinearProgress, Tab, Tabs } from '@mui/material' import { Plus, Trash2, RefreshCw, Search, Send, FileText, Database, Sparkles } from 'lucide-react' import { MetalCard, PhosphorText, Led, PhysicalButton, DoubleBezelCard, TactileBadge } from '@d3ro/ui/components/ds' import { PageHeader, EmptyStateCard } from '../components/shared' +import { InputInsightsView } from '../components/input-insights/InputInsightsView' import { isImeComposingEvent } from '../utils/keyboard' import { d3roPalette, d3roFontSans, d3roTypo, d3roRadius, d3roShadow } from '@d3ro/ui/theme' import { useI18n } from '@d3ro/i18n' import type { RAGDocument, RAGQueryResult, RAGIndexProgress } from '@d3ro/core/types' +type KnowledgeView = 'kb' | 'insights' + export function KnowledgeBasePage(): React.ReactElement { + // 이 화면은 두 축을 가진다: 수집된 지식 문서(RAG)와 자동 수집된 입력 데이터 통계. + const [view, setView] = useState('kb') const { t } = useI18n() const [documents, setDocuments] = useState([]) const [loading, setLoading] = useState(true) @@ -128,6 +133,32 @@ export function KnowledgeBasePage(): React.ReactElement { } /> + setView(next)} + sx={{ + mb: 2.5, + minHeight: 38, + borderBottom: `1px solid ${d3roPalette.border.subtle}`, + '& .MuiTab-root': { + minHeight: 38, + py: 0.5, + textTransform: 'none', + color: d3roPalette.text.inactive, + fontSize: d3roTypo.label.size, + '&.Mui-selected': { color: d3roPalette.accent.main } + }, + '& .MuiTabs-indicator': { bgcolor: d3roPalette.accent.main, height: 2 } + }} + > + + + + + {view === 'insights' ? ( + + ) : ( + <> {/* Indexing Progress Card */} {indexProgress && ( @@ -411,6 +442,8 @@ export function KnowledgeBasePage(): React.ReactElement { )} + + )} ) } diff --git a/apps/desktop/src/renderer/popups/suggestion-overlay/index.html b/apps/desktop/src/renderer/popups/suggestion-overlay/index.html new file mode 100644 index 0000000..aa21fe7 --- /dev/null +++ b/apps/desktop/src/renderer/popups/suggestion-overlay/index.html @@ -0,0 +1,31 @@ + + + + + + + Suggestion + + +
+
+
+ + +
+
+
+
+
+
+ + + + diff --git a/apps/desktop/src/renderer/popups/suggestion-overlay/script.js b/apps/desktop/src/renderer/popups/suggestion-overlay/script.js new file mode 100644 index 0000000..b9b9e87 --- /dev/null +++ b/apps/desktop/src/renderer/popups/suggestion-overlay/script.js @@ -0,0 +1,200 @@ +// Suggestion Overlay 팝업 스크립트 +// 입력 인텔리전스: 다음 문장 제안(ghost text)을 커서 옆에 보여준다. +// +// 상태 세 가지를 화면에 드러낸다: +// warmingUp - 모델을 메모리에 올리는 중 (스피너 + "준비 중") +// generating - 토큰이 만들어지는 중 (스피너 + 도착한 부분 텍스트) +// 후보 도착 - 최대 5개, 목록은 스크롤 가능 +// +// 이 창은 focusable:false 다. 그래서 키 입력(수락/순환/닫기)은 전역 키바인딩이 +// 메인에서 처리하고, 이 파일은 마우스 클릭만 처리한다. + +;(function () { + 'use strict' + + var candidatesContainer = document.getElementById('candidates') + var provenanceContainer = document.getElementById('provenance') + var hintsContainer = document.getElementById('hints') + var statusRow = document.getElementById('status') + var statusText = document.getElementById('statusText') + var panel = document.getElementById('panel') + var closeButton = document.getElementById('close') + + /** @type {string[]} */ + var candidates = [] + var activeIndex = 0 + var generating = false + var warmingUp = false + var partialText = null + var provenance = null + var i18nStrings = {} + var generatingSince = 0 + var tickTimer = null + + function render() { + candidatesContainer.textContent = '' + + var busy = warmingUp || generating + if (statusRow) statusRow.hidden = !busy + if (statusText) { + if (warmingUp) { + statusText.textContent = i18nStrings.suggestionWarming || '...' + } else if (generating) { + // 경과 시간을 보여준다 — 모델이 바쁘면 몇 초 걸리는지 보이는 편이 덜 답답하다. + var seconds = generatingSince + ? Math.max(1, Math.round((Date.now() - generatingSince) / 1000)) + : 1 + var template = i18nStrings.suggestionGenerating || i18nStrings.suggestionLoading || '...' + statusText.textContent = template.replace('{{seconds}}', String(seconds)) + } else { + statusText.textContent = '' + } + } + + if (candidates.length === 0) { + // 스트리밍 중이면 도착한 부분 텍스트를 그대로 보여준다 ("계속 생성되는" 느낌). + if (partialText) { + var streaming = document.createElement('div') + streaming.className = 'suggestion-item streaming' + streaming.textContent = partialText + candidatesContainer.appendChild(streaming) + } + renderHints() + renderProvenance() + return + } + + for (var i = 0; i < candidates.length; i++) { + var item = document.createElement('button') + item.type = 'button' + item.className = i === activeIndex ? 'suggestion-item active' : 'suggestion-item' + item.setAttribute('data-index', String(i)) + item.textContent = candidates[i] + item.addEventListener('click', onItemClick) + candidatesContainer.appendChild(item) + } + + renderHints() + renderProvenance() + } + + function renderProvenance() { + if (!provenanceContainer) return + provenanceContainer.textContent = '' + if (!provenance) return + + var parts = [] + var sourceLabel = provenance.mode === 'local-memory' + ? i18nStrings.suggestionSourceMemory + : i18nStrings.suggestionSourceModel + if (sourceLabel) parts.push(sourceLabel) + var counts = [ + ['continuationCount', 'suggestionContinuations'], + ['relatedCount', 'suggestionRelated'], + ['phraseCount', 'suggestionPhrases'], + ['appPhraseCount', 'suggestionAppPhrases'] + ] + for (var i = 0; i < counts.length; i++) { + var count = provenance[counts[i][0]] || 0 + var countLabel = i18nStrings[counts[i][1]] + if (count > 0 && countLabel) parts.push(countLabel + ' ' + count) + } + provenanceContainer.textContent = parts.join(' · ') + } + + function renderHints() { + hintsContainer.textContent = '' + if (candidates.length <= 1 && !generating) return + + var hint = document.createElement('span') + hint.className = 'hint' + if (candidates.length > 0) { + var nextLabel = i18nStrings.suggestionHintNext || 'Next' + hint.textContent = nextLabel + ' ' + (activeIndex + 1) + '/' + candidates.length + } else { + hint.textContent = i18nStrings.suggestionHintGenerating || '…' + } + hintsContainer.appendChild(hint) + } + + function onItemClick(event) { + var target = event.currentTarget + var index = Number(target.getAttribute('data-index')) + if (!window.popupAPI) return + window.popupAPI.send('suggestionPopup:accept', index) + } + + function applyPayload(payload) { + if (!payload) return + if (payload._i18n) i18nStrings = payload._i18n + candidates = (payload.candidates || []).map(function (candidate) { + return candidate && candidate.text ? candidate.text : String(candidate) + }) + activeIndex = payload.activeIndex || 0 + if (payload.generating !== undefined) generating = payload.generating === true + if (payload.warmingUp !== undefined) warmingUp = payload.warmingUp === true + if (payload.partialText !== undefined) partialText = payload.partialText || null + if (payload.provenance !== undefined) provenance = payload.provenance || null + } + + function startTick() { + if (tickTimer) return + tickTimer = setInterval(function () { + if (!generating && !warmingUp) { + clearInterval(tickTimer) + tickTimer = null + generatingSince = 0 + return + } + render() + }, 500) + } + + function handleShow(payload) { + applyPayload(payload) + if (payload && payload.generating) generatingSince = Date.now() + startTick() + if (panel) panel.classList.add('visible') + render() + } + + function handleUpdate(payload) { + var wasGenerating = generating + applyPayload(payload) + if (generating && !wasGenerating) generatingSince = Date.now() + startTick() + render() + } + + function handleHide() { + if (panel) panel.classList.remove('visible') + if (tickTimer) { + clearInterval(tickTimer) + tickTimer = null + } + generatingSince = 0 + generating = false + warmingUp = false + partialText = null + provenance = null + candidates = [] + activeIndex = 0 + candidatesContainer.textContent = '' + if (provenanceContainer) provenanceContainer.textContent = '' + hintsContainer.textContent = '' + } + + if (closeButton) { + // 마우스로도 닫을 수 있어야 한다 (키보드만 있으면 불편하다는 피드백). + closeButton.addEventListener('click', function () { + handleHide() + if (window.popupAPI) window.popupAPI.send('suggestionPopup:dismiss') + }) + } + + if (window.popupAPI) { + window.popupAPI.on('suggestionPopup:show', handleShow) + window.popupAPI.on('suggestionPopup:update', handleUpdate) + window.popupAPI.on('suggestionPopup:hide', handleHide) + } +})() diff --git a/apps/desktop/src/renderer/popups/suggestion-overlay/style.css b/apps/desktop/src/renderer/popups/suggestion-overlay/style.css new file mode 100644 index 0000000..18d0b42 --- /dev/null +++ b/apps/desktop/src/renderer/popups/suggestion-overlay/style.css @@ -0,0 +1,208 @@ +/* Suggestion Overlay — 입력 인텔리전스 ghost text + * + * 팝업 스타일은 injectPopupTheme() 의 CSS 변수(--d3-*)로 테마를 따라간다. + * 토큰이 주입되지 않는 상황(개발 초기 로드)을 위해 :root 폴백을 둔다. + */ + +:root { + --d3-bg-result: #111a30; + --d3-border-result: rgba(148, 180, 255, 0.12); + --d3-text-result: #eef2fb; + --d3-text-secondary: #93a4c8; + --d3-accent-main: #3b82f6; + --d3-accent-dim: rgba(59, 130, 246, 0.14); + --d3-shadow-popup: 0 8px 32px rgba(3, 7, 18, 0.5); +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +html, +body { + background: transparent; + overflow: hidden; + user-select: none; + -webkit-app-region: no-drag; + font-family: 'Pretendard Variable', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; +} + +#root { + width: 100%; + height: 100%; + padding: 4px; +} + +.suggestion-panel { + display: flex; + flex-direction: column; + gap: 6px; + padding: 8px; + border-radius: 10px; + background: var(--d3-bg-result); + border: 1px solid var(--d3-border-result); + box-shadow: var(--d3-shadow-popup); + opacity: 0; + transform: translateY(-2px); + transition: opacity 90ms ease-out, transform 90ms ease-out; +} + +.suggestion-panel.visible { + opacity: 1; + transform: translateY(0); +} + +/* ── 헤더 (상태 + 닫기) ─────────────────────────────── */ +.header { + display: flex; + align-items: center; + gap: 8px; + min-height: 18px; +} + +.close { + margin-left: auto; + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + padding: 0; + border: 0; + border-radius: 5px; + background: transparent; + color: var(--d3-text-secondary); + cursor: pointer; + transition: background 90ms ease-out, color 90ms ease-out; +} + +.close:hover { + background: var(--d3-accent-dim); + color: var(--d3-text-result); +} + +/* ── 상태 줄 (워밍업 / 생성 중) ─────────────────────── */ +.status { + display: flex; + align-items: center; + gap: 8px; + padding: 2px 4px; +} + +.spinner { + width: 11px; + height: 11px; + flex: 0 0 auto; + border-radius: 50%; + border: 2px solid var(--d3-accent-dim); + border-top-color: var(--d3-accent-main); + animation: d3-spin 720ms linear infinite; +} + +@keyframes d3-spin { + to { + transform: rotate(360deg); + } +} + +.status-text { + color: var(--d3-text-secondary); + font-size: 12px; + font-style: italic; +} + +/* ── 후보 목록 (최대 5개, 스크롤) ───────────────────── */ +.candidates { + display: flex; + flex-direction: column; + gap: 2px; + max-height: 176px; + overflow-y: auto; + overscroll-behavior: contain; +} + +.candidates::-webkit-scrollbar { + width: 8px; +} + +.candidates::-webkit-scrollbar-thumb { + background: var(--d3-accent-dim); + border-radius: 4px; +} + +.candidates::-webkit-scrollbar-track { + background: transparent; +} + +.suggestion-item { + display: block; + width: 100%; + padding: 6px 8px; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--d3-text-result); + font: inherit; + font-size: 14px; + line-height: 1.45; + text-align: left; + cursor: default; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.suggestion-item.active { + background: var(--d3-accent-dim); + color: var(--d3-text-result); +} + +.suggestion-item.loading { + color: var(--d3-text-secondary); + font-style: italic; +} + +/* 스트리밍 중 — 도착한 만큼 보여주고 커서를 붙인다 */ +.suggestion-item.streaming { + color: var(--d3-text-result); + opacity: 0.85; + white-space: normal; +} + +.suggestion-item.streaming::after { + content: '\258C'; + animation: d3-caret-blink 1s steps(1) infinite; + color: var(--d3-accent-main); +} + +.provenance { + min-height: 15px; + padding: 0 8px; + color: var(--d3-text-secondary); + font-size: 11px; + line-height: 1.35; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +@keyframes d3-caret-blink { + 50% { + opacity: 0; + } +} + +.hints { + display: flex; + align-items: center; + gap: 8px; + padding: 0 8px 2px; + color: var(--d3-text-secondary); + font-size: 11px; +} + +.hint { + white-space: nowrap; +} diff --git a/apps/desktop/tests/main/ipc/suggestion-handlers.test.ts b/apps/desktop/tests/main/ipc/suggestion-handlers.test.ts new file mode 100644 index 0000000..07c461c --- /dev/null +++ b/apps/desktop/tests/main/ipc/suggestion-handlers.test.ts @@ -0,0 +1,80 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' + +const handlers = vi.hoisted( + () => new Map Promise>() +) +const eventHandlers = vi.hoisted(() => new Map void>()) +const configSet = vi.hoisted(() => vi.fn()) +const suggestion = vi.hoisted(() => ({ + setEnabled: vi.fn(), + applyConfig: vi.fn(), + dismiss: vi.fn(), + getState: vi.fn(() => ({ enabled: true })) +})) +const hideSuggestionOverlay = vi.hoisted(() => vi.fn()) + +vi.mock('electron', () => ({ + ipcMain: { + handle: vi.fn((channel: string, handler: (event: unknown, params: unknown) => Promise) => { + handlers.set(channel, handler) + }), + on: vi.fn((channel: string, handler: (...args: unknown[]) => void) => { + eventHandlers.set(channel, handler) + }) + } +})) + +vi.mock('../../../src/main/services/ConfigService', () => ({ configSet })) +vi.mock('../../../src/main/services/SuggestionService', () => ({ + getSuggestionService: () => suggestion +})) +vi.mock('../../../src/main/windows/WindowManager', () => ({ + applySuggestionOverlayConfig: vi.fn(), + hideSuggestionOverlay +})) + +beforeEach(async () => { + vi.resetModules() + vi.clearAllMocks() + handlers.clear() + eventHandlers.clear() + const mod = await import('../../../src/main/ipc/suggestion-handlers') + mod.registerSuggestionHandlers() +}) + +describe('SUGGESTION.SET_CONFIG', () => { + it('명시적으로 활성화할 때 서비스의 warm-up 경로를 사용한다', async () => { + const handler = handlers.get(IPC_CHANNELS.SUGGESTION.SET_CONFIG) + if (!handler) throw new Error('suggestion config handler was not registered') + + await handler({}, { enabled: true }) + + expect(suggestion.setEnabled).toHaveBeenCalledWith(true) + expect(configSet).not.toHaveBeenCalledWith('suggestionEnabled', true) + }) + + it('분당 요청 상한을 12로 강제한다', async () => { + const handler = handlers.get(IPC_CHANNELS.SUGGESTION.SET_CONFIG) + if (!handler) throw new Error('suggestion config handler was not registered') + + await handler({}, { maxRequestsPerMinute: 99 }) + + expect(configSet).toHaveBeenCalledWith('suggestionMaxRequestsPerMinute', 12) + }) +}) + +describe('POPUP_SUGGESTION.DISMISS', () => { + it('서비스 이벤트를 기다리지 않고 창을 숨긴 뒤 제안을 취소한다', () => { + const handler = eventHandlers.get(IPC_CHANNELS.POPUP_SUGGESTION.DISMISS) + if (!handler) throw new Error('suggestion dismiss handler was not registered') + + handler({}) + + expect(hideSuggestionOverlay).toHaveBeenCalledTimes(1) + expect(suggestion.dismiss).toHaveBeenCalledWith('dismissed') + expect(hideSuggestionOverlay.mock.invocationCallOrder[0]).toBeLessThan( + suggestion.dismiss.mock.invocationCallOrder[0] + ) + }) +}) diff --git a/apps/desktop/tests/main/services/ConfigService.test.ts b/apps/desktop/tests/main/services/ConfigService.test.ts new file mode 100644 index 0000000..113e1d4 --- /dev/null +++ b/apps/desktop/tests/main/services/ConfigService.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +describe('ConfigService suggestion tuning migration', () => { + afterEach(() => { + vi.doUnmock('electron-store') + vi.resetModules() + }) + + it('revision 3은 새 delay와 분당 상한만 바꾸고 기존 개인 설정을 보존한다', async () => { + const savedBindings = { + 'suggestion-accept': [ + { device: 'keyboard' as const, code: 65, ctrl: true, alt: false, shift: false, meta: false } + ] + } + const persisted = { + suggestionTuningRevision: 3, + suggestionTriggerDelayMs: 300, + suggestionMaxRequestsPerMinute: 12, + suggestionMinPrefixChars: 17, + suggestionDailyBudget: 777, + suggestionRequestTimeoutMs: 23000, + keyBindings: savedBindings + } + + class TestStore> { + store: T + + constructor(options: { defaults: T }) { + this.store = { ...options.defaults, ...persisted } as T + } + + get(key: K): T[K] { + return this.store[key] + } + + set(key: K, value: T[K]): void { + this.store[key] = value + } + + delete(key: string): void { + delete this.store[key as keyof T] + } + } + + vi.doMock('electron-store', () => ({ default: TestStore })) + const { configGet, initConfigService, resetInMemoryConfig } = await import( + '../../../src/main/services/ConfigService' + ) + await initConfigService() + + expect(configGet('suggestionTuningRevision')).toBe(4) + expect(configGet('suggestionTriggerDelayMs')).toBe(600) + expect(configGet('suggestionMaxRequestsPerMinute')).toBe(6) + expect(configGet('suggestionMinPrefixChars')).toBe(17) + expect(configGet('suggestionDailyBudget')).toBe(777) + expect(configGet('suggestionRequestTimeoutMs')).toBe(23000) + expect(configGet('keyBindings')['suggestion-accept']).toEqual(savedBindings['suggestion-accept']) + + resetInMemoryConfig() + }) +}) diff --git a/apps/desktop/tests/main/services/SuggestionService.test.ts b/apps/desktop/tests/main/services/SuggestionService.test.ts new file mode 100644 index 0000000..f6a59fa --- /dev/null +++ b/apps/desktop/tests/main/services/SuggestionService.test.ts @@ -0,0 +1,274 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const config = vi.hoisted(() => ({ + suggestionEnabled: true, + suggestionModelId: 'gemma4:e4b', + llmModelId: 'gemma4:e4b', + inputExcludedApps: [], + suggestionTriggerDelayMs: 600, + suggestionMinPrefixChars: 8, + suggestionMaxRequestsPerMinute: 6, + suggestionDailyBudget: 500, + suggestionRequestTimeoutMs: 8000, + inputLearnTypedText: false, + inputTelemetryEnabled: false, + suggestionOverlayInteractive: true +})) +const localLlm = vi.hoisted(() => ({ + isAvailable: vi.fn(() => true), + streamGenerate: vi.fn() +})) + +vi.mock('../../../src/main/services/ConfigService', () => ({ + configGet: vi.fn((key: keyof typeof config) => config[key]), + configSet: vi.fn() +})) +vi.mock('../../../src/main/services/LoggerService', () => ({ + getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }) +})) +vi.mock('../../../src/main/services/LocalLLMService', () => ({ + getLocalLLMService: () => localLlm +})) +vi.mock('../../../src/main/services/InputTelemetryService', () => ({ + getInputTelemetryService: () => ({ listPhrases: vi.fn(() => []), recordExternalText: vi.fn() }) +})) +vi.mock('../../../src/main/services/PersonalGraphService', () => ({ + getPersonalGraphService: () => ({ retrieveContext: vi.fn(() => ({ continuations: [], related: [] })) }) +})) +vi.mock('../../../src/main/services/TextInsertService', () => ({ + getTextInsertService: () => ({ insertText: vi.fn() }) +})) +vi.mock('../../../src/main/db', () => ({ getDatabase: vi.fn() })) +vi.mock('../../../src/main/db/schema', () => ({ suggestions: {} })) + +const PREFIX = '오늘 회의에서 논의한 내용을 정리해서' +const context = { + prefix: PREFIX, + fullText: PREFIX, + caretOffset: PREFIX.length, + anchor: null, + isPassword: false, + isEditable: true, + isComposing: false, + hasSelection: false, + available: true, + appName: 'notepad.exe', + windowTitle: 'notes', + idleMs: 1000, + capturedAt: Date.now() +} + +interface InternalSuggestionService { + _abort: AbortController | null + _consecutiveFailures: number + _cooldownUntil: number + _inFlight: boolean + _lastRequestAt: number + _generate(prefix: string, currentContext: typeof context, maxCandidates: number, maxChars: number): Promise + _abortStaleGeneration(currentPrefix: string): void + _watchdog(): void + _generationToken: number +} + +function waitForAbort(signal: AbortSignal | undefined): AsyncGenerator { + return (async function* () { + await new Promise((_resolve, reject) => { + if (!signal) { + reject(new Error('missing abort signal')) + return + } + signal.addEventListener('abort', () => reject(new Error('intentional abort')), { once: true }) + }) + yield 'unreachable' + })() +} + +beforeEach(() => { + Object.assign(config, { + suggestionEnabled: true, + suggestionModelId: 'gemma4:e4b', + llmModelId: 'gemma4:e4b', + inputExcludedApps: [], + suggestionTriggerDelayMs: 600, + suggestionMinPrefixChars: 8, + suggestionMaxRequestsPerMinute: 6, + suggestionDailyBudget: 500, + suggestionRequestTimeoutMs: 8000, + inputLearnTypedText: false, + inputTelemetryEnabled: false, + suggestionOverlayInteractive: true + }) + localLlm.isAvailable.mockReset() + localLlm.isAvailable.mockReturnValue(true) + localLlm.streamGenerate.mockReset() +}) + +afterEach(async () => { + const { resetSuggestionServiceForTests } = await import('../../../src/main/services/SuggestionService') + resetSuggestionServiceForTests() + config.suggestionEnabled = true + vi.useRealTimers() + vi.clearAllMocks() +}) + +describe('SuggestionService warm-up', () => { + it('동시 warm-up 호출을 하나의 1토큰 요청으로 합치고 2분만 유지한다', async () => { + localLlm.streamGenerate.mockImplementation(async function* () { + yield 'ok' + }) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const warmingStates: boolean[] = [] + service.on('state-changed', (state) => warmingStates.push(state.warmingUp)) + + await Promise.all([service.warmUp(), service.warmUp()]) + + expect(localLlm.streamGenerate).toHaveBeenCalledTimes(1) + expect(localLlm.streamGenerate).toHaveBeenCalledWith( + 'hi', + expect.objectContaining({ maxTokens: 1, keepAlive: '2m' }) + ) + expect(warmingStates).toEqual([true, false]) + }) + + it('비활성화하면 가용성 재시도 타이머와 warm-up을 취소한다', async () => { + vi.useFakeTimers() + localLlm.isAvailable.mockReturnValue(false) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const warmUp = service.warmUp() + + config.suggestionEnabled = false + service.applyConfig() + await warmUp + + expect(vi.getTimerCount()).toBe(0) + expect(localLlm.streamGenerate).not.toHaveBeenCalled() + }) + + it('dismiss 취소는 실패 쿨다운을 올리거나 재귀 생성하지 않는다', async () => { + localLlm.streamGenerate.mockImplementation((_text: string, options: { signal?: AbortSignal }) => + waitForAbort(options.signal) + ) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const internal = service as unknown as InternalSuggestionService + const cleared: string[] = [] + service.on('cleared', ({ reason }) => cleared.push(reason)) + const generation = internal._generate(PREFIX, context, 3, 240) + + await Promise.resolve() + service.dismiss('dismissed') + await generation + + expect(internal._consecutiveFailures).toBe(0) + expect(internal._cooldownUntil).toBe(0) + expect(localLlm.streamGenerate).toHaveBeenCalledTimes(1) + expect(cleared).toEqual(['dismissed']) + }) + + it('생성 중 동일 접두의 주기 스냅샷은 요청을 취소하지 않는다', async () => { + let signal: AbortSignal | undefined + localLlm.streamGenerate.mockImplementation((_text: string, options: { signal?: AbortSignal }) => { + signal = options.signal + return waitForAbort(options.signal) + }) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const internal = service as unknown as InternalSuggestionService + const generation = internal._generate(PREFIX, context, 3, 240) + + await Promise.resolve() + service.handleTypingContext(context) + + expect(signal?.aborted).toBe(false) + expect(internal._inFlight).toBe(true) + service.dismiss('dismissed') + await generation + }) + + it.each([ + ['선택', { hasSelection: true }, 'selection-active'], + ['포커스 이탈', { isEditable: false }, 'not-editable'] + ])('생성 중 %s은 후보가 없어도 취소하고 정확한 cleared 사유를 낸다', async (_case, update, reason) => { + let signal: AbortSignal | undefined + localLlm.streamGenerate.mockImplementation((_text: string, options: { signal?: AbortSignal }) => { + signal = options.signal + return waitForAbort(options.signal) + }) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const cleared: string[] = [] + service.on('cleared', ({ reason: clearedReason }) => cleared.push(clearedReason)) + const generation = (service as unknown as InternalSuggestionService)._generate(PREFIX, context, 3, 240) + + await Promise.resolve() + expect(service.getState()).toMatchObject({ candidates: [], generating: true }) + service.handleTypingContext({ ...context, ...update }) + await generation + + expect(signal?.aborted).toBe(true) + expect(cleared).toEqual([reason]) + expect(service.getState()).toMatchObject({ candidates: [], generating: false, partialText: null }) + }) + + it('후보 게시 updated 상태에는 생성 플래그와 부분 텍스트가 남지 않는다', async () => { + localLlm.streamGenerate.mockImplementation(async function* () { + yield '다음 단계도 확인하겠습니다.' + }) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const states: Array<{ candidates: unknown[]; generating: boolean; partialText: string | null }> = [] + service.on('updated', (state) => states.push(state)) + + await (service as unknown as InternalSuggestionService)._generate(PREFIX, context, 3, 240) + + const published = states.find((state) => state.candidates.length > 0) + expect(published).toMatchObject({ generating: false, partialText: null }) + }) + + it('stale 취소는 실패 쿨다운을 올리거나 즉시 재시작하지 않는다', async () => { + localLlm.streamGenerate.mockImplementation((_text: string, options: { signal?: AbortSignal }) => + waitForAbort(options.signal) + ) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const internal = service as unknown as InternalSuggestionService + const cleared: string[] = [] + service.on('cleared', ({ reason }) => cleared.push(reason)) + const generation = internal._generate(PREFIX, context, 3, 240) + + await Promise.resolve() + internal._abortStaleGeneration('완전히 다른 문맥으로 바뀐 입력입니다') + await generation + + expect(internal._consecutiveFailures).toBe(0) + expect(internal._cooldownUntil).toBe(0) + expect(localLlm.streamGenerate).toHaveBeenCalledTimes(1) + expect(cleared).toEqual([]) + }) + + it('watchdog는 실제 요청 signal을 abort하고 세대를 무효화한다', async () => { + let signal: AbortSignal | undefined + localLlm.streamGenerate.mockImplementation((_text: string, options: { signal?: AbortSignal }) => { + signal = options.signal + return waitForAbort(options.signal) + }) + const { getSuggestionService } = await import('../../../src/main/services/SuggestionService') + const service = getSuggestionService() + const internal = service as unknown as InternalSuggestionService + const cleared: string[] = [] + service.on('cleared', ({ reason }) => cleared.push(reason)) + const generation = internal._generate(PREFIX, context, 3, 240) + + await Promise.resolve() + internal._lastRequestAt = Date.now() - 13001 + internal._watchdog() + await generation + + expect(signal?.aborted).toBe(true) + expect(internal._inFlight).toBe(false) + expect(internal._consecutiveFailures).toBe(0) + expect(cleared).toEqual([]) + }) +}) diff --git a/apps/desktop/tests/main/services/VoiceConversationService.test.ts b/apps/desktop/tests/main/services/VoiceConversationService.test.ts new file mode 100644 index 0000000..c247bd6 --- /dev/null +++ b/apps/desktop/tests/main/services/VoiceConversationService.test.ts @@ -0,0 +1,194 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { IPC_CHANNELS } from '@d3ro/core/ipc-channels' +import { ErrorCode } from '@d3ro/core/errors' +import { + getVoiceConversationService, + resetVoiceConversationServiceForTests, +} from '../../../src/main/services/VoiceConversationService' + +type ChatOptions = { + signal?: AbortSignal + maxTokens?: number + timeoutMs?: number + keepAlive?: string +} + +const mocks = vi.hoisted(() => ({ + audioStart: vi.fn(async () => undefined), + audioStop: vi.fn(async () => undefined), + chatStream: vi.fn(), + premiumCancel: vi.fn(), + speakSentences: vi.fn(async () => undefined), + ttsStop: vi.fn(), + send: vi.fn(), +})) + +vi.mock('../../../src/main/services/LoggerService', () => ({ + getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})) + +vi.mock('../../../src/main/services/AudioCaptureService', () => ({ + getAudioCaptureService: () => ({ + start: mocks.audioStart, + stop: mocks.audioStop, + on: vi.fn(), + off: vi.fn(), + }), +})) + +vi.mock('../../../src/main/services/LocalSTTService', () => ({ + getLocalSTTService: () => ({ initialize: vi.fn(async () => undefined) }), +})) + +vi.mock('../../../src/main/services/PremiumLLMService', () => ({ + getPremiumLLMService: () => ({ + cancelGeneration: mocks.premiumCancel, + isAvailable: () => false, + }), +})) + +vi.mock('../../../src/main/services/LocalLLMService', () => ({ + getLocalLLMService: () => ({ + isAvailable: () => true, + chatStream: mocks.chatStream, + }), +})) + +vi.mock('../../../src/main/services/TTSPlaybackService', () => ({ + getTTSPlaybackService: () => ({ + speakSentences: mocks.speakSentences, + stop: mocks.ttsStop, + }), +})) + +vi.mock('../../../src/main/services/SoundEffectService', () => ({ + getSoundEffectService: () => ({ play: vi.fn() }), +})) + +vi.mock('../../../src/main/services/ConfigService', () => ({ + configGet: () => undefined, +})) + +vi.mock('../../../src/main/services/LicenseService', () => ({ + getLicenseService: () => ({ + canUse: () => ({ allowed: true }), + promptUpgrade: vi.fn(), + }), +})) + +vi.mock('../../../src/main/windows/WindowManager', () => ({ + getMainWindow: () => ({ + isDestroyed: () => false, + webContents: { send: mocks.send }, + }), +})) + +function waitForAbort(signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve() + return new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })) +} + +async function* waitUntilAborted(options?: ChatOptions): AsyncGenerator { + if (!options?.signal) throw new Error('Expected caller-owned abort signal') + await waitForAbort(options.signal) + return '' +} + +describe('VoiceConversationService local response ownership', () => { + beforeEach(() => { + resetVoiceConversationServiceForTests() + vi.clearAllMocks() + mocks.chatStream.mockImplementation((_messages: unknown, options?: ChatOptions) => waitUntilAborted(options)) + }) + + afterEach(() => { + resetVoiceConversationServiceForTests() + }) + + it('rejects a second send without starting a second local chat or adding duplicate history', async () => { + const service = getVoiceConversationService() + await service.startSession() + + const firstResponse = service.sendTextMessage('first') + await vi.waitFor(() => expect(mocks.chatStream).toHaveBeenCalledTimes(1)) + + await expect(service.sendTextMessage('second')).rejects.toMatchObject({ + code: ErrorCode.ConversationLLMFailed, + }) + expect(mocks.chatStream).toHaveBeenCalledTimes(1) + expect(service.getHistory().map((message) => message.content)).toEqual(['first']) + + service.stopSession() + await firstResponse + }) + + it.each([ + ['stopSession', (service: ReturnType) => service.stopSession()], + ['cancelResponse', (service: ReturnType) => service.cancelResponse()], + ])('aborts the local stream signal when %s is called', async (_action, cancel) => { + const service = getVoiceConversationService() + await service.startSession() + + const response = service.sendTextMessage('cancel me') + await vi.waitFor(() => expect(mocks.chatStream).toHaveBeenCalledTimes(1)) + const options = mocks.chatStream.mock.calls[0][1] as ChatOptions + + cancel(service) + + expect(options.signal?.aborted).toBe(true) + await response + }) + + it('sets bounded local chat options for voice responses', async () => { + const service = getVoiceConversationService() + await service.startSession() + + const response = service.sendTextMessage('options') + await vi.waitFor(() => expect(mocks.chatStream).toHaveBeenCalledTimes(1)) + + expect(mocks.chatStream).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ + signal: expect.any(AbortSignal), + maxTokens: 512, + timeoutMs: 60_000, + keepAlive: '2m', + }), + ) + + service.stopSession() + await response + }) + + it('does not publish late assistant output or start TTS after cancellation', async () => { + let releaseLateToken: (() => void) | undefined + const lateToken = new Promise((resolve) => { + releaseLateToken = resolve + }) + mocks.chatStream.mockImplementation(() => (async function* (): AsyncGenerator { + await lateToken + yield 'late answer.' + return 'late answer.' + })()) + + const service = getVoiceConversationService() + await service.startSession() + const response = service.sendTextMessage('cancel before response') + await vi.waitFor(() => expect(mocks.chatStream).toHaveBeenCalledTimes(1)) + + service.cancelResponse() + mocks.send.mockClear() + releaseLateToken?.() + await response + + const lateConversationEvents = mocks.send.mock.calls.filter(([channel]) => ( + channel === IPC_CHANNELS.VOICE_CONVERSATION.ASSISTANT_DELTA + || channel === IPC_CHANNELS.VOICE_CONVERSATION.ASSISTANT_MESSAGE + || channel === IPC_CHANNELS.VOICE_CONVERSATION.TTS_STARTED + || channel === IPC_CHANNELS.VOICE_CONVERSATION.TTS_FINISHED + )) + expect(lateConversationEvents).toEqual([]) + expect(mocks.speakSentences).not.toHaveBeenCalled() + expect(service.getHistory().map((message) => message.content)).toEqual(['cancel before response']) + }) +}) diff --git a/apps/desktop/tests/main/services/input-flow-domain.test.ts b/apps/desktop/tests/main/services/input-flow-domain.test.ts new file mode 100644 index 0000000..8e1babf --- /dev/null +++ b/apps/desktop/tests/main/services/input-flow-domain.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from 'vitest' +import { + buildLocalSuggestionCandidates, + calculateFrictionInsight, + rankFlowWindows, + recommendAppExclusion, + selectPhraseHints, + type InputAppSuggestionStat, + type InputHourlyStat, + type InputPrivacyReceipt, + type PersonalPhrase +} from '@d3ro/core/input-intelligence' +import { + auditKeyBindingMap, + createDefaultBindingMap, + type KeyBinding +} from '@d3ro/core/keybinding' + +const HOUR = 60 * 60 * 1000 +const DAY = 24 * HOUR + +function binding(code: number, modifiers: Partial = {}): KeyBinding { + return { + device: 'keyboard', + code, + ctrl: false, + alt: false, + shift: false, + meta: false, + ...modifiers + } +} + +describe('로컬 플로우 도메인', () => { + it('편집 마찰을 0~1 비율과 경계 등급으로 계산한다', () => { + expect(calculateFrictionInsight(0, 0)).toEqual({ rate: 0, editsPer100Chars: 0, band: 'steady' }) + expect(calculateFrictionInsight(92, 8)).toMatchObject({ rate: 0.08, editsPer100Chars: 8.7, band: 'watch' }) + expect(calculateFrictionInsight(82, 18)).toMatchObject({ rate: 0.18, editsPer100Chars: 22, band: 'high' }) + }) + + it('플로우 시간대는 밀도·안정성 순으로 고르고 입력 없는 시간은 빼며 원본을 바꾸지 않는다', () => { + const hourly: InputHourlyStat[] = [ + { hour: 11, keystrokes: 1400, clicks: 2, chars: 1200, backspaces: 0, activeMs: HOUR }, + { hour: 9, keystrokes: 1400, clicks: 2, chars: 1200, backspaces: 0, activeMs: HOUR }, + { hour: 13, keystrokes: 200, clicks: 0, chars: 300, backspaces: 100, activeMs: HOUR / 2 }, + { hour: 3, keystrokes: 0, clicks: 0, chars: 0, backspaces: 0, activeMs: 0 } + ] + const before = structuredClone(hourly) + + expect(rankFlowWindows(hourly, 1, 3)).toEqual([ + { hour: 9, score: 100, activeMinutes: 60, chars: 1200, frictionRate: 0 }, + { hour: 11, score: 100, activeMinutes: 60, chars: 1200, frictionRate: 0 }, + { hour: 13, score: 44, activeMinutes: 30, chars: 300, frictionRate: 0.25 } + ]) + expect(hourly).toEqual(before) + }) + + it('활성 일수로 나눠 같은 시간대라도 하루 밀도를 낮춘다', () => { + const [window] = rankFlowWindows( + [{ hour: 9, keystrokes: 1400, clicks: 0, chars: 1200, backspaces: 0, activeMs: HOUR }], + 2 + ) + + expect(window).toMatchObject({ hour: 9, score: 58, activeMinutes: 30, chars: 1200 }) + }) + + it('앱 문체 보너스와 반감기로 개인 문구를 정렬하며 원본 배열을 바꾸지 않는다', () => { + const now = 1_800_000_000_000 + const phrases: PersonalPhrase[] = [ + { + id: 'z-old-frequent', + phrase: '배포 일정을 공유합니다', + count: 16, + source: 'typed', + appName: 'Slack.exe', + lastUsedAt: now - DAY * 90, + createdAt: now - DAY * 90 + }, + { + id: 'b-current-app', + phrase: '검토 결과를 남깁니다', + count: 1, + source: 'typed', + appName: 'Notion.exe', + lastUsedAt: now, + createdAt: now + }, + { + id: 'a-current-app-tie', + phrase: '다음 조치를 확인합니다', + count: 1, + source: 'typed', + appName: 'Notion.exe', + lastUsedAt: now, + createdAt: now + } + ] + const before = structuredClone(phrases) + + expect(selectPhraseHints(phrases, '', 3, { appName: 'notion.exe', now })).toEqual([ + '다음 조치를 확인합니다', + '검토 결과를 남깁니다', + '배포 일정을 공유합니다' + ]) + expect(phrases).toEqual(before) + }) + + it('반복적인 비가독 앱만 제외를 권하고 읽힌 기록이 하나라도 있으면 멈춘다', () => { + expect( + recommendAppExclusion({ appName: 'Legacy.exe', samples: 4, readable: 0, unreadable: 3, empty: 1 }) + ).toEqual({ appName: 'Legacy.exe', reason: 'repeated-unreadable', samples: 4 }) + expect( + recommendAppExclusion({ appName: 'Mixed.exe', samples: 6, readable: 1, unreadable: 5, empty: 0 }) + ).toBeNull() + expect( + recommendAppExclusion({ appName: ' ', samples: 6, readable: 0, unreadable: 6, empty: 0 }) + ).toBeNull() + }) + + it('로컬 제안은 출처 우선순위·접미 중첩 제거·중복 제거를 지킨다', () => { + expect( + buildLocalSuggestionCandidates( + '회의 결과', + { + continuationHints: ['를 공유합니다.'], + relatedHints: ['결과를 정리합니다.', '무관한 전체 문장'], + phraseHints: ['결과를 정리합니다.', '결과를 검토합니다.'] + } + ) + ).toEqual(['를 공유합니다.', '를 정리합니다.', '를 검토합니다.']) + }) + + it('문장이 끝난 뒤에만 겹침 없는 전체 문구를 로컬 제안으로 허용한다', () => { + const hints = { continuationHints: [], relatedHints: ['다음 안건을 정리합니다.'], phraseHints: [] } + + expect(buildLocalSuggestionCandidates('회의를 마쳤다', hints)).toEqual([]) + expect(buildLocalSuggestionCandidates('회의를 마쳤다.', hints)).toEqual(['다음 안건을 정리합니다.']) + }) + + it('프라이버시·앱 품질 계약은 수량과 로컬 경계를 명시한다', () => { + const receipt: InputPrivacyReceipt = { + localOnly: true, + rawKeyContentStored: false, + retention: { + activityDays: 30, + typingSamplesDays: 30, + suggestionDays: 30, + personalPhrases: 'until-deleted' + }, + counts: { activityBuckets: 3, typingSamples: 2, personalPhrases: 4, suggestions: 5 } + } + const appStat: InputAppSuggestionStat = { + appName: 'notion.exe', + total: 4, + accepted: 2, + acceptRate: 0.5, + avgLatencyMs: 310 + } + + expect(receipt.counts.personalPhrases).toBe(4) + expect(receipt.retention).toEqual({ + activityDays: 30, + typingSamplesDays: 30, + suggestionDays: 30, + personalPhrases: 'until-deleted' + }) + expect(appStat.acceptRate).toBe(0.5) + }) +}) + +describe('단축키 안전 감사', () => { + it('유효하지 않은 바인딩과 충돌을 한 번씩 보고하고 홀드·더블프레스 기본 예외는 유지한다', () => { + const map = createDefaultBindingMap() + map.caption = [binding(0x56, { ctrl: true, shift: true })] + map.dictation = [binding(999)] + + const issues = auditKeyBindingMap(map) + + expect(issues).toContainEqual({ + kind: 'invalid', + actionId: 'dictation', + bindingIndex: 0, + binding: binding(999), + reasonKey: 'keybinding.reject.unknownKey', + conflictActionIds: [] + }) + expect(issues).toContainEqual({ + kind: 'conflict', + actionId: 'caption', + bindingIndex: 0, + binding: binding(0x56, { ctrl: true, shift: true }), + reasonKey: null, + conflictActionIds: ['history-popup'] + }) + expect(issues.filter((issue) => issue.kind === 'conflict')).toHaveLength(1) + }) +}) diff --git a/apps/desktop/tests/main/services/input-flow-services.test.ts b/apps/desktop/tests/main/services/input-flow-services.test.ts new file mode 100644 index 0000000..c31b0a1 --- /dev/null +++ b/apps/desktop/tests/main/services/input-flow-services.test.ts @@ -0,0 +1,581 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { inputActivity, personalPhrases, suggestions, typingSamples } from '../../../src/main/db/schema' + +type ActivityRow = { + id: string + date: string + hour: number + appName: string + keystrokes: number + shortcuts: number + backspaces: number + clicks: number + doubleClicks: number + scrollTicks: number + mouseDistancePx: number + chars: number + words: number + sentences: number + activeMs: number + updatedAt: number +} + +type PhraseRow = { + id: string + phrase: string + count: number + source: 'typed' | 'voice' | 'suggestion' | 'clipboard' + appName: string | null + lastUsedAt: number | null + createdAt: number +} + +type SuggestionRow = { + id: string + appName: string | null + prefixText: string + suggestionText: string + candidateCount: number + model: string | null + latencyMs: number | null + accepted: boolean + createdAt: number +} + +type SampleRow = { + id: string + text: string + wordCount: number + charCount: number + appName: string | null + windowTitle: string | null + source: 'typed' | 'voice' | 'suggestion' | 'clipboard' + createdAt: number +} + +type StreamOptions = { signal?: AbortSignal } +type StreamFactory = (prompt: string, options: StreamOptions) => AsyncIterable + +const harness = vi.hoisted(() => { + const config = new Map() + const failure = { read: false, delete: false } + const model: { available: boolean; streamGenerate: StreamFactory } = { + available: false, + streamGenerate: async function* () { + return + } + } + const rows: { + activity: ActivityRow[] + phrases: PhraseRow[] + samples: SampleRow[] + suggestions: SuggestionRow[] + } = { activity: [], phrases: [], samples: [], suggestions: [] } + + const rowsFor = (table: unknown): ActivityRow[] | PhraseRow[] | SampleRow[] | SuggestionRow[] => { + if (table === inputActivity) return rows.activity + if (table === personalPhrases) return rows.phrases + if (table === typingSamples) return rows.samples + return rows.suggestions + } + + const database = { + select(selection?: Record) { + if (failure.read) throw new Error('receipt read failed') + return { + from(table: unknown) { + const selectedRows = rowsFor(table) + const chain = { + where: () => chain, + orderBy: () => chain, + limit: () => chain, + all: () => { + const keys = Object.keys(selection ?? {}) + if (table === suggestions && keys.includes('appName')) { + return rows.suggestions.map(({ appName, accepted, latencyMs }) => ({ appName, accepted, latencyMs })) + } + return [...selectedRows] + }, + get: () => { + const keys = Object.keys(selection ?? {}) + if (keys.includes('total')) { + const suggestionRows = rows.suggestions + const latencyRows = suggestionRows.filter((row) => row.latencyMs !== null) + return { + total: suggestionRows.length, + accepted: suggestionRows.filter((row) => row.accepted).length, + avgLatency: + latencyRows.length === 0 + ? null + : latencyRows.reduce((total, row) => total + (row.latencyMs ?? 0), 0) / + latencyRows.length + } + } + if (keys.includes('value')) return { value: selectedRows.length } + return selectedRows[0] + } + } + return chain + } + } + }, + insert(table: unknown) { + return { + values(value: Record) { + const selectedRows = rowsFor(table) + const apply = () => { + if (table === personalPhrases) { + const phrase = value.phrase as string + const existing = rows.phrases.find((row) => row.phrase === phrase) + if (existing) { + existing.count += 1 + existing.lastUsedAt = value.lastUsedAt as number + existing.appName = value.appName as string | null + return + } + } + selectedRows.push(value as never) + } + return { + run: () => apply(), + onConflictDoUpdate: () => ({ run: () => apply() }) + } + } + } + }, + delete(table: unknown) { + const selectedRows = rowsFor(table) + const chain = { + where: () => chain, + run: () => { + if (failure.delete) throw new Error('delete failed') + selectedRows.splice(0, selectedRows.length) + return { changes: 1 } + } + } + return chain + }, + update: () => ({ set: () => ({ where: () => ({ run: () => ({ changes: 1 }) }) }) }) + } + + const uia = { + isAvailable: () => false, + lastReason: 'test', + lastSuccessAt: 0, + getSnapshot: vi.fn(async () => ({ + available: false, + reason: 'test', + isPassword: false, + isEditable: false, + isComposing: false, + hasSelection: false, + textSource: 'none' as const, + text: '', + caretOffset: null, + caretRect: null, + elementRect: null, + windowTitle: null, + appName: null, + processId: null, + capturedAt: Date.now() + })) + } + return { + config, + failure, + model, + rows, + database, + uia, + graph: { continuations: [] as string[], related: [] as string[] } + } +}) + +vi.mock('electron', () => ({ screen: { getPrimaryDisplay: () => ({ scaleFactor: 1 }) } })) +vi.mock('uiohook-napi', () => ({ uIOhook: { on: vi.fn(), removeListener: vi.fn() } })) +vi.mock('../../../src/main/db', () => ({ getDatabase: () => harness.database })) +vi.mock('../../../src/main/services/ConfigService', () => ({ + configGet: (key: string) => harness.config.get(key), + configSet: (key: string, value: unknown) => harness.config.set(key, value) +})) +vi.mock('../../../src/main/services/LoggerService', () => ({ + getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }) +})) +vi.mock('../../../src/main/services/global-input-hook', () => ({ acquireGlobalInputHook: () => () => undefined })) +vi.mock('../../../src/main/services/KeyBindingService', () => ({ uiohookCodeToVk: () => null })) +vi.mock('../../../src/main/services/UiaContextService', () => ({ + getUiaContextService: () => harness.uia +})) +vi.mock('../../../src/main/utils/win32-foreground', () => ({ getForegroundWindowInfo: () => null })) +vi.mock('../../../src/main/services/PersonalGraphService', () => ({ + getPersonalGraphService: () => ({ + clearAll: vi.fn(), + runMaintenance: vi.fn(), + indexText: vi.fn(), + retrieveContext: () => harness.graph + }) +})) +vi.mock('../../../src/main/services/LocalLLMService', () => ({ + getLocalLLMService: () => ({ + isAvailable: () => harness.model.available, + streamGenerate: (prompt: string, options: StreamOptions) => harness.model.streamGenerate(prompt, options) + }) +})) +vi.mock('../../../src/main/services/TextInsertService', () => ({ getTextInsertService: () => ({ insertText: vi.fn() }) })) + +import { + getInputTelemetryService, + resetInputTelemetryServiceForTests +} from '../../../src/main/services/InputTelemetryService' +import { + getSuggestionService, + resetSuggestionServiceForTests +} from '../../../src/main/services/SuggestionService' + +const now = Date.now() + +function activity(overrides: Partial = {}): ActivityRow { + return { + id: crypto.randomUUID(), + date: new Date(now).toISOString().slice(0, 10), + hour: 9, + appName: 'Notion.exe', + keystrokes: 0, + shortcuts: 0, + backspaces: 0, + clicks: 0, + doubleClicks: 0, + scrollTicks: 0, + mouseDistancePx: 0, + chars: 0, + words: 0, + sentences: 0, + activeMs: 0, + updatedAt: now, + ...overrides + } +} + +function typingContext(overrides: Partial['handleTypingContext']>[0]> = {}) { + return { + prefix: '오늘 회의 결과를', + fullText: '오늘 회의 결과를', + caretOffset: 9, + anchor: { x: 1, y: 1, width: 1, height: 1 }, + isPassword: false, + isEditable: true, + isComposing: false, + hasSelection: false, + available: true, + appName: 'Notion.exe', + windowTitle: '회의록', + idleMs: 300, + capturedAt: now, + ...overrides + } +} + +beforeEach(() => { + harness.config.clear() + harness.config.set('inputExcludedApps', []) + harness.config.set('inputLearnTypedText', true) + harness.config.set('suggestionEnabled', true) + harness.config.set('suggestionMinPrefixChars', 4) + harness.config.set('suggestionTriggerDelayMs', 100) + harness.config.set('suggestionMaxRequestsPerMinute', 20) + harness.config.set('suggestionDailyBudget', 100) + harness.failure.read = false + harness.failure.delete = false + harness.model.available = false + harness.model.streamGenerate = async function* () { + return + } + harness.rows.activity.splice(0) + harness.rows.phrases.splice(0) + harness.rows.samples.splice(0) + harness.rows.suggestions.splice(0) + harness.graph.continuations = [] + harness.graph.related = [] + resetInputTelemetryServiceForTests() + resetSuggestionServiceForTests() + harness.uia.getSnapshot.mockClear() +}) + +describe('입력 플로우 서비스', () => { + it('제안 표시 중에만 오래된 키 입력과 mouse-up 후 UIA 포커스를 다시 확인한다', async () => { + const telemetry = getInputTelemetryService() as unknown as { + _running: boolean + _lastKeyAt: number + _sampleWhileTyping: () => Promise + _handleMouseUp: (event: { x: number; y: number }) => void + setSuggestionPresentationActive: (active: boolean) => void + } + telemetry._running = true + telemetry._lastKeyAt = Date.now() - 6000 + + await telemetry._sampleWhileTyping() + expect(harness.uia.getSnapshot).not.toHaveBeenCalled() + + telemetry.setSuggestionPresentationActive(true) + await telemetry._sampleWhileTyping() + expect(harness.uia.getSnapshot).toHaveBeenCalledTimes(1) + + vi.useFakeTimers() + telemetry._handleMouseUp({ x: 1, y: 1 }) + await vi.advanceTimersByTimeAsync(120) + expect(harness.uia.getSnapshot).toHaveBeenCalledTimes(2) + vi.useRealTimers() + }) + + it('시간대 집계에 플로우·마찰과 앱별 제안 품질을 함께 반환한다', () => { + harness.rows.activity.push( + activity({ hour: 9, chars: 1200, backspaces: 30, activeMs: 60 * 60 * 1000, keystrokes: 1300 }), + activity({ id: crypto.randomUUID(), hour: 14, chars: 120, backspaces: 60, activeMs: 20 * 60 * 1000 }) + ) + harness.rows.suggestions.push( + { id: 's1', appName: 'Notion.exe', prefixText: 'a', suggestionText: 'b', candidateCount: 1, model: 'm', latencyMs: 100, accepted: true, createdAt: now }, + { id: 's2', appName: 'Notion.exe', prefixText: 'a', suggestionText: 'c', candidateCount: 1, model: 'm', latencyMs: 300, accepted: false, createdAt: now }, + { id: 's3', appName: 'Slack.exe', prefixText: 'a', suggestionText: 'd', candidateCount: 1, model: 'm', latencyMs: null, accepted: true, createdAt: now } + ) + + const summary = getInputTelemetryService().getSummary(1) + + expect(summary.hourly[9]).toMatchObject({ chars: 1200, backspaces: 30, activeMs: 3600000 }) + expect(summary.friction).toMatchObject({ band: 'steady', editsPer100Chars: 6.8 }) + expect(summary.flowWindows[0]).toMatchObject({ hour: 9 }) + expect(summary.suggestionApps).toEqual([ + { appName: 'Notion.exe', total: 2, accepted: 1, acceptRate: 0.5, avgLatencyMs: 200 }, + { appName: 'Slack.exe', total: 1, accepted: 1, acceptRate: 1, avgLatencyMs: null } + ]) + }) + + it('개인정보 영수증은 로컬 보존 경계와 실제 행 수를 반환한다', () => { + harness.rows.activity.push(activity()) + harness.rows.samples.push({ id: 'sample', text: '학습 문장입니다', wordCount: 2, charCount: 7, appName: null, windowTitle: null, source: 'typed', createdAt: now }) + harness.rows.phrases.push({ id: 'phrase', phrase: '학습 문장입니다', count: 1, source: 'typed', appName: null, lastUsedAt: now, createdAt: now }) + harness.rows.suggestions.push({ id: 's', appName: null, prefixText: '', suggestionText: '후보', candidateCount: 1, model: 'm', latencyMs: 10, accepted: false, createdAt: now }) + + expect(getInputTelemetryService().getPrivacyReceipt()).toMatchObject({ + localOnly: true, + rawKeyContentStored: false, + retention: { + activityDays: 30, + typingSamplesDays: 30, + suggestionDays: 30, + personalPhrases: 'until-deleted' + }, + counts: { activityBuckets: 1, typingSamples: 1, personalPhrases: 1, suggestions: 1 } + }) + }) + + it('개인정보 영수증 조회 실패를 빈 영수증으로 위장하지 않는다', () => { + harness.failure.read = true + + expect(() => getInputTelemetryService().getPrivacyReceipt()).toThrow('receipt read failed') + }) + + it('전체 삭제는 제안 이력까지 제거한다', () => { + harness.rows.activity.push(activity()) + harness.rows.suggestions.push({ id: 's', appName: null, prefixText: '', suggestionText: '후보', candidateCount: 1, model: 'm', latencyMs: 10, accepted: false, createdAt: now }) + + getInputTelemetryService().clearAll() + + expect(getInputTelemetryService().getPrivacyReceipt().counts).toEqual({ + activityBuckets: 0, + typingSamples: 0, + personalPhrases: 0, + suggestions: 0 + }) + }) + + it('전체 삭제 실패를 성공처럼 반환하지 않고 메모리 초기화도 건너뛴다', () => { + harness.rows.activity.push(activity()) + const service = getInputTelemetryService() as unknown as { + _pending: Map + clearAll: () => void + } + service._pending.set('pending', {}) + harness.failure.delete = true + + expect(() => service.clearAll()).toThrow('delete failed') + expect(harness.rows.activity).toHaveLength(1) + expect(service._pending.size).toBe(1) + }) + + it('보존 정리는 만료 제안 이력도 같은 경계로 정리한다', () => { + harness.rows.suggestions.push({ + id: 'expired', + appName: null, + prefixText: '', + suggestionText: '만료 후보', + candidateCount: 1, + model: 'm', + latencyMs: 10, + accepted: false, + createdAt: now - 31 * 24 * 60 * 60 * 1000 + }) + const service = getInputTelemetryService() as unknown as { + _pruneOldData: (at: number) => void + } + + service._pruneOldData(now) + + expect(getInputTelemetryService().getPrivacyReceipt().counts.suggestions).toBe(0) + }) + + it('학습 문구에 마지막 앱을 저장하고 목록 계약에 매핑한다', () => { + const service = getInputTelemetryService() as unknown as { + _learnText: (text: string, meta: { appName: string | null; windowTitle: string | null; source: 'typed' }) => void + } + + service._learnText('배포 일정을 공유합니다. 다음 조치를 확인합니다.', { + appName: 'Slack.exe', + windowTitle: '채널', + source: 'typed' + }) + + expect(getInputTelemetryService().listPhrases()).toEqual( + expect.arrayContaining([expect.objectContaining({ appName: 'Slack.exe' })]) + ) + }) + + it('비밀번호 스냅샷은 제외 추천 증거에 넣지 않고 반복 비가독 앱만 추천한다', () => { + const service = getInputTelemetryService() as unknown as { + _foreground: { appName: string | null } + _recordReadabilityEvidence: (snapshot: { isPassword: boolean; isEditable: boolean; textSource: 'none' | 'value'; text: string }) => void + } + service._foreground.appName = 'Legacy.exe' + for (let index = 0; index < 4; index += 1) { + service._recordReadabilityEvidence({ isPassword: false, isEditable: false, textSource: 'none', text: '' }) + } + service._recordReadabilityEvidence({ isPassword: true, isEditable: false, textSource: 'none', text: '' }) + + expect(getInputTelemetryService().getState().exclusionRecommendation).toEqual({ + appName: 'Legacy.exe', + reason: 'repeated-unreadable', + samples: 4 + }) + }) + + it('모델이 없을 때 정책을 지키며 로컬 기억 후보와 출처를 게시한다', () => { + harness.rows.phrases.push({ id: 'phrase', phrase: '검토 결과를 공유합니다', count: 4, source: 'typed', appName: 'Notion.exe', lastUsedAt: now, createdAt: now }) + harness.graph.continuations = ['다음 조치를 정리하겠습니다'] + + getSuggestionService().handleTypingContext({ + prefix: '오늘 회의 결과를', + fullText: '오늘 회의 결과를', + caretOffset: 9, + anchor: { x: 1, y: 1, width: 1, height: 1 }, + isPassword: false, + isEditable: true, + isComposing: false, + available: true, + appName: 'Notion.exe', + windowTitle: '회의록', + idleMs: 300, + capturedAt: now + }) + + expect(getSuggestionService().getState()).toMatchObject({ + visible: true, + provenance: { mode: 'local-memory', continuationCount: 1, appPhraseCount: 1 } + }) + expect(harness.rows.suggestions[0]).toMatchObject({ model: 'local-memory' }) + }) + + it('timeout abort가 스트림에서 throw되어도 로컬 기억 fallback을 한 번 게시한다', async () => { + const context = typingContext() + const service = getSuggestionService() as unknown as { + _lastContext: typeof context | null + _lastContextAt: number + _generate: (prefix: string, context: typeof context, maxCandidates: number, maxChars: number) => Promise + } + harness.config.set('llmModelId', 'local-model') + harness.config.set('suggestionRequestTimeoutMs', 10) + harness.model.available = true + harness.graph.continuations = ['다음 조치를 정리하겠습니다'] + harness.model.streamGenerate = (_prompt, options) => ({ + async *[Symbol.asyncIterator]() { + await new Promise((_resolve, reject) => { + options.signal?.addEventListener('abort', () => reject(new Error('timeout abort')), { once: true }) + }) + } + }) + service._lastContext = context + service._lastContextAt = Date.now() + + await service._generate(context.prefix, context, 3, 160) + + expect(getSuggestionService().getState()).toMatchObject({ + visible: true, + provenance: { mode: 'local-memory', continuationCount: 1 } + }) + expect(harness.rows.suggestions).toHaveLength(1) + }) + + it('timeout 중 현재 문맥과 세대가 바뀌면 로컬 기억 fallback을 게시하지 않는다', async () => { + const context = typingContext() + const service = getSuggestionService() as unknown as { + _generationToken: number + _lastContext: typeof context | null + _lastContextAt: number + _generate: (prefix: string, context: typeof context, maxCandidates: number, maxChars: number) => Promise + } + harness.config.set('llmModelId', 'local-model') + harness.config.set('suggestionRequestTimeoutMs', 20) + harness.model.available = true + harness.graph.continuations = ['다음 조치를 정리하겠습니다'] + harness.model.streamGenerate = (_prompt, options) => ({ + async *[Symbol.asyncIterator]() { + await new Promise((_resolve, reject) => { + options.signal?.addEventListener('abort', () => reject(new Error('timeout abort')), { once: true }) + }) + } + }) + service._lastContext = context + service._lastContextAt = Date.now() + const changed = typingContext({ prefix: '다른 문맥입니다', appName: 'Slack.exe', windowTitle: '채널' }) + const invalidate = setTimeout(() => { + service._generationToken += 1 + service._lastContext = changed + service._lastContextAt = Date.now() + }, 1) + + await service._generate(context.prefix, context, 3, 160) + clearTimeout(invalidate) + + expect(getSuggestionService().getState().visible).toBe(false) + expect(harness.rows.suggestions).toHaveLength(0) + }) + + it('세대가 달라진 로컬 후보는 게시하지 않아 명시 취소 뒤 되살아나지 않는다', () => { + const service = getSuggestionService() as unknown as { + _generationToken: number + _publishLocalMemory: ( + prefix: string, + context: Parameters['handleTypingContext']>[0], + maxCandidates: number, + maxChars: number, + latencyMs: number, + token: number + ) => boolean + } + service._generationToken = 2 + + expect( + service._publishLocalMemory( + '오늘 회의 결과를', + { + prefix: '오늘 회의 결과를', fullText: '', caretOffset: null, anchor: null, isPassword: false, + isEditable: true, isComposing: false, hasSelection: false, available: true, appName: 'Notion.exe', windowTitle: null, + idleMs: 300, capturedAt: now + }, + 3, + 160, + 0, + 1 + ) + ).toBe(false) + expect(getSuggestionService().getState().visible).toBe(false) + }) +}) diff --git a/apps/desktop/tests/main/services/input-intelligence.test.ts b/apps/desktop/tests/main/services/input-intelligence.test.ts new file mode 100644 index 0000000..c729e73 --- /dev/null +++ b/apps/desktop/tests/main/services/input-intelligence.test.ts @@ -0,0 +1,470 @@ +// tests/main/services/input-intelligence.test.ts +// 입력 인텔리전스 정책/집계 순수 함수 테스트. +// +// 여기 있는 함수들이 서비스의 판단 근거다: +// - 어떤 키를 어떤 종류로 세는가 (내용은 저장하지 않는다) +// - 케어앞 텍스트 diff 로 얼마나 셌는가 +// - 언제 제안을 요청/스킵/삭제하는가 +// - 오버레이를 어디에 붙이는가 +// 따라서 LLM 없이도 전부 검증할 수 있어야 한다. + +import { describe, it, expect } from 'vitest' +import { + INPUT_TELEMETRY_DEFAULTS, + PASTE_INSERTION_THRESHOLD_CHARS, + SUGGESTION_DEFAULTS, + anchorFloatingPanel, + classifyKeyStroke, + computeTypedDelta, + countSentences, + countWords, + decideSuggestion, + decideSuggestionRefresh, + emptyActivityBucket, + endsSentence, + extractPhrases, + isAppExcluded, + isWordBoundaryKey, + manhattanDistance, + mergeActivityBucket, + parseSuggestionCandidates, + pixelsToMeters, + sanitizeSuggestionLine, + selectPhraseHints, + summarizeActivity, + prefixTail, + textBeforeCaret, + type PersonalPhrase, + type SuggestionPolicyInput +} from '@d3ro/core/input-intelligence' + +const NO_MODS = { ctrl: false, alt: false, shift: false, meta: false } + +describe('classifyKeyStroke', () => { + it('문자/숫자/기능 키를 종류로 나눈다', () => { + expect(classifyKeyStroke(0x41, NO_MODS)).toBe('letter') // A + expect(classifyKeyStroke(0x5a, NO_MODS)).toBe('letter') // Z + expect(classifyKeyStroke(0x30, NO_MODS)).toBe('digit') // 0 + expect(classifyKeyStroke(0x60, NO_MODS)).toBe('digit') // numpad 0 + expect(classifyKeyStroke(0x20, NO_MODS)).toBe('space') + expect(classifyKeyStroke(0x0d, NO_MODS)).toBe('enter') + expect(classifyKeyStroke(0x08, NO_MODS)).toBe('backspace') + expect(classifyKeyStroke(0x2e, NO_MODS)).toBe('delete') + expect(classifyKeyStroke(0x27, NO_MODS)).toBe('arrow') + expect(classifyKeyStroke(0x70, NO_MODS)).toBe('function') + expect(classifyKeyStroke(0x11, NO_MODS)).toBe('modifier') + expect(classifyKeyStroke(0xe5, NO_MODS)).toBe('ime') + }) + + it('수정자 조합은 단축키로 다', () => { + expect(classifyKeyStroke(0x41, { ...NO_MODS, ctrl: true })).toBe('shortcut') + expect(classifyKeyStroke(0x41, { ...NO_MODS, alt: true })).toBe('shortcut') + expect(classifyKeyStroke(0x41, { ...NO_MODS, meta: true })).toBe('shortcut') + }) + + it('알 수 없는 키는 other 이며 키 내용을 담지 않는다', () => { + expect(classifyKeyStroke(0x0000, NO_MODS)).toBe('other') + }) + + it('단어 경계 키 판정', () => { + expect(isWordBoundaryKey('space')).toBe(true) + expect(isWordBoundaryKey('enter')).toBe(true) + expect(isWordBoundaryKey('letter')).toBe(false) + }) +}) + +describe('텍스트 지표', () => { + it('단어 수 — 공백 분리 토큰 중 문자/숫자가 있는 것만', () => { + expect(countWords('')).toBe(0) + expect(countWords('hello world')).toBe(2) + expect(countWords('안녕하세요 오늘 날씨')).toBe(3) + expect(countWords(' , . ! ')).toBe(0) + expect(countWords('v1.2 배포 준비')).toBe(3) + }) + + it('문장 수 — 종결 부호와 개행', () => { + expect(countSentences('하나. 둘! 셋?')).toBe(3) + expect(countSentences('no terminator')).toBe(0) + expect(countSentences('첫 줄\n둘째 줄')).toBe(1) + expect(countSentences('첫 줄\n둘째 줄\n')).toBe(2) + }) + + it('문장 종결 판정', () => { + expect(endsSentence('났습니다.')).toBe(true) + expect(endsSentence('아직 쓰는 중')).toBe(false) + expect(endsSentence('')).toBe(false) + }) + + it('케어 앞 텍스트만 어낸다', () => { + expect(textBeforeCaret('hello world', 5)).toBe('hello') + expect(textBeforeCaret('hello', null)).toBe('hello') + expect(textBeforeCaret('hello', 99)).toBe('hello') + }) +}) + +describe('computeTypedDelta', () => { + it('끝에 이어 쓰면 삽입 구간만 센다', () => { + const delta = computeTypedDelta('안녕하세요', '안녕하세요 반갑', { keepText: true }) + expect(delta.insertedChars).toBe(3) + expect(delta.insertedText).toBe(' 반갑') + expect(delta.replaced).toBe(false) + }) + + it('IME 커밋 텍스트(한글)도 그대로 계산된다', () => { + const delta = computeTypedDelta('', '오늘 회의는', { keepText: true }) + expect(delta.insertedChars).toBe(6) + expect(delta.insertedWords).toBe(2) + expect(delta.replaced).toBe(false) + }) + + it('중간 삽입도 찾아낸다', () => { + const delta = computeTypedDelta('abc', 'abXc', { keepText: true }) + expect(delta.insertedText).toBe('X') + }) + + it('제만 있으면 0 이다', () => { + const delta = computeTypedDelta('abcdef', 'abc', { keepText: true }) + expect(delta.insertedChars).toBe(0) + expect(delta.insertedText).toBe('') + }) + + it('여넣기 크기 삽입은 통계에서 제외한다', () => { + const pasted = 'x'.repeat(PASTE_INSERTION_THRESHOLD_CHARS + 1) + const delta = computeTypedDelta('', pasted, { keepText: true }) + expect(delta.replaced).toBe(true) + expect(delta.insertedChars).toBe(0) + expect(delta.insertedText).toBe('') + }) + + it('학습이 꺼져 있으면 텍스트를 담지 않는다', () => { + const delta = computeTypedDelta('', 'hello', { keepText: false }) + expect(delta.insertedChars).toBe(5) + expect(delta.insertedText).toBe('') + }) +}) + +describe('decideSuggestion', () => { + function policy(overrides: Partial = {}): SuggestionPolicyInput { + return { + enabled: true, + modelAvailable: true, + overlayVisible: false, + composing: false, + hasSelection: false, + isPassword: false, + isEditable: true, + appName: 'chrome.exe', + excludedApps: [], + prefix: '오늘 회의에서 논의한 내용을 정리해서', + idleMs: SUGGESTION_DEFAULTS.triggerDelayMs + 50, + triggerDelayMs: SUGGESTION_DEFAULTS.triggerDelayMs, + minPrefixChars: SUGGESTION_DEFAULTS.minPrefixChars, + sinceLastRequestMs: SUGGESTION_DEFAULTS.minIntervalMs + 1, + minIntervalMs: SUGGESTION_DEFAULTS.minIntervalMs, + requestsThisMinute: 0, + maxRequestsPerMinute: SUGGESTION_DEFAULTS.maxRequestsPerMinute, + requestsToday: 0, + dailyBudget: SUGGESTION_DEFAULTS.dailyBudget, + ...overrides + } + } + + it('조건이 모두 맞으면 요청한다', () => { + const decision = decideSuggestion(policy()) + expect(decision).toEqual({ action: 'request', prefix: '오늘 회의에서 논의한 내용을 정리해서' }) + }) + + it('비밀번호 필드는 다른 조건보다 먼저 차단한다', () => { + expect(decideSuggestion(policy({ isPassword: true }))).toEqual({ + action: 'clear', + reason: 'password-field' + }) + }) + + it('IME 조합 중에도 손을 멈추면 제안을 만든다 (한국어 필수)', () => { + // 조합을 하드 차단하면 한국어에서 "한 번 뜨고 이후 안 뜨는" 증상이 된다. + const idleEnough = SUGGESTION_DEFAULTS.triggerDelayMs * 2 + 100 + expect(decideSuggestion(policy({ composing: true, idleMs: idleEnough })).action).toBe('request') + }) + + it('IME 조합 중 타이핑이 이어지면 조용히 넘어간다', () => { + const decision = decideSuggestion( + policy({ composing: true, idleMs: SUGGESTION_DEFAULTS.triggerDelayMs - 50 }) + ) + expect(decision).toEqual({ action: 'skip', reason: 'composing' }) + }) + + it('비활성/편집 불가/제외 앱은 clear', () => { + expect(decideSuggestion(policy({ enabled: false }))).toEqual({ action: 'clear', reason: 'disabled' }) + expect(decideSuggestion(policy({ isEditable: false }))).toEqual({ + action: 'clear', + reason: 'not-editable' + }) + expect(decideSuggestion(policy({ excludedApps: ['Chrome'] }))).toEqual({ + action: 'clear', + reason: 'excluded-app' + }) + }) + + it('비축소 선택 중에는 제안을 즉시 지운다', () => { + expect(decideSuggestion(policy({ hasSelection: true }))).toEqual({ + action: 'clear', + reason: 'selection-active' + }) + }) + + it('문장이 마침표로 끝나도 다음 문장을 제안한다 (기능 목적)', () => { + // "다음 문장 제안" 이므로 종결 부호 뒤가 오히려 제안이 필요한 지점이다. + expect(decideSuggestion(policy({ prefix: '오늘 회의는 여기서 끝났습니다.' }))).toEqual({ + action: 'request', + prefix: '오늘 회의는 여기서 끝났습니다.' + }) + }) + + it('접두가 짧으면 지운다', () => { + expect(decideSuggestion(policy({ prefix: '짧음' }))).toEqual({ + action: 'clear', + reason: 'prefix-too-short' + }) + expect(decideSuggestion(policy({ prefix: ' ' }))).toEqual({ + action: 'clear', + reason: 'empty-prefix' + }) + }) + + it('디바운스/모델 없음/레이트 리밋/예산 초과는 skip', () => { + expect(decideSuggestion(policy({ idleMs: 10 }))).toEqual({ action: 'skip', reason: 'debounce' }) + expect(decideSuggestion(policy({ modelAvailable: false }))).toEqual({ + action: 'skip', + reason: 'model-unavailable' + }) + expect(decideSuggestion(policy({ requestsThisMinute: 99 }))).toEqual({ + action: 'skip', + reason: 'rate-limited' + }) + expect(decideSuggestion(policy({ sinceLastRequestMs: 10 }))).toEqual({ + action: 'skip', + reason: 'rate-limited' + }) + expect(decideSuggestion(policy({ requestsToday: 999, dailyBudget: 500 }))).toEqual({ + action: 'skip', + reason: 'budget-exhausted' + }) + }) + + it('이미 떠 있는 제안은 다시 요청하지 않는다', () => { + expect(decideSuggestion(policy({ overlayVisible: true }))).toEqual({ + action: 'skip', + reason: 'already-visible' + }) + }) +}) + +describe('표시 중 제안 재생성 정책', () => { + it('연속 접두가 12자 미만 성장하면 기존 제안을 유지한다', () => { + expect(decideSuggestionRefresh('회의 결과를 공유합니다', '회의 결과를 공유합니다 내일')).toBe('keep') + }) + + it('연속 접두가 12자 이상 성장하면 재생성한다', () => { + expect(decideSuggestionRefresh('회의 결과를 공유합니다', '회의 결과를 공유합니다 다음 안건도 검토해 주세요')).toBe( + 'regenerate' + ) + }) + + it('생성 접두의 앞부분이 바뀌면 stale 로 처리한다', () => { + expect(decideSuggestionRefresh('회의 결과를 공유합니다', '프로젝트 결과를 공유합니다')).toBe('stale') + }) +}) + +describe('isAppExcluded', () => { + it('대소문자와 .exe 를 무시하고 비교한다', () => { + expect(isAppExcluded('KeePassXC.exe', ['keepassxc.exe'])).toBe(true) + expect(isAppExcluded('KeePassXC', ['keepassxc.exe'])).toBe(true) + expect(isAppExcluded('chrome.exe', ['keepassxc.exe'])).toBe(false) + }) + + it(' 목록/빈 이름은 제외하지 않는다', () => { + expect(isAppExcluded('chrome.exe', [])).toBe(false) + expect(isAppExcluded('', ['chrome.exe'])).toBe(false) + }) +}) + +describe('후보 정제', () => { + it('번호/따옴표/불릿을 걷어낸다', () => { + expect(sanitizeSuggestionLine('1. 다음 문장입니다.')).toBe('다음 문장입니다.') + expect(sanitizeSuggestionLine('- "quoted line"')).toBe('quoted line') + expect(sanitizeSuggestionLine('* 별표 항목')).toBe('별표 항목') + }) + + it('지시문을 되풀이한 줄은 버린다 (프롬프트 누출 방어)', () => { + expect(sanitizeSuggestionLine('Suggestion: ...')).toBeNull() + expect(sanitizeSuggestionLine('다음 문장: 이어집니다')).toBeNull() + expect(sanitizeSuggestionLine('{{prefix}}')).toBeNull() + }) + + it('너무 짧은 줄은 버린다', () => { + expect(sanitizeSuggestionLine('a')).toBeNull() + expect(sanitizeSuggestionLine(' ')).toBeNull() + }) + + it('길이를 제한한다', () => { + const long = 'word '.repeat(80) + const trimmed = sanitizeSuggestionLine(long, 40) + expect(trimmed).not.toBeNull() + expect((trimmed ?? '').length).toBeLessThanOrEqual(40) + }) + + it('접두를 되풀이하는 후보는 제외하고 중복도 제거한다', () => { + const prefix = '오늘 회의에서 논의한' + const raw = [ + '오늘 회의에서 논의한 내용을 정리합니다.', + '내용을 정리합니다.', + '내용을 정리합니다.', + '결론부터 공유드립니다.' + ].join('\n') + const candidates = parseSuggestionCandidates(raw, prefix, 3) + expect(candidates).toEqual(['내용을 정리합니다.', '결론부터 공유드립니다.']) + }) + + it('후보 개수 상한을 지킨다', () => { + const raw = '첫 번째 후보입니다.\n두 번째 후보입니다.\n세 번째 후보입니다.' + expect(parseSuggestionCandidates(raw, '', 2)).toHaveLength(2) + }) +}) + +describe('anchorFloatingPanel', () => { + const workArea = { x: 0, y: 0, width: 1920, height: 1080 } + const size = { width: 460, height: 96 } + + it('케어 아래에 붙인다', () => { + const position = anchorFloatingPanel({ x: 400, y: 300, width: 2, height: 20 }, { x: 0, y: 0 }, size, workArea) + expect(position).toEqual({ x: 400, y: 326 }) + }) + + it('아래 공간이 없으면 위로 뒤집는다', () => { + const position = anchorFloatingPanel( + { x: 400, y: 1000, width: 2, height: 20 }, + { x: 0, y: 0 }, + size, + workArea + ) + expect(position.y).toBe(1000 - 6 - 96) + }) + + it('작업영역 밖으로 나가지 않는다', () => { + const position = anchorFloatingPanel( + { x: 1900, y: 10, width: 2, height: 20 }, + { x: 0, y: 0 }, + size, + workArea + ) + expect(position.x).toBe(workArea.width - size.width) + expect(position.x + size.width).toBeLessThanOrEqual(workArea.width) + }) + + it('앵커가 없으면 커서를 쓴다', () => { + const position = anchorFloatingPanel(null, { x: 200, y: 500 }, size, workArea) + expect(position).toEqual({ x: 200, y: 506 }) + }) +}) + +describe('마우스 이동', () => { + it('맨해튼 거리 — 축별 절대값 합 (ActivityWatch 방식)', () => { + expect(manhattanDistance({ x: 0, y: 0 }, { x: 3, y: 4 })).toBe(7) + expect(manhattanDistance({ x: 10, y: 10 }, { x: 10, y: 12 })).toBe(2) + }) + + it('셀을 미터로 환산한다', () => { + // 96 DPI 에서 1인치 = 96px = 0.0254m + expect(pixelsToMeters(96, 1)).toBeCloseTo(0.0254, 6) + expect(pixelsToMeters(192, 2)).toBeCloseTo(0.0254, 6) + }) +}) + +describe('집계 버킷', () => { + it('빈 버킷은 모든 카운터가 0 이다', () => { + const bucket = emptyActivityBucket() + expect(Object.values(bucket).every((value) => value === 0)).toBe(true) + }) + + it('부분 delta 를 누적한다', () => { + const bucket = emptyActivityBucket() + mergeActivityBucket(bucket, { keystrokes: 3, clicks: 1, mouseDistancePx: 120 }) + mergeActivityBucket(bucket, { keystrokes: 2 }) + expect(bucket.keystrokes).toBe(5) + expect(bucket.clicks).toBe(1) + expect(bucket.mouseDistancePx).toBe(120) + expect(bucket.words).toBe(0) + }) + + it('일 평균을 낸다', () => { + const totals = emptyActivityBucket() + mergeActivityBucket(totals, { + keystrokes: 7000, + clicks: 140, + words: 1400, + mouseDistancePx: 96 * 3 + }) + const averages = summarizeActivity(totals, 7) + expect(averages.keystrokes).toBe(1000) + expect(averages.clicks).toBe(20) + expect(averages.words).toBe(200) + expect(averages.mouseDistanceMeters).toBeGreaterThan(0) + }) + + it('0 일 구간에도 0 나눗셈을 하지 않는다', () => { + const averages = summarizeActivity(emptyActivityBucket(), 0) + expect(averages.keystrokes).toBe(0) + expect(averages.mouseDistanceMeters).toBe(0) + }) +}) + +describe('개인 문구', () => { + it('문장 종결 단위로 나누고 짧은 조각은 버린다', () => { + const phrases = extractPhrases('오늘 회의는 짧았습니다. 내일 일정을 공유드릴게요. 짧음.') + expect(phrases).toContain('오늘 회의는 짧았습니다') + expect(phrases).toContain('내일 일정을 공유드릴게요') + expect(phrases).not.toContain('짧음') + }) + + it('프롬프트 힌트는 사용 빈도 순으로 뽑고 접두 중복을 피한다', () => { + const now = Date.now() + const phrases: PersonalPhrase[] = [ + { id: 'a', phrase: '자주 쓰는 문장입니다', count: 5, source: 'typed', appName: null, lastUsedAt: now, createdAt: now }, + { id: 'b', phrase: '가 쓰는 문장입니다', count: 1, source: 'typed', appName: null, lastUsedAt: now, createdAt: now }, + { id: 'c', phrase: '자주 쓰는 문장입니다', count: 9, source: 'suggestion', lastUsedAt: now, createdAt: now } + ] + const hints = selectPhraseHints(phrases, '', 2) + expect(hints[0]).toBe('자주 쓰는 문장입니다') + expect(hints).toHaveLength(2) + }) +}) + +describe('기본값 (실측 근거)', () => { + it('자동 제안의 요청 상한은 보수적 기본 정책을 지킨다', () => { + expect(SUGGESTION_DEFAULTS.triggerDelayMs).toBe(600) + expect(SUGGESTION_DEFAULTS.minIntervalMs).toBe(5000) + expect(SUGGESTION_DEFAULTS.maxRequestsPerMinute).toBe(6) + expect(SUGGESTION_DEFAULTS.dailyBudget).toBe(500) + expect(SUGGESTION_DEFAULTS.maxCandidates).toBe(3) + expect(SUGGESTION_DEFAULTS.maxOutputTokens).toBe(64) + expect(SUGGESTION_DEFAULTS.regenerateAfterChars).toBe(12) + }) + + it('prefixTail 은 공백을 정규화하고 마지막 n자를 취한다', () => { + expect(prefixTail('오늘 회의에서 논의한 내용')).toBe('오늘 회의에서 논의한 내용') + expect(prefixTail('12345678901234567890')).toBe('78901234567890') + expect(prefixTail('짧음')).toBe('짧음') + expect(prefixTail('가나다라마바사아자차카타파하', 4)).toBe('카타파하') + }) + + it('flush 주기는 ActivityWatch heartbeat(5초)와 같다', () => { + expect(INPUT_TELEMETRY_DEFAULTS.flushIntervalMs).toBe(5000) + }) + + it('응답 제한은 콜드 로딩(실측 24.7초)을 오래 붙잡지 않는다', () => { + // gemma4:e4b 실측: 콜드 24.7초 / 워밍업 후 4.9초(32토큰). 상한이 없으면 그 시간 동안 붙잡힌다. + expect(SUGGESTION_DEFAULTS.requestTimeoutMs).toBeGreaterThanOrEqual(5000) + expect(SUGGESTION_DEFAULTS.requestTimeoutMs).toBeLessThanOrEqual(30000) + }) +}) diff --git a/apps/desktop/tests/main/services/llm-prompts.test.ts b/apps/desktop/tests/main/services/llm-prompts.test.ts index 17a6d57..933fe1e 100644 --- a/apps/desktop/tests/main/services/llm-prompts.test.ts +++ b/apps/desktop/tests/main/services/llm-prompts.test.ts @@ -20,8 +20,10 @@ import { renderInstructionPrompt, buildInstructionInvocation, resolveSystemPrompt, + buildSuggestionPrompt, DEFAULT_TARGET_LANGUAGE, BASE_SYSTEM_PROMPTS, + SUGGESTION_NO_THINK_PREFIX, } from '../../../src/main/services/llm-prompts' beforeEach(() => { @@ -167,3 +169,47 @@ describe('resolveSystemPrompt', () => { } }) }) + +describe('buildSuggestionPrompt', () => { + it('지시문은 시스템 프롬프트에만 있고 사용자 텍스트에는 들어가지 않는다', () => { + const { systemPrompt, text } = buildSuggestionPrompt({ + prefix: '오늘 회의에서 논의한 내용을 정리해서', + appName: 'chrome.exe', + windowTitle: '회의록 - Chrome', + phraseHints: ['지난번에는 이렇게 정리했습니다'], + candidates: 3, + maxChars: 120 + }) + + // 지시문이 사용자 텍스트 자리로 새면 모델이 지시문 자체를 다듬어 돌려준다 + // (commit 9c2b4d4 회와 같은 부류). + expect(systemPrompt).toContain('다음 문장') + expect(text).not.toContain('규칙:') + expect(text).toContain('오늘 회의에서 논의한 내용을 정리해서') + expect(text).toContain('지난번에는 이렇게 정리했습니다') + expect(text).toContain('chrome.exe') + }) + + it('후보 수와 길이 상한을 프롬프트에 반영한다', () => { + const { systemPrompt, text } = buildSuggestionPrompt({ prefix: 'abc', candidates: 2, maxChars: 60 }) + + expect(systemPrompt).toContain('60') + expect(text).toContain('2') + }) + + it('추론 모델용 no_think prefix 를 붙인다', () => { + const { systemPrompt } = buildSuggestionPrompt({ prefix: 'abc' }) + expect(systemPrompt.startsWith(SUGGESTION_NO_THINK_PREFIX)).toBe(true) + }) + + it('후보 수/길이는 안전 범위로 클램프한다', () => { + const { systemPrompt, text } = buildSuggestionPrompt({ + prefix: 'abc', + candidates: 99, + maxChars: 99999 + }) + + expect(systemPrompt).toContain('400') + expect(text).toContain('5') + }) +}) diff --git a/apps/desktop/tests/main/services/local-llm-service.test.ts b/apps/desktop/tests/main/services/local-llm-service.test.ts new file mode 100644 index 0000000..c1ad163 --- /dev/null +++ b/apps/desktop/tests/main/services/local-llm-service.test.ts @@ -0,0 +1,181 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ErrorCode } from '@d3ro/core/errors' +import { initInMemoryConfig, resetInMemoryConfig } from '../../../src/main/services/ConfigService' +import { + getLocalLLMService, + resetLocalLLMServiceForTests, +} from '../../../src/main/services/LocalLLMService' + +const encoder = new TextEncoder() + +function completedStream(lines: string[]): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(lines.join('\n'))) + controller.close() + }, + }) +} + +function completedGenerateResponse(): Response { + return new Response(JSON.stringify({ + model: 'gemma4:e4b', + response: 'ok', + done: true, + }), { status: 200 }) +} + +function markAvailable(): void { + const service = getLocalLLMService() as unknown as { _available: boolean } + service._available = true +} + +describe('LocalLLMService request lifecycle', () => { + beforeEach(() => { + initInMemoryConfig() + resetLocalLLMServiceForTests() + markAvailable() + }) + + afterEach(() => { + resetLocalLLMServiceForTests() + resetInMemoryConfig() + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('sends the bounded default num_predict to chat requests', async () => { + const cancel = vi.fn() + const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as { options: { num_predict: number } } + expect(body.options.num_predict).toBe(512) + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode([ + JSON.stringify({ message: { content: 'hello' }, done: false }), + JSON.stringify({ message: { content: '' }, done: true }), + ].join('\n') + '\n')) + }, + cancel, + }), { status: 200 }) + }) + vi.stubGlobal('fetch', fetchMock) + + const output: string[] = [] + for await (const token of getLocalLLMService().chatStream([{ role: 'user', content: 'hello' }])) { + output.push(token) + } + + expect(output).toEqual(['hello']) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(cancel).toHaveBeenCalledTimes(1) + }) + + it('propagates external cancellation and request deadlines to fetch', async () => { + const signals: AbortSignal[] = [] + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { + const signal = init?.signal as AbortSignal + signals.push(signal) + signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true }) + })) + vi.stubGlobal('fetch', fetchMock) + + const external = new AbortController() + const externalRequest = getLocalLLMService().generate('first', { signal: external.signal }) + const externalExpectation = expect(externalRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled }) + external.abort() + await externalExpectation + expect(signals[0].aborted).toBe(true) + + vi.useFakeTimers() + const timeoutRequest = getLocalLLMService().generate('second', { timeoutMs: 25 }) + const timeoutExpectation = expect(timeoutRequest).rejects.toMatchObject({ code: ErrorCode.LLMProcessingTimeout }) + await vi.advanceTimersByTimeAsync(25) + await timeoutExpectation + expect(signals[1].aborted).toBe(true) + }) + + it('keeps other active requests cancellable after one request completes', async () => { + const signals: AbortSignal[] = [] + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => { + const signal = init?.signal as AbortSignal + signals.push(signal) + if (signals.length === 1) return Promise.resolve(completedGenerateResponse()) + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')), { once: true }) + }) + }) + vi.stubGlobal('fetch', fetchMock) + + const first = getLocalLLMService().generate('first') + const second = getLocalLLMService().generate('second') + const third = getLocalLLMService().generate('third') + await expect(first).resolves.toMatchObject({ text: 'ok' }) + + const secondExpectation = expect(second).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled }) + const thirdExpectation = expect(third).rejects.toMatchObject({ code: ErrorCode.LLMProcessingCancelled }) + getLocalLLMService().cancelGeneration() + + await secondExpectation + await thirdExpectation + expect(signals[1].aborted).toBe(true) + expect(signals[2].aborted).toBe(true) + }) + + it('rejects a stream that reaches EOF without a done frame', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(completedStream([ + JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }), + ]), { status: 200 }))) + + const stream = getLocalLLMService().streamGenerate('hello') + await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false }) + await expect(stream.next()).rejects.toMatchObject({ code: ErrorCode.LLMProcessingFailed }) + }) + + it('finishes on a done frame without waiting for the server to close the stream', async () => { + const cancel = vi.fn() + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode([ + JSON.stringify({ model: 'gemma4:e4b', response: 'partial', done: false }), + JSON.stringify({ model: 'gemma4:e4b', response: '', done: true }), + ].join('\n') + '\n')) + }, + cancel, + }) + vi.stubGlobal('fetch', vi.fn(async () => new Response(body, { status: 200 }))) + + const stream = getLocalLLMService().streamGenerate('hello', { timeoutMs: 500 }) + await expect(stream.next()).resolves.toMatchObject({ value: 'partial', done: false }) + await expect(stream.next()).resolves.toMatchObject({ value: expect.objectContaining({ text: 'partial' }), done: true }) + expect(cancel).toHaveBeenCalledTimes(1) + }) + + it('deduplicates concurrent ensureRunning calls and polling startup', async () => { + const service = getLocalLLMService() + const privateService = service as unknown as { + _ensureRunning: () => Promise<'running' | 'starting' | 'not-installed' | 'failed'> + _checkAvailability: () => Promise + } + let release: ((value: 'running') => void) | undefined + const pending = new Promise<'running'>((resolve) => { + release = resolve + }) + const ensure = vi.spyOn(privateService, '_ensureRunning').mockReturnValue(pending) + + const first = service.ensureRunning() + const second = service.ensureRunning() + expect(ensure).toHaveBeenCalledTimes(1) + release?.('running') + await expect(Promise.all([first, second])).resolves.toEqual(['running', 'running']) + + vi.useFakeTimers() + const availability = vi.spyOn(privateService, '_checkAvailability').mockResolvedValue() + service.startPolling() + service.startPolling() + expect(availability).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(5000) + expect(availability).toHaveBeenCalledTimes(2) + service.stopPolling() + }) +}) diff --git a/apps/desktop/tests/main/services/windows-child-process-hide.test.ts b/apps/desktop/tests/main/services/windows-child-process-hide.test.ts new file mode 100644 index 0000000..bb95efa --- /dev/null +++ b/apps/desktop/tests/main/services/windows-child-process-hide.test.ts @@ -0,0 +1,164 @@ +import { EventEmitter } from 'events' +import { readFileSync } from 'fs' +import { resolve } from 'path' +import { promisify } from 'util' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const childProcess = vi.hoisted(() => ({ + exec: vi.fn(), + execFile: vi.fn(), + execFileAsync: vi.fn(), + spawn: vi.fn(), +})) + +vi.mock('child_process', () => childProcess) + +vi.mock('../../../src/main/services/LoggerService', () => ({ + getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }), +})) + +vi.mock('../../../src/main/services/ConfigService', () => ({ + configGet: vi.fn(), + configSet: vi.fn(), +})) + +vi.mock('../../../src/main/services/stt/STTManager', () => ({ + getSTTManager: vi.fn(), +})) + +vi.mock('../../../src/main/services/HistoryService', () => ({ + getHistoryService: vi.fn(), +})) + +vi.mock('../../../src/main/services/RuntimeProvisioner', () => ({ + getRuntimeProvisioner: vi.fn(), +})) + +vi.mock('../../../src/main/services/PremiumLLMService', () => ({ + getPremiumLLMService: vi.fn(), +})) + +function createProcess(args: unknown[]): EventEmitter & { + stdout: EventEmitter + stderr: EventEmitter + kill: ReturnType +} { + const process = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + kill: vi.fn(), + }) + + queueMicrotask(() => { + if (args.includes('null')) { + process.stderr.emit('data', Buffer.from('Duration: 00:00:01.00')) + } + if (args.includes('pipe:1')) { + process.stdout.emit('data', Buffer.from('pcm')) + } + process.emit('close', 0) + }) + + return process +} + +function mockSuccessfulExec(): void { + childProcess.exec.mockImplementation((...args: unknown[]) => { + const callback = args.find((arg): arg is (error: Error | null, stdout: string) => void => typeof arg === 'function') + callback?.(null, '[{"InstanceId":"device-1","FriendlyName":"Mic"}]') + }) +} + +function expectWindowsHideOnAllCalls(mock: { mock: { calls: unknown[][] } }): void { + for (const call of mock.mock.calls) { + const options = call.find( + (arg): arg is { windowsHide?: boolean } => typeof arg === 'object' && arg !== null && !Array.isArray(arg), + ) + expect(options?.windowsHide).toBe(true) + } +} + +beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + mockSuccessfulExec() + childProcess.spawn.mockImplementation((_command: unknown, args: unknown[]) => createProcess(args)) + Reflect.set( + childProcess.execFile, + promisify.custom, + childProcess.execFileAsync, + ) + childProcess.execFileAsync.mockResolvedValue({ stdout: 'TestApp\nTest window', stderr: '' }) +}) + +describe('Windows child process visibility', () => { + it('hides the PowerShell SAPI process used for TTS playback', async () => { + const serviceModule = await import('../../../src/main/services/TTSPlaybackService') + serviceModule.resetTTSPlaybackServiceForTests() + const service = serviceModule.getTTSPlaybackService() as unknown as { + _speakOneWindows(text: string): Promise + } + + await service._speakOneWindows('테스트') + + expect(childProcess.spawn).toHaveBeenCalledTimes(1) + expect(childProcess.spawn.mock.calls[0]?.[0]).toBe('powershell') + expect(childProcess.spawn.mock.calls[0]?.[2]).toMatchObject({ stdio: 'pipe', windowsHide: true }) + }) + + it('hides cmd and PowerShell commands invoked by voice actions', async () => { + const serviceModule = await import('../../../src/main/services/VoiceActionService') + serviceModule.resetVoiceActionServiceForTests() + const service = serviceModule.getVoiceActionService() as unknown as { + _openApp(appName: string): Promise + _simulateKeyboard(combo: string): Promise + _runCommand(command: string): Promise + } + + await service._openApp('notepad') + await service._simulateKeyboard('volumeup') + await service._simulateKeyboard('volumedown') + await service._simulateKeyboard('volumemute') + await service._simulateKeyboard('ctrl+c') + await service._runCommand('echo test') + + expect(childProcess.exec).toHaveBeenCalledTimes(6) + expect(childProcess.exec.mock.calls[0]?.[1]).toMatchObject({ shell: 'cmd.exe', windowsHide: true }) + expect(childProcess.exec.mock.calls[5]?.[1]).toMatchObject({ timeout: 10000, windowsHide: true }) + expectWindowsHideOnAllCalls(childProcess.exec) + }) + + it('hides Windows device discovery and active-window PowerShell calls', async () => { + const audioModule = await import('../../../src/main/services/AudioCaptureService') + const audioService = audioModule.getAudioCaptureService() as unknown as { + _getDevicesWindows(): Promise + } + await audioService._getDevicesWindows() + + const contextModule = await import('../../../src/main/services/ScreenContextService') + const contextService = contextModule.getScreenContextService() + await contextService.captureContext(false) + + expect(childProcess.exec.mock.calls[0]?.[1]).toMatchObject({ + encoding: 'utf8', + timeout: 3000, + windowsHide: true, + }) + expect(childProcess.execFileAsync.mock.calls[0]?.[2]).toMatchObject({ + timeout: 3000, + windowsHide: true, + }) + }) + + it('declares hidden ffmpeg processes for conversion, duration, and chunk extraction', () => { + const source = readFileSync( + resolve(process.cwd(), 'src/main/services/FileTranscriptionService.ts'), + 'utf8', + ) + const ffmpegSpawns = source.match( + /spawn\(ffmpegPath, args, \{ stdio: \['pipe', 'pipe', 'pipe'\], windowsHide: true \}\)/g, + ) + + expect(ffmpegSpawns).toHaveLength(3) + }) +}) diff --git a/apps/mobile-rn/android/app/build.gradle b/apps/mobile-rn/android/app/build.gradle index 4f5db6f..0144c96 100644 --- a/apps/mobile-rn/android/app/build.gradle +++ b/apps/mobile-rn/android/app/build.gradle @@ -151,8 +151,8 @@ def versionSettingsValid = configuredVersionName != null && configuredVersionName ==~ strictSemver && configuredVersionCodeValue != null && configuredVersionCodeValue <= 2100000000L -def resolvedVersionName = versionSettingsValid ? configuredVersionName : "1.4.0" -def resolvedVersionCode = versionSettingsValid ? configuredVersionCodeValue.toInteger() : 1041000 +def resolvedVersionName = versionSettingsValid ? configuredVersionName : "1.5.0" +def resolvedVersionCode = versionSettingsValid ? configuredVersionCodeValue.toInteger() : 1050000 def requiredReleaseSettings = [ D3RO_RELEASE_STORE_FILE: releaseStoreFilePath, diff --git a/apps/mobile-rn/ios/D3ROVoice.xcodeproj/project.pbxproj b/apps/mobile-rn/ios/D3ROVoice.xcodeproj/project.pbxproj index abc4b24..3f221d2 100644 --- a/apps/mobile-rn/ios/D3ROVoice.xcodeproj/project.pbxproj +++ b/apps/mobile-rn/ios/D3ROVoice.xcodeproj/project.pbxproj @@ -257,7 +257,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; - CURRENT_PROJECT_VERSION = 1041000; + CURRENT_PROJECT_VERSION = 1050000; ENABLE_BITCODE = NO; INFOPLIST_FILE = D3ROVoice/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 15.1; @@ -265,7 +265,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.4.0; + MARKETING_VERSION = 1.5.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", @@ -287,14 +287,14 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; - CURRENT_PROJECT_VERSION = 1041000; + CURRENT_PROJECT_VERSION = 1050000; INFOPLIST_FILE = D3ROVoice/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 15.1; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.4.0; + MARKETING_VERSION = 1.5.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", diff --git a/apps/mobile-rn/package-lock.json b/apps/mobile-rn/package-lock.json index e0f3dec..5549a1c 100644 --- a/apps/mobile-rn/package-lock.json +++ b/apps/mobile-rn/package-lock.json @@ -1,12 +1,12 @@ { "name": "@d3ro/mobile-rn", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@d3ro/mobile-rn", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@d3ro/api-client": "file:../../packages/api-client", "@d3ro/core": "file:../../packages/core", @@ -62,7 +62,7 @@ }, "../..": { "name": "d3ro-voice-monorepo", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "workspaces": [ "apps/desktop", @@ -81,7 +81,7 @@ }, "../../packages/api-client": { "name": "@d3ro/api-client", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@d3ro/core": "*", @@ -98,7 +98,7 @@ }, "../../packages/core": { "name": "@d3ro/core", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "docx": "^9.6.1" @@ -109,7 +109,7 @@ }, "../../packages/i18n": { "name": "@d3ro/i18n", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@types/react": "^19.0.0" @@ -120,7 +120,7 @@ }, "../../packages/ui-native": { "name": "@d3ro/ui-native", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@types/react": "*" diff --git a/apps/mobile-rn/package.json b/apps/mobile-rn/package.json index a7bf9a1..28ffb61 100644 --- a/apps/mobile-rn/package.json +++ b/apps/mobile-rn/package.json @@ -1,6 +1,6 @@ { "name": "@d3ro/mobile-rn", - "version": "1.4.0", + "version": "1.5.0", "private": true, "scripts": { "android": "react-native run-android", diff --git a/apps/web/package.json b/apps/web/package.json index a8372a0..070b1fb 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@d3ro/web", - "version": "1.4.0", + "version": "1.5.0", "private": true, "description": "D3RO Voice 웹 앱 — Next.js 기반 SaaS 인터페이스", "scripts": { diff --git a/apps/web/src/components/layout/sidebar.tsx b/apps/web/src/components/layout/sidebar.tsx index 59d4d0b..c30cb8e 100644 --- a/apps/web/src/components/layout/sidebar.tsx +++ b/apps/web/src/components/layout/sidebar.tsx @@ -139,7 +139,7 @@ export function Sidebar(): React.ReactElement { - v1.4.0 + v1.5.0 diff --git a/apps/web/src/lib/desktop-release.ts b/apps/web/src/lib/desktop-release.ts index 89b9d34..b988637 100644 --- a/apps/web/src/lib/desktop-release.ts +++ b/apps/web/src/lib/desktop-release.ts @@ -4,10 +4,10 @@ // 설치 파일은 canonical Forgejo feed에서만 배포한다. 웹/사이트 빌드 산출물에는 // 설치 바이너리가 포함되지 않으므로 `/releases/...` 로컬 경로는 배포 환경에서 404다. -export const DESKTOP_VERSION = '1.4.0' +export const DESKTOP_VERSION = '1.5.0' /** 릴리스 게시일. `release/product-version.json`의 releaseDate와 같아야 한다. */ -export const DESKTOP_RELEASE_DATE = '2026-09-21' +export const DESKTOP_RELEASE_DATE = '2026-09-23' const FORGEJO_ORIGIN = 'https://git.chanpaca.net' const FORGEJO_OWNER = 'yunchan' diff --git a/design.md b/design.md index 7ccbc68..78810e1 100644 --- a/design.md +++ b/design.md @@ -130,6 +130,12 @@ - 한글 문자열에 `letterSpacing` 양수 값을 직접 넣지 않는다. - 11px 미만 텍스트를 추가하지 않는다 (라틴 텔레메트리 예외 하한 9px). +## 2026-09-22 Local Flow Intelligence extension + +- 기존 **swiss-minimal × dark-instrument** 방향과 token/typography/a11y 규칙을 그대로 적용한다. Flow Radar·Edit Friction·App Quality는 색만으로 상태를 구분하지 않고 텍스트 라벨과 수치를 함께 보인다. +- Privacy Receipt은 `dl`의 정의/값 구조로 보존 기간과 실제 row count를 짝지어 읽게 한다. raw key 이벤트/키코드/콘텐츠 스트림 미저장과, 옵트인 학습 텍스트의 별도 보존 정책을 혼동시키지 않는다. `personal_phrases`는 나이 기반 자동 만료가 없고 개별 삭제·전체 삭제·동의 철회로 제거됨을 정확히 보인다. +- suggestion overlay는 좁은 폭을 유지한다. Why This Suggestion은 한 줄의 provenance와 count만 보이며 raw memory evidence를 노출하지 않는다. + ## 2026-08-24 광고 수익화 E2E RED 스위트 (652 시나리오) - 위치: `apps/desktop/tests/red/ads/` (ads-mediation 370 / admob-ssv 142 / ad-settlement 140) diff --git a/docs/map/00-index.md b/docs/map/00-index.md index 846c2fe..ab70016 100644 --- a/docs/map/00-index.md +++ b/docs/map/00-index.md @@ -2,8 +2,12 @@ > Status: ACTIVE > Last full audit: 2026-09-13 -> Last update: 2026-09-21 — LLM instruction-prompt fix (`9c2b4d4`): the custom-instruction path inserted the instruction's own wording instead of the processed result and had **never worked in any shipped release** (`v0.1.0-alpha`..`v1.4.0`, introduced `fea923d` 2026-04-05, not a regression). `llm-prompts.ts` is now the SSOT for prompt resolution and placeholder substitution, shared by `VoiceModeService` / `ChainService` / `LLM.PROCESS`. AI-04/05/06/07 are demoted to `[~]` on desktop — fixed with unit tests, but **not verified in a running app** and the four related `tests/red/*.usecase.test.ts` could not execute (`better-sqlite3` ABI). New: GAP-LLM-01 (no target-language setting), GAP-LLM-02 (this fix unverified); GAP-INFRA-06 amended (the ABI masks verification, not just dev-env switching cost); GAP-I18N-01 amended (`popup.error.default` missing in 10 locales). Earlier the same day: CAP-16 (desktop key bindings rebuilt on one `@d3ro/core/keybinding` SSOT — multiple bindings per action, mouse buttons, `HOTKEY` → `KEYBINDING` IPC group), verified on Windows by a manual run, so CAP-16 and CAP-02 are `[x]` and GAP-KEY-01 is closed. Still open: GAP-KEY-02/03, GAP-QA-02, GAP-I18N-01/02, GAP-INFRA-06, GAP-LLM-01/02; `11` §7 holds accepted design constraints (things deliberately kept, not gaps) -> Scope: entire monorepo `D:/workspace/D3ROVoice` at product version `1.4.0` (`release/product-version.json`, released 2026-09-21) +> Last update: 2026-09-23 — **v1.5.0 릴리스.** CHANGELOG `[1.5.0]` 을 확정하고 버전 SSOT를 1.5.0(android/iOS 1050000)으로 올렸으며, `site/src/release.ts` 와 `apps/web/src/lib/desktop-release.ts` 다운로드 링크를 1.5.0으로 동기화했다. 이 릴리스는 입력 인텔리전스(`INPUT-01`~`INPUT-18`, 전부 데스크톱 `[~]`), 커스텀 인스트럭션 수정(AI-04/05), `LocalLLMService` 요청별 취소·상한을 포함한다. 인증서가 없어 무서명 업데이터 게시 예외(GAP-REL-06)를 유지한다. +> +> Previous update: 2026-09-22 — **Gemma/Ollama 폭주 방어 계약을 기록했다.** 19:14:12 부팅 워밍업이 `keep_alive: 30m`으로 `gemma4:e4b`를 19:44:12까지 VRAM 3,226,342,521 bytes / context 4096으로 강제 상주시킨 것이 관측됐으며, 같은 시점 Windows GPU Engine PID 표본에는 Ollama의 활성 compute가 없었다. 즉 당시 상태는 무한 추론이 아니라 강제 residency였다. 19:11:17~19:11:58의 자동 제안 연속 생성은 기존 900 ms·12/min·5 candidates·128 tokens·한 글자 재생성 정책이 허용한 burst였다. 현재 구현 계약은 부팅 warmup 제거, `keep_alive: 2m`, 제안 600 ms / 최소 5 s 간격 / 기본 6회·hard max 12회 per min / 3 candidates / 64 tokens / 12-char growth / 8 s timeout, 그리고 요청별 취소·상한·종료 정리다. 독립 표적 검증은 6 test files / 69 tests passed / 0 failed, 변경 코드·테스트 ESLint와 `git diff --check`도 exit 0이다. Raw Ollama에서는 cold bounded 요청이 client hard timeout 15.044 s에 취소된 뒤 `/api/ps`가 비었고 `/api/version`은 80 ms에 회복했다. 명시 warmup은 HTTP 200 / 16.639 s, 후속 warm 요청은 body `options.num_predict=1`, `keep_alive='2m'`로 553 ms HTTP 200 / `done:true` / `eval_count:1` / `done_reason:length`였고 `/api/ps` expiry는 약 119.9 s였다. **19:48:59 +09:00에는 새 generate/unload/kill/retry 없이 충분히 지난 뒤 한 번의 `/api/ps`가 HTTP 200 / 45.8 ms / `{models:[]}`였고 `/api/version`은 HTTP 200 / 7.3 ms / `0.32.13`이었다.** 이는 raw API 수준의 expiry 뒤 unload 확인일 뿐 앱 재시작·GUI·실제 타이핑 증거는 아니므로 상태는 `[~]`로 유지한다 (`11` GAP-LLM-04, GAP-INPUT-06). +> +> Previous update: 2026-09-21 — **Input intelligence (입력 레메트리 + 다음 문장 제안) 신규**. 데스크톱에 입력 수집기(`InputTelemetryService`), UIA 컨텍스트 브리지(사이드카 `GET /uia/focus`), 제안 서비스(`SuggestionService`), 어렛 커 오버레이, 설정 > 입력 탭(동의·정책·주간 인사이트·개인 문구)을 추가했다. 카탈로그에 `INPUT-01`~`INPUT-08`(전부 데스크톱 `[~]` — 유닛 48건은 GREEN 이지만 **실앱 타이핑 검증 전**), 백로그에 GAP-INPUT-01~05 + GAP-LLM-03, §7 에 CONSTRAINT-INPUT-01(키 내용 미저장 — ActivityWatch 정책 채택)을 기록. 설계 근거는 조사 기반이다: 어렛은 `GetGUIThreadInfo` 가 아니라 UIA `TextPattern.GetSelection`(Chromium 은 `TextPattern2` 미구현), 타이핑 스트는 키코드 복원이 아니라 UIA 스냅샷 diff(한/일 IME 대응), 디바운스/토큰 한도는 인라인 컴플리션 실측값(Continue 350 / Tabby 250 / twinny 300 ms, 출력 64~256 토큰). 의존성: `koffi` 3.3.1(포그라운드 창 FFI), 사이드카 `uiautomation` 2.0.29 + `comtypes`. 데스크톱 유닛 총계 1409(+49), Electron ABI 실행에서 신규 실패 0건. 당시의 24.7초/4.9초 지연 설명과 `keep_alive: 30m`·부팅 워밍업 처방은 **현재 상태가 아닌 과거 가설/완화 이력**이며, 최신 운영 결론은 위 2026-09-22 항목과 `11` GAP-LLM-04를 따른다. 직전: LLM instruction-prompt fix (`9c2b4d4`): the custom-instruction path inserted the instruction's own wording instead of the processed result and had **never worked in any shipped release** (`v0.1.0-alpha`..`v1.4.0`, introduced `fea923d` 2026-04-05, not a regression). `llm-prompts.ts` is now the SSOT for prompt resolution and placeholder substitution, shared by `VoiceModeService` / `ChainService` / `LLM.PROCESS`. AI-04/05/06/07 are demoted to `[~]` on desktop — fixed with unit tests, but **not verified in a running app** and the four related `tests/red/*.usecase.test.ts` could not execute (`better-sqlite3` ABI). New: GAP-LLM-01 (no target-language setting), GAP-LLM-02 (this fix unverified); GAP-INFRA-06 amended (the ABI masks verification, not just dev-env switching cost); GAP-I18N-01 amended (`popup.error.default` missing in 10 locales). Earlier the same day: CAP-16 (desktop key bindings rebuilt on one `@d3ro/core/keybinding` SSOT — multiple bindings per action, mouse buttons, `HOTKEY` → `KEYBINDING` IPC group), verified on Windows by a manual run, so CAP-16 and CAP-02 are `[x]` and GAP-KEY-01 is closed. Still open: GAP-KEY-02/03, GAP-QA-02, GAP-I18N-01/02, GAP-INFRA-06, GAP-LLM-01/02, GAP-INPUT-01~05; `11` §7 holds accepted design constraints (things deliberately kept, not gaps) +> Scope: entire monorepo `D:/workspace/D3ROVoice` at product version `1.5.0` (`release/product-version.json`, released 2026-09-23) > Purpose: let any agent (or human) answer two questions in under a minute: > 1. **What infrastructure exists?** (build, CI, services, APIs, data, packages, deploy) > 2. **How far is each feature developed?** (per surface, with file anchors and status) diff --git a/docs/map/02-infrastructure.md b/docs/map/02-infrastructure.md index 63504c5..69a77fe 100644 --- a/docs/map/02-infrastructure.md +++ b/docs/map/02-infrastructure.md @@ -60,6 +60,8 @@ Note: root `package.json` declares npm workspaces `apps/desktop`, `apps/web`, `a | Deno | 2.8.1 | CI (`edge-functions-quality`) | | JDK | 17 | mobile CI | | Electron | 33.4.11 | `apps/desktop/electron-builder.yml` | +| koffi | 3.3.1 | `apps/desktop/package.json` — FFI into user32/kernel32 for foreground window title/pid/exe/bounds (`utils/win32-foreground.ts`). Chosen over `get-windows` because that package needs an install script this repo does not run; koffi ships N-API 8 prebuilds as optional deps. `asarUnpack` covers `koffi`/`@koromix`. | +| uiautomation / comtypes | 2.0.29 / 1.4.17 | `apps/desktop/sidecar/requirements.txt` (Windows only) — UI Automation snapshot behind `GET /uia/focus` (`sidecar/uia_bridge.py`). Dev install: `apps/desktop/sidecar/.venv/Scripts/python.exe -m pip install uiautomation comtypes`. | | React Native | 0.85 | `apps/mobile-rn/package.json` | | Turborepo | turbo.json tasks: build/typecheck/test/lint/dev | `turbo.json` | @@ -192,7 +194,7 @@ Full detail: [`09-supabase-backend.md`](./09-supabase-backend.md). | File | Purpose | |---|---| -| `release/product-version.json` | version `1.4.0`, `androidVersionCode`/`iosBuildNumber` `1041000`, releaseDate `2026-09-21`, desktop license keyId | +| `release/product-version.json` | version `1.5.0`, `androidVersionCode`/`iosBuildNumber` `1050000`, releaseDate `2026-09-23`, desktop license keyId | | `release/android-release-identity.json` | package `com.d3ro.voice`, Play app ID, app-signing SHA-256, upload cert SHA-256, evidence keyId, AdMob unit IDs | | `release/desktop-license-public.pem` | Ed25519 public key for desktop offline licenses | | `release/mobile-release-evidence-public.pem` | Ed25519 public key for mobile release evidence | diff --git a/docs/map/03-shared-packages.md b/docs/map/03-shared-packages.md index 531132d..c35671b 100644 --- a/docs/map/03-shared-packages.md +++ b/docs/map/03-shared-packages.md @@ -15,8 +15,9 @@ The canonical place for types and cross-surface logic. Both desktop and web/mobi |---|---|---| | Types | `./types` | Domain types shared across surfaces | | Errors | `./errors` | `D3ROError`, `ErrorCode` | -| IPC channels | `./ipc-channels` | `IPC_CHANNELS` object + `IPCChannel` union. **SSOT** for every desktop IPC channel (VOICE, AUDIO, STT, TTS, LLM, KEYBINDING, CONFIG, HISTORY, DICTIONARY, WINDOW, SYSTEM, STATS, MEMO, VOICE_COMMAND, CONTEXT, CHAIN, CAPTION, FILE_TRANSCRIPTION, MEETING_SUMMARY, DICTATION_TEMPLATE, VOICE_CONVERSATION, RAG, VOICE_ACTION, MEETING_MODE, MEETING_DOC_TEMPLATE, MEETING_CHAT, LICENSE, CLOUD_SYNC, INSTRUCTION, SYSTEM_AUDIO, POPUP_RESULT, POPUP_HISTORY, POPUP_COMMAND, POPUP_CAPTION, VOICE_PARTIAL, CLIPBOARD, APP, ONLINE_AUTH, ADS, SUPPORT, PAYMENT) | -| Key bindings | `./keybinding` | **SSOT** for every global shortcut in the app: `KeyBinding` (`device`/`code`/`ctrl`/`alt`/`shift`/`meta`), `KEY_CATALOG` (10 selectable groups incl. mouse), `KEYBINDING_ACTIONS` (6 rebindable actions), `bindingKey`/`normalizeBinding`/`validateBinding`/`detectBindingConflicts`/`formatBindingSegments`/`searchKeyCatalog`/`parseBindingMap`. Persisted as `AppConfig.keyBindings`. i18n keys are exposed as plain `string` so core stays independent of `@d3ro/i18n`; consumers narrow at the boundary (`asTranslationKey`) and a contract test guards the keys — accepted constraint, `11` §7 CONSTRAINT-I18N-01. Tests: `__tests__/keybinding*.test.ts` via `vitest.config.ts` (`npm run test --workspace=@d3ro/core`), 117 cases as of 2026-09-21 | +| IPC channels | `./ipc-channels` | `IPC_CHANNELS` object + `IPCChannel` union. **SSOT** for every desktop IPC channel (VOICE, AUDIO, STT, TTS, LLM, KEYBINDING, CONFIG, HISTORY, DICTIONARY, WINDOW, SYSTEM, STATS, MEMO, VOICE_COMMAND, CONTEXT, CHAIN, CAPTION, FILE_TRANSCRIPTION, MEETING_SUMMARY, DICTATION_TEMPLATE, VOICE_CONVERSATION, RAG, VOICE_ACTION, MEETING_MODE, MEETING_DOC_TEMPLATE, MEETING_CHAT, LICENSE, CLOUD_SYNC, INSTRUCTION, SYSTEM_AUDIO, POPUP_RESULT, POPUP_HISTORY, POPUP_COMMAND, POPUP_CAPTION, VOICE_PARTIAL, CLIPBOARD, APP, ONLINE_AUTH, ADS, SUPPORT, PAYMENT, INPUT_TELEMETRY, SUGGESTION, POPUP_SUGGESTION) | +| Key bindings | `./keybinding` | **SSOT** for every global shortcut in the app: `KeyBinding` (`device`/`code`/`ctrl`/`alt`/`shift`/`meta`), `KEY_CATALOG` (10 selectable groups incl. mouse), `KEYBINDING_ACTIONS` (9 rebindable actions incl. the three suggestion actions), `bindingKey`/`normalizeBinding`/`validateBinding`/`detectBindingConflicts`/`formatBindingSegments`/`searchKeyCatalog`/`parseBindingMap`. Persisted as `AppConfig.keyBindings`. i18n keys are exposed as plain `string` so core stays independent of `@d3ro/i18n`; consumers narrow at the boundary (`asTranslationKey`) and a contract test guards the keys — accepted constraint, `11` §7 CONSTRAINT-I18N-01. Tests: `__tests__/keybinding*.test.ts` via `vitest.config.ts` (`npm run test --workspace=@d3ro/core`), 117 cases as of 2026-09-21 | +| Input intelligence | `./input-intelligence` | **SSOT** for the input-telemetry and next-sentence-suggestion domain (added 2026-09-21): key classification (`classifyKeyStroke`), text metrics (`countWords`/`countSentences`/`endsSentence`/`textBeforeCaret`), `computeTypedDelta` (UIA snapshot diff — the IME-safe way to count typed text), `decideSuggestion` + `isAppExcluded` (when to request / skip / clear), `parseSuggestionCandidates`/`sanitizeSuggestionLine` (prompt-leak and prefix-echo defence), `anchorFloatingPanel` (caret-anchored overlay placement), `mergeActivityBucket`/`summarizeActivity`/`pixelsToMeters`, `extractPhrases`/`selectPhraseHints`, `INPUT_TELEMETRY_DEFAULTS`/`SUGGESTION_DEFAULTS` | | Constants | `./constants` | Shared constants | | Crypto license | `./utils/crypto-license` | Ed25519 license sign/verify (used by admin issuer + desktop verifier) | | PII | `pii-redactor`, `secure-memory` | Redaction + secure memory helpers | diff --git a/docs/map/04-desktop-app.md b/docs/map/04-desktop-app.md index 2429191..714fd73 100644 --- a/docs/map/04-desktop-app.md +++ b/docs/map/04-desktop-app.md @@ -45,7 +45,7 @@ Singleton + `EventEmitter` pattern (`getXService()` accessors). ### LLM layer | Service | Purpose | |---|---| -| `LocalLLMService` | Ollama REST (models, pull w/ progress, server start, NDJSON streaming) | +| `LocalLLMService` | Ollama REST (models, pull w/ progress, server start, NDJSON streaming). The 2026-09-22 runaway guard gives each request its own `AbortController` (external caller signals are relayed and detached on completion), requires `done` before an NDJSON stream succeeds, and clears incomplete streams. Generation/stream requests are bounded to 2048 tokens / 120 s; chat is bounded to 512 / 60 s. Ollama server spawn and polling are deduplicated, and lifecycle dispose terminates owned work. | | `PremiumLLMService` | Claude via Supabase `llm-proxy`, local fallback | | `OnlineLLMService` | JWT-authenticated .NET backend client | | `llm-prompts.ts` | **SSOT for prompt resolution, placeholder substitution, and argument placement.** `resolveSystemPrompt` (`:118`) maps an `LLMAction` to its base prompt and handles `custom` explicitly instead of dropping silently to `refine`. `renderInstructionPrompt` (`:78`) substitutes `{{text}}` / `{{userPrompt}}` / `{{targetLanguage}}` and **warns by name** for any placeholder left standing rather than letting it reach the model. `buildInstructionInvocation` (`:101`) decides where an instruction goes in `processText(text, action, targetLanguage, customPrompt)`: the instruction becomes the **system prompt** and the transcript the **text**, except for instructions that spell out `{{text}}`, which keep the old meaning for backward compatibility. `resolveTargetLanguage` (`:50`) is the one place translate targets are decided (still `English`, see `11` GAP-LLM-01). **All three LLM entry paths call the same functions** — `VoiceModeService` (`:838`), `ChainService` (`:196`), and the `LLM.PROCESS` IPC handler (`llm-handlers.ts:96`) — so no caller re-implements the rules | @@ -69,7 +69,7 @@ Singleton + `EventEmitter` pattern (`getXService()` accessors). | `FileTranscriptionService` | Audio/video file → ffmpeg → 30s chunks → STT merge (events progress, complete, error, state-changed) | | `MeetingSummaryService` | Post-caption LLM summary | | `DictationTemplateService` | Field-by-field voice form filling | -| `VoiceConversationService` | STT→LLM→TTS loop, 10-turn memory | +| `VoiceConversationService` | STT→LLM→TTS loop, 10-turn memory. Local conversations are single-flight and pass their request signal through to local LLM chat, so a voice cancel aborts its own active request rather than only changing UI/watchdog state. | | `TTSPlaybackService` | Platform TTS (macOS `say`, Windows SAPI), sentence queue | | `VoiceActionService` | Voice → LLM JSON action plan → OS execution (dangerous blocked) | @@ -79,6 +79,18 @@ Singleton + `EventEmitter` pattern (`getXService()` accessors). | `MeetingModeService` | Meeting recording: live transcript, timestamp memos, doc generation/export, diarization | | `MeetingDocTemplateService` | Meeting-doc templates (built-ins + CRUD) | +### Input intelligence (2026-09-21; Local Flow Intelligence extension 2026-09-22) +| Service / file | Purpose | +|---|---| +| `InputTelemetryService` | Global input telemetry: keystroke/click/scroll counters, mouse travel (Manhattan sum of per-axis deltas), active time, per-hour-per-app buckets flushed every 5 s; typed-text learning via UIA snapshot diffs. It extends the existing telemetry store with Flow Radar (hourly activity density + chars + edit stability; a potential-focus score, **not** a real session), Edit Friction (chars/backspaces; edits per 100 chars), App Quality (per-app suggestion total/accepted/accept rate/average latency), Privacy Receipt (actual row counts and retention), and Smart Exclusion evidence. Raw key events, key codes and content streams are never retained; separately, opt-in learned text may be retained in `typing_samples` and `personal_phrases` under the stated receipt policy. Consent is opt-in (`inputTelemetryEnabled`, `inputLearnTypedText`). | +| `UiaContextService` | Client for the sidecar UIA bridge (`GET /uia/focus`): focused text, caret rect/offset, `IsPassword`, IME composition state and `hasSelection`. The sidecar derives `hasSelection` only by comparing the TextPattern selection range Start/End endpoints: it does not call `GetText` on that range or add a separate selected-text payload (the existing focused-field text can still include a selection). Fail-closed on password/unavailable; transient failures back off for 1.5/3/6/12/30 s and permanent failures for 5 min. | +| `SuggestionService` | Next-sentence ghost text: 600 ms debounce, then `buildSuggestionPrompt` → `LocalLLMService.streamGenerate` with a caller-owned `AbortSignal` → candidate ranking and accept (via `TextInsertService`). Presentation-active includes candidates plus `generating`, `warmingUp` and `partialText`; clear/dismiss aborts the active request, invalidates its generation token, clears TTL state, then emits `cleared`/hide so a spinner-only popup closes. A successful final candidate payload resets `generating=false` and `partialText=null`. App DNA extends phrase ranking with `appName` context and a 1.75× same-app bonus; Memory Decay uses a 30-day half-life plus frequency. Why This Suggestion exposes only `local-model`/`local-memory` provenance and continuation/related/phrase/appPhrase **counts** to the overlay, never raw evidence text. Instant Recall uses local-memory provenance without another model-budget spend only when the model is unavailable, raises a non-cancellation error, truly times out, or returns empty; it never publishes after dismiss, new typing, token/context mismatch or staleness. Its runaway limits are a 5 s minimum interval, 6 requests/min default (hard-config maximum 12), 3 candidates, 64 output tokens, 12-character regeneration growth and an 8 s timeout; app startup does not warm the model and suggestion requests use `keep_alive: 2m`. | +| `global-input-hook.ts` | Ref-counted owner of the single process-wide `uiohook` hook so `KeyBindingService` and telemetry can both attach without one stopping the other. | +| `utils/win32-foreground.ts` | Foreground window title/pid/exe/bounds via `koffi` FFI into user32/kernel32 (chosen over `get-windows`, which needs an install script this repo does not run). | +| `sidecar/uia_bridge.py` | Windows UIA snapshot (`uiautomation` 2.0.29, comtypes) on a dedicated COM-initialised thread with a 1.5 s budget; never walks the UIA tree (Chrome/VS Code tree walks take 10-30 s). Warm calls measure 0-15 ms. | + +The existing `input-telemetry-handlers` and `suggestion-handlers` IPC extensions expose the receipt and the flow/suggestion summaries. Receipt reads or deletes report an IPC error when storage access fails; they do not return invented counts or claim a purge succeeded. The receipt is local-only: `input_activity`, `typing_samples`, and `suggestions` retain 30 days; `personal_phrases` has no age-based automatic expiry and is removed by individual deletion, delete-all, or consent withdrawal. Smart Exclusion never records password evidence and recommends only the current app after at least four observations, zero readable results, and a problematic ratio of at least 75%; its one-click action adds an exclusion and never auto-excludes. Shortcut Safety Audit is core `auditKeyBindingMap` applied by the Settings UI; it surfaces invalid/conflict issues while preserving the existing hold/double-press exceptions. + ### Account / infra / monetization | Service | Purpose | |---|---| @@ -104,7 +116,7 @@ Singleton + `EventEmitter` pattern (`getXService()` accessors). ## 3. IPC layer -Registry: `src/main/ipc/index.ts` calls 29 `registerXHandlers()` in fixed order. Channel SSOT: `packages/core/src/ipc-channels.ts`. +Registry: `src/main/ipc/index.ts` calls 31 `registerXHandlers()` in fixed order. Channel SSOT: `packages/core/src/ipc-channels.ts`. | Handler | Channel group(s) | |---|---| @@ -118,6 +130,7 @@ Registry: `src/main/ipc/index.ts` calls 29 `registerXHandlers()` in fixed order. | `dictionary-handlers` | DICTIONARY | | `file-transcription-handlers` | FILE_TRANSCRIPTION | | `history-handlers` | HISTORY + `stats:getSummary` | +| `input-telemetry-handlers` | INPUT_TELEMETRY | | `instruction-handlers` | INSTRUCTION | | `keybinding-handlers` | KEYBINDING | | `license-handlers` | LICENSE | @@ -129,6 +142,7 @@ Registry: `src/main/ipc/index.ts` calls 29 `registerXHandlers()` in fixed order. | `payment-handlers` | PAYMENT | | `rag-handlers` | RAG | | `stt-handlers` | STT | +| `suggestion-handlers` | SUGGESTION + POPUP_SUGGESTION | | `support-handlers` | SUPPORT | | `system-handlers` | SYSTEM | | `template-handlers` | DICTATION_TEMPLATE | @@ -142,19 +156,20 @@ The **`KEYBINDING`** group replaced the old per-action `HOTKEY` group. `HOTKEY` **`LLM.PROCESS` normalizes at the IPC boundary.** The handler runs `buildInstructionInvocation` itself when `action === 'custom'` with a `customPrompt` (`llm-handlers.ts:94-108`), so the renderer passes the **raw instruction text** and never duplicates the substitution or argument-placement rules. This is what makes `VoiceModeService`, `ChainService`, and `LLM.PROCESS` literally share one implementation. No channel or type changed for this; `LLMProcessParams` is unchanged. -Preload exposes **`window.electronAPI`** with 33 namespaces: `platform, audio, config, voice, stt, keybinding, llm (incl. premium), history, dictionary, stats, window, system, instruction, app, memo, voiceCommand, context, chain, caption, license, fileTranscription, meetingSummary, dictationTemplate, rag, voiceAction, voiceConversation, meetingMode, meetingChat, meetingDocTemplate, cloudSync, onlineAuth, ads, support, payment`. The `keybinding` bridge is 9 methods mirroring the channels above (`src/preload/index.ts:323`), replacing the 11-method `hotkey` bridge. Envelope: `IPCResult` (success/error); `app.onDataChanged` is the global refresh channel. +Preload exposes **`window.electronAPI`** with 35 namespaces: `platform, audio, config, voice, stt, keybinding, llm (incl. premium), history, dictionary, stats, window, system, instruction, app, memo, voiceCommand, context, chain, caption, license, fileTranscription, meetingSummary, dictationTemplate, rag, voiceAction, voiceConversation, meetingMode, meetingChat, meetingDocTemplate, cloudSync, onlineAuth, ads, support, payment, inputTelemetry, suggestion`. The `keybinding` bridge is 9 methods mirroring the channels above (`src/preload/index.ts:323`), replacing the 11-method `hotkey` bridge. Envelope: `IPCResult` (success/error); `app.onDataChanged` is the global refresh channel. --- ## 4. Windows & popups -`windows/WindowManager.ts` creates 6 windows: main (borderless, custom TitleBar; macOS `hiddenInset`), recording-tip, result-popup, history-popup, command-popup, caption-overlay. Injects popup theme CSS + i18n strings; 2-phase resize. `windows/TrayManager.ts` — tray icon + menu + double-click show. +`windows/WindowManager.ts` creates 7 windows: main (borderless, custom TitleBar; macOS `hiddenInset`), recording-tip, result-popup, history-popup, command-popup, caption-overlay, suggestion-overlay. Injects popup theme CSS + i18n strings; 2-phase resize. `windows/TrayManager.ts` — tray icon + menu + double-click show. **Popup invariants** (each shipped broken once — do not regress): - 팝업 HTML의 스크립트는 반드시 `