From cdeeff4c1194c286641ce2f253d28b80c0438898 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Sat, 24 Oct 2026 13:15:29 +0900 Subject: [PATCH] fix(suggestion): insert accepted suggestions reliably and learn typing as whole sentences Acceptance waited for modifiers using only the uiohook-tracked set; a missed keyup made it time out after 1.5 s and paste while Ctrl/Alt counted as held, so nothing was inserted. Trust GetAsyncKeyState instead, and suppress new suggestions while an accept is in flight. Typed deltas were saved per snapshot (every 0.4-0.8 s), storing mid-word fragments. Buffer them per field and save one diff on idle, sentence end, send or focus change; drop short pieces, URLs, trailing jamo and truncated emoji. --- .../main/services/InputTelemetryService.ts | 142 ++++++++++++++++-- .../src/main/services/SuggestionService.ts | 37 ++++- .../src/main/services/modifier-state.ts | 17 ++- .../desktop/src/main/utils/win32-key-state.ts | 59 ++++++++ .../input-telemetry-redteam-r1-1.test.ts | 54 +++++++ .../main/services/modifier-state.test.ts | 51 +++++++ docs/map/10-feature-catalog.md | 4 +- 7 files changed, 341 insertions(+), 23 deletions(-) create mode 100644 apps/desktop/src/main/utils/win32-key-state.ts create mode 100644 apps/desktop/tests/main/services/modifier-state.test.ts diff --git a/apps/desktop/src/main/services/InputTelemetryService.ts b/apps/desktop/src/main/services/InputTelemetryService.ts index c791f75..340bee9 100644 --- a/apps/desktop/src/main/services/InputTelemetryService.ts +++ b/apps/desktop/src/main/services/InputTelemetryService.ts @@ -138,6 +138,14 @@ const MAX_SAMPLE_CHARS = 500 * settle·UIA 조회(최대 1.5초)를 거쳐 온다 — 그 합을 넉넉히 덮는다. */ const PROGRAMMATIC_INSERT_TTL_MS = 10_000 +/** + * 타이핑 학습은 스냅샷 diff(0.4~0.8초마다)를 바로 저장하지 않고 필드별로 모았다가 문장 단위로 저장한다. + * 바로 저장하면 "찬받고 초콜릿을 받은 아이. 아빠" / "도 잘했으니까…" 처럼 단어 중간에서 잘린 + * 조각이 표본·문구로 쌓여 제안 힌트를 오염시켰다(실측 2026-10-24 DB). + */ +const LEARN_IDLE_FLUSH_MS = 2000 +/** diff 붙여넣기 판정(200자)에 걸리기 전에 끊어 저장한다. */ +const LEARN_MAX_PENDING_CHARS = 150 const RETENTION_PRUNE_INTERVAL_MS = 6 * 60 * 60 * 1000 const MAX_READABILITY_APPS = 64 @@ -233,6 +241,14 @@ class InputTelemetryService extends EventEmitter { private _appReadability = new Map() /** 곧(또는 방금) 프로그램이 삽입한 텍스트 — 다음 diff 에서 "타이핑" 으로 학습하지 않는다. */ private _expectedInserts: Array<{ id: number; text: string; expiresAt: number }> = [] + /** 아직 저장하지 않은 타이핑 구간 — 같은 필드에서 baseline → latest 로 바뀐 부분을 한 번에 학습한다. */ + private _pendingLearn: { + baseline: string + latest: string + appName: string | null + windowTitle: string | null + } | null = null + private _learnFlushTimer: ReturnType | null = null private _expectedInsertSeq = 0 /** 통계에서 뺄 프로그램 키 남은 수와 유효 시각 */ private _programmaticKeys = 0 @@ -383,6 +399,7 @@ class InputTelemetryService extends EventEmitter { this._settleTimer = null } this.setSuggestionPresentationActive(false) + this.flushPendingLearning() this._flush() this._lastSnapshot = null this._lastSnapshotKey = '' @@ -833,28 +850,103 @@ class InputTelemetryService extends EventEmitter { this._lastSnapshot = snapshot this._lastSnapshotKey = key - // 필드가 바뀌면 이전 텍스트와 diff 할 수 없다 — 기준선만 저장하고 통계는 건너뛴다. - if (focusChanged || !previous) return + // 필드가 바뀌면 이전 텍스트와 diff 할 수 없다 — 모아 둔 학습을 저장하고 기준선만 잡는다. + if (focusChanged || !previous) { + this.flushPendingLearning() + return + } - const delta = computeTypedDelta(previous.text, snapshot.text, { - keepText: configGet('inputLearnTypedText') === true - }) - if (delta.replaced) return - if (delta.insertedChars === 0 && delta.insertedWords === 0) return + const keepText = configGet('inputLearnTypedText') === true + const delta = computeTypedDelta(previous.text, snapshot.text, { keepText }) + if (delta.replaced) { + // 붙여넣기 — 직접 친 글이 아니다. 그 앞까지 친 것만 저장하고 새로 모은다. + this._flushPendingLearningAt(previous.text) + return + } + if (delta.insertedChars === 0 && delta.insertedWords === 0) { + // 지우기만 했다. 칸이 비었거나 기준선보다 짧아졌으면(메시지 전송 등) 비우기 직전까지가 친 글이다. + const pending = this._pendingLearn + if (pending && (snapshot.text.length === 0 || snapshot.text.length < pending.baseline.length)) { + this._flushPendingLearningAt(previous.text) + } else if (pending) { + pending.latest = snapshot.text + } + return + } const bucket = this._bucket() bucket.chars += delta.insertedChars bucket.words += delta.insertedWords bucket.sentences += delta.insertedSentences - const typedText = delta.insertedText ? this._withoutExpectedInsert(delta.insertedText) : '' - if (typedText) { - this._learnText(typedText, { - appName: this._foreground.appName, - windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle, - source: 'typed' - }) + if (!delta.insertedText) return + const typedText = this._withoutExpectedInsert(delta.insertedText) + if (typedText !== delta.insertedText) { + // 프로그램 삽입(제안 수락 등)이 섞였다 — 그 앞까지 모은 것을 저장하고, 이번에 직접 친 + // 나머지만 바로 학습한다. 다음 diff 부터는 삽입 뒤 텍스트를 기준선으로 다시 모은다. + this._flushPendingLearningAt(previous.text) + if (typedText) { + this._learnText(typedText, { + appName: this._foreground.appName, + windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle, + source: 'typed' + }) + } + return } + + if (!this._pendingLearn) { + this._pendingLearn = { + baseline: previous.text, + latest: snapshot.text, + appName: this._foreground.appName, + windowTitle: snapshot.windowTitle ?? this._foreground.windowTitle + } + } else { + this._pendingLearn.latest = snapshot.text + } + + const grown = snapshot.text.length - this._pendingLearn.baseline.length + if (/[.!?…。?!\n]\s*$/u.test(snapshot.text) || grown >= LEARN_MAX_PENDING_CHARS) { + this.flushPendingLearning() + return + } + this._scheduleLearnFlush() + } + + private _scheduleLearnFlush(): void { + if (this._learnFlushTimer) clearTimeout(this._learnFlushTimer) + this._learnFlushTimer = setTimeout(() => { + this._learnFlushTimer = null + this.flushPendingLearning() + }, LEARN_IDLE_FLUSH_MS) + } + + /** 모아 둔 타이핑 구간을 지금 학습한다 (멈춤·문장 끝·포커스 이동·종료 시). */ + flushPendingLearning(): void { + this._flushPendingLearningAt(null) + } + + private _flushPendingLearningAt(latestOverride: string | null): void { + const pending = this._pendingLearn + this._discardPendingLearning() + if (!pending || configGet('inputLearnTypedText') !== true) return + // baseline → latest 를 한 번에 diff 한다 — 친 뒤 지운 오타·IME 가 바꾼 글자는 최종 결과만 남는다. + const delta = computeTypedDelta(pending.baseline, latestOverride ?? pending.latest, { keepText: true }) + if (delta.replaced || !delta.insertedText) return + this._learnText(delta.insertedText, { + appName: pending.appName, + windowTitle: pending.windowTitle, + source: 'typed' + }) + } + + private _discardPendingLearning(): void { + if (this._learnFlushTimer) { + clearTimeout(this._learnFlushTimer) + this._learnFlushTimer = null + } + this._pendingLearn = null } /** @@ -944,10 +1036,13 @@ class InputTelemetryService extends EventEmitter { const text = sanitizeSample( raw .split(/\r?\n/u) - .filter((line) => isLearnablePhrase(line)) + .map(stripTrailingJamo) + .filter((line) => isLearnablePhrase(line) && !isUrlLike(line)) .join('\n') ) if (!text) return + // 직접 친 글이 너무 짧으면 문체 표본이 아니다("처음", "pow") — 음성·수락 문장은 이미 문장 단위다. + if (meta.source === 'typed' && text.replace(/\s/gu, '').length < MIN_TYPED_SAMPLE_CHARS) return try { const db = getDatabase() @@ -1481,6 +1576,7 @@ class InputTelemetryService extends EventEmitter { this._sessionTotals = emptyActivityBucket() this._lastSnapshot = null this._lastSnapshotKey = '' + this._discardPendingLearning() logger.info('입력 텔레메트리 데이터 전체 삭제') } catch (error) { logger.warn(`입력 데이터 삭제 실패: ${error instanceof Error ? error.message : String(error)}`) @@ -1624,12 +1720,28 @@ function formatLocalDate(date: Date): string { return `${year}-${month}-${day}` } +const MIN_TYPED_SAMPLE_CHARS = 4 + +/** 주소·쿼리 문자열 한 줄은 문체 표본이 아니다. */ +function isUrlLike(line: string): boolean { + const trimmed = line.trim() + if (/^(https?:\/\/|www\.)\S*$/iu.test(trimmed)) return true + return !/\s/u.test(trimmed) && /[/=&?#]/u.test(trimmed) +} + +/** 조합이 덜 끝난 채 잡힌 끝 자모("아냐 ㅇ")를 뗀다. */ +function stripTrailingJamo(line: string): string { + return line.replace(/\s*[\u3131-\u318e]+$/u, '') +} + /** 제어문자 제거 + 공백 정규화. 2자 미만은 버린다. */ function sanitizeSample(raw: string): string { let stripped = '' for (const char of raw) { const code = char.codePointAt(0) ?? 0 // 제어문자(개행/탭 포함)는 공백으로 바다 — 학습 표본에 제어문자를 남기지 않는다. + // 사용 영역(PUA)·짝 잃은 서로게이트는 이모지가 16비트로 잘려 들어온 흔적이다(🍫 → U+F36B) — 버린다. + if ((code >= 0xe000 && code <= 0xf8ff) || (code >= 0xd800 && code <= 0xdfff)) continue stripped += code < 0x20 || code === 0x7f ? ' ' : char } const cleaned = stripped.replace(/\s+/gu, ' ').trim() diff --git a/apps/desktop/src/main/services/SuggestionService.ts b/apps/desktop/src/main/services/SuggestionService.ts index 36b6a1d..4a89e91 100644 --- a/apps/desktop/src/main/services/SuggestionService.ts +++ b/apps/desktop/src/main/services/SuggestionService.ts @@ -304,6 +304,8 @@ export class SuggestionService extends EventEmitter { private _lastRequestedPrefix = '' /** 사용자가 X 로 닫은 뒤 재표시를 막는 시각 (ms) */ private _userDismissedUntil = 0 + /** 수락 삽입이 진행 중이다(수정자 키 떼기 대기 포함) — 그사이 같은 글로 새 제안 창이 다시 뜨지 않게 한다. */ + private _acceptInFlight = false /** 스트리밍 누적 텍스트 */ private _partialText = '' /** 마지막 스트리밍 방출 시각 (IPC 과다 방출 방지) */ @@ -646,6 +648,7 @@ export class SuggestionService extends EventEmitter { this._lastContext = context const now = Date.now() this._lastContextAt = now + if (this._acceptInFlight) return this._watchdog() const config = this.readPolicyConfig() @@ -1939,6 +1942,34 @@ export class SuggestionService extends EventEmitter { // 단축키로 수락했다면 사용자가 아직 Ctrl+Alt 를 쥐고 있다. 그대로 Ctrl+V 를 보내면 // 대상 앱에 Ctrl+Alt+V 가 들어가 붙여넣기가 되지 않는다(실측) — 뗄 때까지 기다린다. + this._acceptInFlight = true + try { + return await this._insertAccepted({ + text, + deleteCount, + composing, + appName, + windowTitle, + targetWindow, + sessionRecordId, + candidateText: candidate.text + }) + } finally { + this._acceptInFlight = false + } + } + + private async _insertAccepted(params: { + text: string + deleteCount: number + composing: boolean + appName: string | null + windowTitle: string | null + targetWindow: number | null + sessionRecordId: string | null + candidateText: string + }): Promise { + const { text, deleteCount, composing, appName, windowTitle, targetWindow, sessionRecordId, candidateText } = params const modifierWaitStartedAt = Date.now() if (!(await waitForModifiersReleased())) { logger.warn('수정자 키가 떼어지지 않은 채 제안을 삽입한다 (1.5초 초과)') @@ -1995,12 +2026,12 @@ export class SuggestionService extends EventEmitter { // 세션 행을 알면 그 행을 수락으로 표시한다 — 텍스트로 찾으면 로컬 기억의 2번째 이후 후보는 // 행이 없어 사라지거나 같은 문구의 옛 행(다른 앱·다른 날)이 수락으로 잡혔다. if (sessionRecordId && this._repository.markSessionAccepted) { - this._repository.markSessionAccepted(sessionRecordId, candidate.text) + this._repository.markSessionAccepted(sessionRecordId, candidateText) } else { - this._repository.markAccepted(candidate.text) + this._repository.markAccepted(candidateText) } // 수락한 문장은 사용자 문체의 확실한 표본이다 (학습 동의 시에만 저장됨). - this._learning.recordAccepted(candidate.text, { appName, windowTitle }) + this._learning.recordAccepted(candidateText, { appName, windowTitle }) return { ok: true } } diff --git a/apps/desktop/src/main/services/modifier-state.ts b/apps/desktop/src/main/services/modifier-state.ts index 80634ad..3b9a68f 100644 --- a/apps/desktop/src/main/services/modifier-state.ts +++ b/apps/desktop/src/main/services/modifier-state.ts @@ -9,6 +9,7 @@ import { uIOhook, type UiohookKeyboardEvent } from 'uiohook-napi' import { acquireGlobalInputHook } from './global-input-hook' +import { isAnyModifierPhysicallyDown } from '../utils/win32-key-state' /** uiohook 키 코드 (UiohookKey.Ctrl/CtrlRight/Alt/AltRight/Shift/ShiftRight/Meta/MetaRight) */ const MODIFIER_KEYCODES: ReadonlySet = new Set([29, 3613, 56, 3640, 42, 54, 3675, 3676]) @@ -44,12 +45,22 @@ export function trackModifierKeys(): void { * 시간 안에 떼지 않으면(떼는 이벤트를 놓친 경우 포함) 추적 상태를 비우고 false 를 * 돌려준다 — 호출자는 그래도 진행한다(영원히 막히면 안 된다). */ -export async function waitForModifiersReleased(timeoutMs = 1500): Promise { +export async function waitForModifiersReleased( + timeoutMs = 1500, + readPhysicalState: () => boolean | null = isAnyModifierPhysicallyDown +): Promise { + // OS 가 실제 상태를 알려 주면 그것이 정본이다 — 훅 추적은 떼는 이벤트를 놓치면 영영 눌린 채로 남는다. + const anyHeld = (): boolean => { + const physical = readPhysicalState() + if (physical === null) return held.size > 0 + if (!physical) held.clear() + return physical + } const deadline = Date.now() + timeoutMs - while (held.size > 0 && Date.now() < deadline) { + while (anyHeld() && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 15)) } - if (held.size === 0) return true + if (!anyHeld()) return true held.clear() return false } diff --git a/apps/desktop/src/main/utils/win32-key-state.ts b/apps/desktop/src/main/utils/win32-key-state.ts new file mode 100644 index 0000000..a7e104c --- /dev/null +++ b/apps/desktop/src/main/utils/win32-key-state.ts @@ -0,0 +1,59 @@ +// src/main/utils/win32-key-state.ts +// +// koffi(FFI)로 OS 가 보는 실제 수정자 키 눌림 상태를 읽는다 (GetAsyncKeyState). +// +// 왜 필요한가: uiohook 은 떼는 이벤트를 놓칠 수 있다(관리자 권한 창에서 뗀 키, 보안 데스크톱 등). +// 그러면 추적 집합에 키가 눌린 채 남아 제안 수락이 매번 1.5초 시간 초과까지 기다렸고, +// 그 사이 사용자가 실제로 쥐고 있던 키와 섞여 Ctrl+V 가 Ctrl+Alt+V 로 들어가 삽입되지 않았다 +// (실측 2026-10-24: 대기 1505ms 후 "삽입 성공" 로그, 입력창 길이는 그대로). +// +// 실패(비 Windows, 모듈 로드 실패)는 null — 호출자는 uiohook 추적으로 돌아간다. + +import { getLogger } from '../services/LoggerService' + +const logger = getLogger('win32-key-state') + +/** VK_SHIFT, VK_CONTROL, VK_MENU(Alt), VK_LWIN, VK_RWIN */ +const MODIFIER_VKS = [0x10, 0x11, 0x12, 0x5b, 0x5c] as const + +type GetAsyncKeyState = (vk: number) => number + +let getAsyncKeyState: GetAsyncKeyState | null = null +let loadFailed = false + +function load(): GetAsyncKeyState | null { + if (getAsyncKeyState) return getAsyncKeyState + if (loadFailed) return null + if (process.platform !== 'win32') { + loadFailed = true + return null + } + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const koffi = require('koffi') as { + load: (lib: string) => { func: (signature: string) => GetAsyncKeyState } + } + getAsyncKeyState = koffi.load('user32.dll').func('int16_t GetAsyncKeyState(int vKey)') + return getAsyncKeyState + } catch (error) { + loadFailed = true + logger.warn( + `GetAsyncKeyState 사용 불가 — 수정자 키는 입력 훅 추적으로만 판단 (${ + error instanceof Error ? error.message : String(error) + })` + ) + return null + } +} + +/** 수정자 키(Ctrl/Alt/Shift/Win) 중 하나라도 지금 물리적으로 눌려 있는지. 알 수 없으면 null. */ +export function isAnyModifierPhysicallyDown(): boolean | null { + const read = load() + if (!read) return null + try { + // 최상위 비트(0x8000)가 "지금 눌림" — int16 이라 음수로 온다. + return MODIFIER_VKS.some((vk) => (read(vk) & 0x8000) !== 0) + } catch { + return null + } +} diff --git a/apps/desktop/tests/main/services/input-telemetry-redteam-r1-1.test.ts b/apps/desktop/tests/main/services/input-telemetry-redteam-r1-1.test.ts index b8dc8c1..9f6d4c8 100644 --- a/apps/desktop/tests/main/services/input-telemetry-redteam-r1-1.test.ts +++ b/apps/desktop/tests/main/services/input-telemetry-redteam-r1-1.test.ts @@ -84,6 +84,7 @@ describe('버그 3: 프로그램이 삽입한 텍스트를 타이핑으로 다 telemetry.expectProgrammaticInsert(ACCEPTED) internal._applyTypedDelta(snapshot(`안녕하세요 여러분 ${ACCEPTED}`)) + telemetry.flushPendingLearning() expect(typedSamples()).toEqual([]) }) @@ -95,6 +96,7 @@ describe('버그 3: 프로그램이 삽입한 텍스트를 타이핑으로 다 telemetry.expectProgrammaticInsert(ACCEPTED) internal._applyTypedDelta(snapshot(`시작 ${ACCEPTED} 확인 부탁드립니다`)) + telemetry.flushPendingLearning() expect(typedSamples()).toEqual(['확인 부탁드립니다']) }) @@ -107,6 +109,7 @@ describe('버그 3: 프로그램이 삽입한 텍스트를 타이핑으로 다 internal._applyTypedDelta(snapshot(`첫 줄 ${ACCEPTED}`)) internal._applyTypedDelta(snapshot(`첫 줄 ${ACCEPTED}\n${ACCEPTED}`)) + telemetry.flushPendingLearning() expect(typedSamples()).toContain(ACCEPTED) }) @@ -118,6 +121,7 @@ describe('버그 3: 프로그램이 삽입한 텍스트를 타이핑으로 다 cancel() internal._applyTypedDelta(snapshot(`앞 ${ACCEPTED}`)) + telemetry.flushPendingLearning() expect(typedSamples()).toEqual([ACCEPTED]) }) @@ -130,10 +134,60 @@ describe('버그 3: 프로그램이 삽입한 텍스트를 타이핑으로 다 internal._applyTypedDelta(snapshot(`x ${ACCEPTED.slice(0, 8)}`)) internal._applyTypedDelta(snapshot(`x ${ACCEPTED}`)) + telemetry.flushPendingLearning() expect(typedSamples()).toEqual([]) }) }) +describe('타이핑 학습은 스냅샷 조각이 아니라 문장 단위로 저장한다', () => { + it('0.4~0.8초 간격 스냅샷이 단어 중간에서 끊겨도 하나의 표본으로 합친다', () => { + const telemetry = getInputTelemetryService() + const internal = telemetry as unknown as TelemetryInternal + internal._applyTypedDelta(snapshot('')) + internal._applyTypedDelta(snapshot('칭찬받고 초콜릿을 받은 아이. 아빠')) + internal._applyTypedDelta(snapshot('칭찬받고 초콜릿을 받은 아이. 아빠도 잘했으니까 하나 줘야죠')) + expect(typedSamples()).toEqual([]) + + telemetry.flushPendingLearning() + expect(typedSamples()).toEqual(['칭찬받고 초콜릿을 받은 아이. 아빠도 잘했으니까 하나 줘야죠']) + }) + + it('친 뒤 지운 오타는 남기지 않고, 칸을 비우면(전송) 비우기 직전 글을 저장한다', () => { + const telemetry = getInputTelemetryService() + const internal = telemetry as unknown as TelemetryInternal + internal._applyTypedDelta(snapshot('')) + internal._applyTypedDelta(snapshot('내일 회의 갑ㅅ')) + internal._applyTypedDelta(snapshot('내일 회의 ')) + internal._applyTypedDelta(snapshot('내일 회의 같이 가자')) + internal._applyTypedDelta(snapshot('')) + + expect(typedSamples()).toEqual(['내일 회의 같이 가자']) + }) + + it('문장 끝 부호에서 바로 저장한다', () => { + const internal = getInputTelemetryService() as unknown as TelemetryInternal + internal._applyTypedDelta(snapshot('')) + internal._applyTypedDelta(snapshot('확인했습니다.')) + + expect(typedSamples()).toEqual(['확인했습니다.']) + }) + + it('짧은 조각·주소·덜 조합된 자모·깨진 이모지는 거른다', () => { + const telemetry = getInputTelemetryService() + const internal = telemetry as unknown as TelemetryInternal + for (const text of ['처음', 'https://youtube.com/shorts/32P7siSb5LY', 'd=ud&', '아냐 ㅇ']) { + internal._applyTypedDelta(snapshot('')) + internal._applyTypedDelta(snapshot(text)) + telemetry.flushPendingLearning() + } + internal._applyTypedDelta(snapshot('')) + internal._applyTypedDelta(snapshot('마음은 고마운데 아빠 표정이 ')) + telemetry.flushPendingLearning() + + expect(typedSamples()).toEqual(['마음은 고마운데 아빠 표정이']) + }) +}) + describe('버그 9: 문구 삭제는 그 문장의 그래프 엣지도 지운다', () => { const ANCHOR = '오늘 회의 결과를 공유드립니다' const PRIVATE = '계좌 비밀번호는 따로 보내드리겠습니다' diff --git a/apps/desktop/tests/main/services/modifier-state.test.ts b/apps/desktop/tests/main/services/modifier-state.test.ts new file mode 100644 index 0000000..96285b9 --- /dev/null +++ b/apps/desktop/tests/main/services/modifier-state.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest' + +type KeyListener = (event: { keycode: number }) => void +const listeners: Record = { keydown: [], keyup: [] } + +vi.mock('uiohook-napi', () => ({ + uIOhook: { on: (name: string, listener: KeyListener) => listeners[name]?.push(listener) } +})) +vi.mock('../../../src/main/services/global-input-hook', () => ({ acquireGlobalInputHook: () => () => {} })) +vi.mock('../../../src/main/utils/win32-key-state', () => ({ isAnyModifierPhysicallyDown: () => null })) + +const { trackModifierKeys, waitForModifiersReleased, forgetHeldModifiers } = await import( + '../../../src/main/services/modifier-state' +) + +const CTRL = 29 + +function press(keycode: number): void { + for (const listener of listeners.keydown) listener({ keycode }) +} + +describe('waitForModifiersReleased', () => { + trackModifierKeys() + + it('훅이 떼는 이벤트를 놓쳐도 OS 가 떼었다고 하면 바로 진행한다', async () => { + forgetHeldModifiers() + press(CTRL) // keyup 을 놓친 상태 + const startedAt = Date.now() + expect(await waitForModifiersReleased(1500, () => false)).toBe(true) + expect(Date.now() - startedAt).toBeLessThan(100) + // 남은 추적 상태도 비워 다음 수락이 다시 기다리지 않는다 + expect(await waitForModifiersReleased(50, () => null)).toBe(true) + }) + + it('OS 가 아직 눌림이라 하면 뗄 때까지 기다린다', async () => { + forgetHeldModifiers() + let down = true + setTimeout(() => { + down = false + }, 60) + const startedAt = Date.now() + expect(await waitForModifiersReleased(1500, () => down)).toBe(true) + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(50) + }) + + it('OS 상태를 모르면 훅 추적으로 판단하고 시간 초과 시 false', async () => { + forgetHeldModifiers() + press(CTRL) + expect(await waitForModifiersReleased(40, () => null)).toBe(false) + }) +}) diff --git a/docs/map/10-feature-catalog.md b/docs/map/10-feature-catalog.md index fef0bf9..cc40a1c 100644 --- a/docs/map/10-feature-catalog.md +++ b/docs/map/10-feature-catalog.md @@ -168,9 +168,9 @@ end-to-end behaviour has **not been verified by typing in a real app** (`11` GAP | INPUT-01 | Keyboard/mouse telemetry capture (opt-in) | [~] | [-] | [-] | [-] | `InputTelemetryService` — keystroke/click/scroll counters, mouse travel as the Manhattan sum of per-axis pixel deltas, active time, per-hour×app buckets flushed every 5 s. Key **contents** are never stored (ActivityWatch `aw-watcher-input` data-minimisation policy, adopted deliberately — see `11` §7). Hook ownership is ref-counted so `KeyBindingService` keeps working (`global-input-hook.ts`). | | INPUT-02 | Foreground-app attribution | [~] | [-] | [-] | [-] | `utils/win32-foreground.ts` via `koffi` FFI (title/pid/exe/bounds), sampled at most 1×/s. `get-windows` was rejected: it needs an install script this repo does not run. | | INPUT-03 | Weekly input insights | [~] | [-] | [-] | [-] | `INPUT_TELEMETRY.getSummary` aggregates `input_activity` into totals, daily series, top hours and top apps; rendered in Settings → Input and as a dashboard card. Daily average mouse travel is converted px → m using the display scale factor. | -| INPUT-04 | Typed-text learning (UIA, password-excluded) | [~] | [-] | [-] | [-] | Text is read from the focused field via `GET /uia/focus` (sidecar UIA bridge) and diffed longest-common-prefix/suffix, so **IME-committed Hangul/kana is counted correctly** — keycodes cannot reconstruct CJK text. UIA sends `hasSelection` only, derived by TextPattern range Start/End comparison without calling `GetText` on the selection range or adding a selected-text payload; the existing focused-field text can still include a selection. A non-collapsed selection immediately clears suggestions as `selection-active`. `IsPassword` is checked before any read (fail-closed); IME composition suppresses both stats and suggestions. **2026-09-24:** text equal to the control name (empty-field placeholder, e.g. "메시지 입력") is treated as empty (`withoutPlaceholderText`); a suggestion requires `editedSinceFocus` + typing within `recentTypingWindowMs` (8 s), otherwise `not-typing` — clicking into a pre-filled field no longer triggers suggestions. | +| INPUT-04 | Typed-text learning (UIA, password-excluded) | [~] | [-] | [-] | [-] | Text is read from the focused field via `GET /uia/focus` (sidecar UIA bridge) and diffed longest-common-prefix/suffix, so **IME-committed Hangul/kana is counted correctly** — keycodes cannot reconstruct CJK text. UIA sends `hasSelection` only, derived by TextPattern range Start/End comparison without calling `GetText` on the selection range or adding a selected-text payload; the existing focused-field text can still include a selection. A non-collapsed selection immediately clears suggestions as `selection-active`. `IsPassword` is checked before any read (fail-closed); IME composition suppresses both stats and suggestions. **2026-09-24:** text equal to the control name (empty-field placeholder, e.g. "메시지 입력") is treated as empty (`withoutPlaceholderText`); a suggestion requires `editedSinceFocus` + typing within `recentTypingWindowMs` (8 s), otherwise `not-typing` — clicking into a pre-filled field no longer triggers suggestions. **2026-10-24:** typed deltas are no longer saved per snapshot (that stored mid-word fragments every 0.4–0.8 s). They are buffered per field and saved as one baseline→latest diff on 2 s idle, sentence-ending punctuation, 150-char growth, field clear (send), focus change or stop. Typed samples under 4 non-space chars, URL/query-like lines, trailing incomplete jamo and PUA/lone-surrogate chars (emoji truncated to 16 bits upstream, root cause unconfirmed) are dropped. Rows already in the DB are not cleaned up. | | INPUT-05 | Personal phrase corpus (typed + voice) | [~] | [-] | [-] | [-] | Sentence-level phrases from typed text and from voice history (`HistoryService.create` feeds `recordExternalText`), ranked by frequency/recency as prompt hints; users can delete individual phrases. **2026-09-24:** typed text from `LEARNING_EXCLUDED_APPS` (terminals, code editors, Agent Switchboard) is never learned, and every phrase must pass `isLearnablePhrase` (no box/block/geometric glyphs, ≥60 % letters). `_pruneUnlearnableCorpus` re-applies both rules to existing phrases/samples/edges on start and every retention cycle (first run on the author machine removed 69 phrases / 91 samples of terminal status lines and agent chats). | -| INPUT-06 | Next-sentence suggestion (ghost text) | [~] | [-] | [-] | [-] | `SuggestionService` + `buildSuggestionPrompt` (`llm-prompts.ts` SSOT, instruction stays in the system prompt). The 2026-09-22 guard contract is 600 ms debounce, ≥5 s between requests, 6 requests/min by default (hard-config maximum 12), 3 candidates, 64 output tokens, 12-character growth before regeneration, 8 s request timeout and `keep_alive: 2m`; boot warmup is removed. Each request has its own cancellation signal. Presentation-active includes candidates, `generating`, `warmingUp` and `partialText`; clear/dismiss aborts, invalidates the generation token, clears TTL state and emits `cleared`/hide, and a final success resets `generating=false`/`partialText=null`. Focused evidence for the lifecycle and Windows-child-process change: five test files / 80 tests passed; desktop typecheck/lint, Python `py_compile`, and `git diff --check` exited 0 (core 131-test pass was independently verified earlier). Status remains `[~]`: this is not app-restart, GUI overlay, or real automatic-typing evidence. **2026-09-24 (supersedes the numbers above):** `keep_alive: 10m`, warm-up on boot, on Ollama reconnect and whenever the model is presumed cold (cold reload measured 11.4 s > 8 s timeout); a session asks for 1 candidate, then a sequential fill loop appends one unique candidate at a time up to `maxCandidatesTotal` 12 (avoid-list in the prompt, stops after 2 empty fills); only the first request consumes rate/daily budget; stale aborts refund the budget; any further typing ends the session (`matchesSessionPrefix`, IME last-syllable tolerant); prompt forbids assistant-style questions. Terminals (`TERMINAL_APPS`) never get suggestions. | +| INPUT-06 | Next-sentence suggestion (ghost text) | [~] | [-] | [-] | [-] | `SuggestionService` + `buildSuggestionPrompt` (`llm-prompts.ts` SSOT, instruction stays in the system prompt). The 2026-09-22 guard contract is 600 ms debounce, ≥5 s between requests, 6 requests/min by default (hard-config maximum 12), 3 candidates, 64 output tokens, 12-character growth before regeneration, 8 s request timeout and `keep_alive: 2m`; boot warmup is removed. Each request has its own cancellation signal. Presentation-active includes candidates, `generating`, `warmingUp` and `partialText`; clear/dismiss aborts, invalidates the generation token, clears TTL state and emits `cleared`/hide, and a final success resets `generating=false`/`partialText=null`. Focused evidence for the lifecycle and Windows-child-process change: five test files / 80 tests passed; desktop typecheck/lint, Python `py_compile`, and `git diff --check` exited 0 (core 131-test pass was independently verified earlier). Status remains `[~]`: this is not app-restart, GUI overlay, or real automatic-typing evidence. **2026-09-24 (supersedes the numbers above):** `keep_alive: 10m`, warm-up on boot, on Ollama reconnect and whenever the model is presumed cold (cold reload measured 11.4 s > 8 s timeout); a session asks for 1 candidate, then a sequential fill loop appends one unique candidate at a time up to `maxCandidatesTotal` 12 (avoid-list in the prompt, stops after 2 empty fills); only the first request consumes rate/daily budget; stale aborts refund the budget; any further typing ends the session (`matchesSessionPrefix`, IME last-syllable tolerant); prompt forbids assistant-style questions. Terminals (`TERMINAL_APPS`) never get suggestions. **2026-10-24 accept fix:** `waitForModifiersReleased` now trusts OS `GetAsyncKeyState` (`utils/win32-key-state.ts`) over the uiohook-tracked set. A missed keyup made acceptance wait the full 1.5 s and then paste Ctrl+V while modifiers were still considered held; in the logged case the field length stayed 15 after "inserted" (6 chars). New suggestions are also suppressed while an accept is in flight, so the overlay no longer reappears during the wait. | | INPUT-07 | Caret-anchored suggestion overlay | [~] | [-] | [-] | [-] | `suggestion-overlay` popup placed by `anchorFloatingPanel` (caret → element → cursor fallback, flip above when the caret is near the bottom, clamped to the work area). Non-focusable; click-through unless `suggestionOverlayInteractive`. While actually visible it continues periodic UIA validation after 5 s and revalidates 120 ms after mouse-up; unavailable/non-editable focus or a non-collapsed selection aborts and hides it. X first hides the renderer panel, then main IPC directly hides `BrowserWindow` and dismisses the service, so late tokened results cannot revive it. Accept/next/prev/dismiss are four global key bindings (`suggestion-accept`/`next`/`prev`/`dismiss`, default `Ctrl+Alt+→/↓/↑/←`), and the overlay has a mouse close button. Up to three candidates are shown in a scrollable list with a warm-up/generating spinner. **2026-09-24:** one page of 3 fixed-height (2-line) numbered items, footer = range/progress + source + key guide derived from the live bindings (`buildSuggestionKeyHints`, shared modifier shown once). Keys: Ctrl+Alt+↑/↓ move (the page follows the active item; at the last item `next` waits while filling, otherwise wraps; the 1.6.0 Left/Right page actions were removed as unreachable before a page filled and colliding with Intel display rotation), Ctrl+Alt+Enter accept, plain Esc closes while visible (overlay-scoped, not a binding), Ctrl+Alt+Backspace secondary dismiss; old default bindings migrate at tuning revision 5. Streaming updates never reposition/re-present the window (`decideSuggestionOverlayAction`); placement uses `anchorKind` — caret: below the line, element: outside the element (below → above → right → left → inner bottom-right). Navigation re-arms the visible TTL. Window 460×208. | | INPUT-08 | Per-app exclusions & consent controls | [~] | [-] | [-] | [-] | `inputExcludedApps` (executable names, case-insensitive) blocks both collection context and suggestions; telemetry master switch, pause, text-learning toggle and "delete collected data" all live in Settings → Input. 30-day retention prune runs on start. | | INPUT-09 | Flow Radar | [~] | [-] | [-] | [-] | `rankFlowWindows` ranks hourly aggregate activity density, character volume and edit stability into potential-focus time windows. It is not a real-session detector or session record. |