d3ro-voice/apps/desktop/tests/main/services/input-flow-services.test.ts
Yun Chan 2fe20fa7b5
Some checks failed
deploy-site / deploy (push) Failing after 39s
release / release-windows (push) Failing after 3m41s
portable-unsigned / portable-windows (push) Failing after 12m23s
release: ship v1.6.0 with paged suggestions and a cleaner phrase memory
Next-sentence suggestions now arrive one at a time up to twelve, shown three
per page with Ctrl+Alt+Up/Down to move, Left/Right to page, Enter to accept
and Esc to close; old default bindings migrate and the panel guide follows the
live bindings. The overlay is redesigned, stays put while candidates stream
and sits outside the input box when no caret is reported.

The personal phrase memory stops learning from terminals, code editors and
the coding-agent hub, ignores symbol-heavy lines and empty-field placeholders,
and prunes existing entries that break those rules.

Fixes suggestion keys starting dictation, installs stuck on a pre-1.5.0
speech engine without the focus endpoint, Ollama runner windows flashing
while typing, the speech engine starting twice, and cold-model timeouts.
Live captions can be dragged to a remembered position and show a waiting
notice until the first line arrives.

Bumps the product version to 1.6.0 (Android/iOS build 1060000).
2026-09-24 19:56:28 +09:00

588 lines
21 KiB
TypeScript

import { beforeEach, describe, expect, it, vi } from 'vitest'
import { inputActivity, personalPhrases, suggestions, typingSamples } from '../../../src/main/db/schema'
type ActivityRow = {
id: string
date: string
hour: number
appName: string
keystrokes: number
shortcuts: number
backspaces: number
clicks: number
doubleClicks: number
scrollTicks: number
mouseDistancePx: number
chars: number
words: number
sentences: number
activeMs: number
updatedAt: number
}
type PhraseRow = {
id: string
phrase: string
count: number
source: 'typed' | 'voice' | 'suggestion' | 'clipboard'
appName: string | null
lastUsedAt: number | null
createdAt: number
}
type SuggestionRow = {
id: string
appName: string | null
prefixText: string
suggestionText: string
candidateCount: number
model: string | null
latencyMs: number | null
accepted: boolean
createdAt: number
}
type SampleRow = {
id: string
text: string
wordCount: number
charCount: number
appName: string | null
windowTitle: string | null
source: 'typed' | 'voice' | 'suggestion' | 'clipboard'
createdAt: number
}
type StreamOptions = { signal?: AbortSignal }
type StreamFactory = (prompt: string, options: StreamOptions) => AsyncIterable<string>
const harness = vi.hoisted(() => {
const config = new Map<string, unknown>()
const failure = { read: false, delete: false }
const model: { available: boolean; streamGenerate: StreamFactory } = {
available: false,
streamGenerate: async function* () {
return
}
}
const rows: {
activity: ActivityRow[]
phrases: PhraseRow[]
samples: SampleRow[]
suggestions: SuggestionRow[]
} = { activity: [], phrases: [], samples: [], suggestions: [] }
const rowsFor = (table: unknown): ActivityRow[] | PhraseRow[] | SampleRow[] | SuggestionRow[] => {
if (table === inputActivity) return rows.activity
if (table === personalPhrases) return rows.phrases
if (table === typingSamples) return rows.samples
return rows.suggestions
}
const database = {
select(selection?: Record<string, unknown>) {
if (failure.read) throw new Error('receipt read failed')
return {
from(table: unknown) {
const selectedRows = rowsFor(table)
const chain = {
where: () => chain,
orderBy: () => chain,
limit: () => chain,
all: () => {
const keys = Object.keys(selection ?? {})
if (table === suggestions && keys.includes('appName')) {
return rows.suggestions.map(({ appName, accepted, latencyMs }) => ({ appName, accepted, latencyMs }))
}
return [...selectedRows]
},
get: () => {
const keys = Object.keys(selection ?? {})
if (keys.includes('total')) {
const suggestionRows = rows.suggestions
const latencyRows = suggestionRows.filter((row) => row.latencyMs !== null)
return {
total: suggestionRows.length,
accepted: suggestionRows.filter((row) => row.accepted).length,
avgLatency:
latencyRows.length === 0
? null
: latencyRows.reduce((total, row) => total + (row.latencyMs ?? 0), 0) /
latencyRows.length
}
}
if (keys.includes('value')) return { value: selectedRows.length }
return selectedRows[0]
}
}
return chain
}
}
},
insert(table: unknown) {
return {
values(value: Record<string, unknown>) {
const selectedRows = rowsFor(table)
const apply = () => {
if (table === personalPhrases) {
const phrase = value.phrase as string
const existing = rows.phrases.find((row) => row.phrase === phrase)
if (existing) {
existing.count += 1
existing.lastUsedAt = value.lastUsedAt as number
existing.appName = value.appName as string | null
return
}
}
selectedRows.push(value as never)
}
return {
run: () => apply(),
onConflictDoUpdate: () => ({ run: () => apply() })
}
}
}
},
delete(table: unknown) {
const selectedRows = rowsFor(table)
const chain = {
where: () => chain,
run: () => {
if (failure.delete) throw new Error('delete failed')
selectedRows.splice(0, selectedRows.length)
return { changes: 1 }
}
}
return chain
},
update: () => ({ set: () => ({ where: () => ({ run: () => ({ changes: 1 }) }) }) })
}
const uia = {
isAvailable: () => false,
lastReason: 'test',
lastSuccessAt: 0,
getSnapshot: vi.fn(async () => ({
available: false,
reason: 'test',
isPassword: false,
isEditable: false,
isComposing: false,
hasSelection: false,
textSource: 'none' as const,
text: '',
caretOffset: null,
caretRect: null,
elementRect: null,
windowTitle: null,
appName: null,
processId: null,
capturedAt: Date.now()
}))
}
return {
config,
failure,
model,
rows,
database,
uia,
graph: { continuations: [] as string[], related: [] as string[] }
}
})
vi.mock('electron', () => ({ screen: { getPrimaryDisplay: () => ({ scaleFactor: 1 }) } }))
vi.mock('uiohook-napi', () => ({ uIOhook: { on: vi.fn(), removeListener: vi.fn() } }))
vi.mock('../../../src/main/db', () => ({ getDatabase: () => harness.database }))
vi.mock('../../../src/main/services/ConfigService', () => ({
configGet: (key: string) => harness.config.get(key),
configSet: (key: string, value: unknown) => harness.config.set(key, value)
}))
vi.mock('../../../src/main/services/LoggerService', () => ({
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() })
}))
vi.mock('../../../src/main/services/global-input-hook', () => ({ acquireGlobalInputHook: () => () => undefined }))
vi.mock('../../../src/main/services/KeyBindingService', () => ({ uiohookCodeToVk: () => null }))
vi.mock('../../../src/main/services/UiaContextService', () => ({
getUiaContextService: () => harness.uia
}))
vi.mock('../../../src/main/utils/win32-foreground', () => ({ getForegroundWindowInfo: () => null }))
vi.mock('../../../src/main/services/PersonalGraphService', () => ({
getPersonalGraphService: () => ({
clearAll: vi.fn(),
runMaintenance: vi.fn(),
indexText: vi.fn(),
retrieveContext: () => harness.graph
})
}))
vi.mock('../../../src/main/services/LocalLLMService', () => ({
getLocalLLMService: () => ({
isAvailable: () => harness.model.available,
streamGenerate: (prompt: string, options: StreamOptions) => harness.model.streamGenerate(prompt, options)
})
}))
vi.mock('../../../src/main/services/TextInsertService', () => ({ getTextInsertService: () => ({ insertText: vi.fn() }) }))
import {
getInputTelemetryService,
resetInputTelemetryServiceForTests
} from '../../../src/main/services/InputTelemetryService'
import {
getSuggestionService,
resetSuggestionServiceForTests
} from '../../../src/main/services/SuggestionService'
const now = Date.now()
function activity(overrides: Partial<ActivityRow> = {}): ActivityRow {
return {
id: crypto.randomUUID(),
date: new Date(now).toISOString().slice(0, 10),
hour: 9,
appName: 'Notion.exe',
keystrokes: 0,
shortcuts: 0,
backspaces: 0,
clicks: 0,
doubleClicks: 0,
scrollTicks: 0,
mouseDistancePx: 0,
chars: 0,
words: 0,
sentences: 0,
activeMs: 0,
updatedAt: now,
...overrides
}
}
function typingContext(overrides: Partial<Parameters<ReturnType<typeof getSuggestionService>['handleTypingContext']>[0]> = {}) {
return {
prefix: '오늘 회의 결과를',
fullText: '오늘 회의 결과를',
caretOffset: 9,
anchor: { x: 1, y: 1, width: 1, height: 1 },
anchorKind: 'caret' as const,
isPassword: false,
isEditable: true,
isComposing: false,
hasSelection: false,
available: true,
appName: 'Notion.exe',
windowTitle: '회의록',
idleMs: 300,
capturedAt: now,
editedSinceFocus: true,
typedRecently: true,
...overrides
}
}
beforeEach(() => {
harness.config.clear()
harness.config.set('inputExcludedApps', [])
harness.config.set('inputLearnTypedText', true)
harness.config.set('suggestionEnabled', true)
harness.config.set('suggestionMinPrefixChars', 4)
harness.config.set('suggestionTriggerDelayMs', 100)
harness.config.set('suggestionMaxRequestsPerMinute', 20)
harness.config.set('suggestionDailyBudget', 100)
harness.failure.read = false
harness.failure.delete = false
harness.model.available = false
harness.model.streamGenerate = async function* () {
return
}
harness.rows.activity.splice(0)
harness.rows.phrases.splice(0)
harness.rows.samples.splice(0)
harness.rows.suggestions.splice(0)
harness.graph.continuations = []
harness.graph.related = []
resetInputTelemetryServiceForTests()
resetSuggestionServiceForTests()
harness.uia.getSnapshot.mockClear()
})
describe('입력 플로우 서비스', () => {
it('제안 표시 중에만 오래된 키 입력과 mouse-up 후 UIA 포커스를 다시 확인한다', async () => {
const telemetry = getInputTelemetryService() as unknown as {
_running: boolean
_lastKeyAt: number
_sampleWhileTyping: () => Promise<void>
_handleMouseUp: (event: { x: number; y: number }) => void
setSuggestionPresentationActive: (active: boolean) => void
}
telemetry._running = true
telemetry._lastKeyAt = Date.now() - 6000
await telemetry._sampleWhileTyping()
expect(harness.uia.getSnapshot).not.toHaveBeenCalled()
telemetry.setSuggestionPresentationActive(true)
await telemetry._sampleWhileTyping()
expect(harness.uia.getSnapshot).toHaveBeenCalledTimes(1)
vi.useFakeTimers()
telemetry._handleMouseUp({ x: 1, y: 1 })
await vi.advanceTimersByTimeAsync(120)
expect(harness.uia.getSnapshot).toHaveBeenCalledTimes(2)
vi.useRealTimers()
})
it('시간대 집계에 플로우·마찰과 앱별 제안 품질을 함께 반환한다', () => {
harness.rows.activity.push(
activity({ hour: 9, chars: 1200, backspaces: 30, activeMs: 60 * 60 * 1000, keystrokes: 1300 }),
activity({ id: crypto.randomUUID(), hour: 14, chars: 120, backspaces: 60, activeMs: 20 * 60 * 1000 })
)
harness.rows.suggestions.push(
{ id: 's1', appName: 'Notion.exe', prefixText: 'a', suggestionText: 'b', candidateCount: 1, model: 'm', latencyMs: 100, accepted: true, createdAt: now },
{ id: 's2', appName: 'Notion.exe', prefixText: 'a', suggestionText: 'c', candidateCount: 1, model: 'm', latencyMs: 300, accepted: false, createdAt: now },
{ id: 's3', appName: 'Slack.exe', prefixText: 'a', suggestionText: 'd', candidateCount: 1, model: 'm', latencyMs: null, accepted: true, createdAt: now }
)
const summary = getInputTelemetryService().getSummary(1)
expect(summary.hourly[9]).toMatchObject({ chars: 1200, backspaces: 30, activeMs: 3600000 })
expect(summary.friction).toMatchObject({ band: 'steady', editsPer100Chars: 6.8 })
expect(summary.flowWindows[0]).toMatchObject({ hour: 9 })
expect(summary.suggestionApps).toEqual([
{ appName: 'Notion.exe', total: 2, accepted: 1, acceptRate: 0.5, avgLatencyMs: 200 },
{ appName: 'Slack.exe', total: 1, accepted: 1, acceptRate: 1, avgLatencyMs: null }
])
})
it('개인정보 영수증은 로컬 보존 경계와 실제 행 수를 반환한다', () => {
harness.rows.activity.push(activity())
harness.rows.samples.push({ id: 'sample', text: '학습 문장입니다', wordCount: 2, charCount: 7, appName: null, windowTitle: null, source: 'typed', createdAt: now })
harness.rows.phrases.push({ id: 'phrase', phrase: '학습 문장입니다', count: 1, source: 'typed', appName: null, lastUsedAt: now, createdAt: now })
harness.rows.suggestions.push({ id: 's', appName: null, prefixText: '', suggestionText: '후보', candidateCount: 1, model: 'm', latencyMs: 10, accepted: false, createdAt: now })
expect(getInputTelemetryService().getPrivacyReceipt()).toMatchObject({
localOnly: true,
rawKeyContentStored: false,
retention: {
activityDays: 30,
typingSamplesDays: 30,
suggestionDays: 30,
personalPhrases: 'until-deleted'
},
counts: { activityBuckets: 1, typingSamples: 1, personalPhrases: 1, suggestions: 1 }
})
})
it('개인정보 영수증 조회 실패를 빈 영수증으로 위장하지 않는다', () => {
harness.failure.read = true
expect(() => getInputTelemetryService().getPrivacyReceipt()).toThrow('receipt read failed')
})
it('전체 삭제는 제안 이력까지 제거한다', () => {
harness.rows.activity.push(activity())
harness.rows.suggestions.push({ id: 's', appName: null, prefixText: '', suggestionText: '후보', candidateCount: 1, model: 'm', latencyMs: 10, accepted: false, createdAt: now })
getInputTelemetryService().clearAll()
expect(getInputTelemetryService().getPrivacyReceipt().counts).toEqual({
activityBuckets: 0,
typingSamples: 0,
personalPhrases: 0,
suggestions: 0
})
})
it('전체 삭제 실패를 성공처럼 반환하지 않고 메모리 초기화도 건너뛴다', () => {
harness.rows.activity.push(activity())
const service = getInputTelemetryService() as unknown as {
_pending: Map<string, unknown>
clearAll: () => void
}
service._pending.set('pending', {})
harness.failure.delete = true
expect(() => service.clearAll()).toThrow('delete failed')
expect(harness.rows.activity).toHaveLength(1)
expect(service._pending.size).toBe(1)
})
it('보존 정리는 만료 제안 이력도 같은 경계로 정리한다', () => {
harness.rows.suggestions.push({
id: 'expired',
appName: null,
prefixText: '',
suggestionText: '만료 후보',
candidateCount: 1,
model: 'm',
latencyMs: 10,
accepted: false,
createdAt: now - 31 * 24 * 60 * 60 * 1000
})
const service = getInputTelemetryService() as unknown as {
_pruneOldData: (at: number) => void
}
service._pruneOldData(now)
expect(getInputTelemetryService().getPrivacyReceipt().counts.suggestions).toBe(0)
})
it('학습 문구에 마지막 앱을 저장하고 목록 계약에 매핑한다', () => {
const service = getInputTelemetryService() as unknown as {
_learnText: (text: string, meta: { appName: string | null; windowTitle: string | null; source: 'typed' }) => void
}
service._learnText('배포 일정을 공유합니다. 다음 조치를 확인합니다.', {
appName: 'Slack.exe',
windowTitle: '채널',
source: 'typed'
})
expect(getInputTelemetryService().listPhrases()).toEqual(
expect.arrayContaining([expect.objectContaining({ appName: 'Slack.exe' })])
)
})
it('비밀번호 스냅샷은 제외 추천 증거에 넣지 않고 반복 비가독 앱만 추천한다', () => {
const service = getInputTelemetryService() as unknown as {
_foreground: { appName: string | null }
_recordReadabilityEvidence: (snapshot: { isPassword: boolean; isEditable: boolean; textSource: 'none' | 'value'; text: string }) => void
}
service._foreground.appName = 'Legacy.exe'
for (let index = 0; index < 4; index += 1) {
service._recordReadabilityEvidence({ isPassword: false, isEditable: false, textSource: 'none', text: '' })
}
service._recordReadabilityEvidence({ isPassword: true, isEditable: false, textSource: 'none', text: '' })
expect(getInputTelemetryService().getState().exclusionRecommendation).toEqual({
appName: 'Legacy.exe',
reason: 'repeated-unreadable',
samples: 4
})
})
it('모델이 없을 때 정책을 지키며 로컬 기억 후보와 출처를 게시한다', () => {
harness.rows.phrases.push({ id: 'phrase', phrase: '검토 결과를 공유합니다', count: 4, source: 'typed', appName: 'Notion.exe', lastUsedAt: now, createdAt: now })
harness.graph.continuations = ['다음 조치를 정리하겠습니다']
getSuggestionService().handleTypingContext({
prefix: '오늘 회의 결과를',
fullText: '오늘 회의 결과를',
caretOffset: 9,
anchor: { x: 1, y: 1, width: 1, height: 1 },
anchorKind: 'caret',
isPassword: false,
isEditable: true,
isComposing: false,
hasSelection: false,
available: true,
appName: 'Notion.exe',
windowTitle: '회의록',
idleMs: 300,
capturedAt: now,
editedSinceFocus: true,
typedRecently: true
})
expect(getSuggestionService().getState()).toMatchObject({
visible: true,
provenance: { mode: 'local-memory', continuationCount: 1, appPhraseCount: 1 }
})
expect(harness.rows.suggestions[0]).toMatchObject({ model: 'local-memory' })
})
it('timeout abort가 스트림에서 throw되어도 로컬 기억 fallback을 한 번 게시한다', async () => {
const context = typingContext()
const service = getSuggestionService() as unknown as {
_lastContext: typeof context | null
_lastContextAt: number
_generate: (prefix: string, context: typeof context, maxCandidates: number, maxChars: number) => Promise<void>
}
harness.config.set('llmModelId', 'local-model')
harness.config.set('suggestionRequestTimeoutMs', 10)
harness.model.available = true
harness.graph.continuations = ['다음 조치를 정리하겠습니다']
harness.model.streamGenerate = (_prompt, options) => ({
async *[Symbol.asyncIterator]() {
await new Promise<void>((_resolve, reject) => {
options.signal?.addEventListener('abort', () => reject(new Error('timeout abort')), { once: true })
})
}
})
service._lastContext = context
service._lastContextAt = Date.now()
await service._generate(context.prefix, context, 3, 160)
expect(getSuggestionService().getState()).toMatchObject({
visible: true,
provenance: { mode: 'local-memory', continuationCount: 1 }
})
expect(harness.rows.suggestions).toHaveLength(1)
})
it('timeout 중 현재 문맥과 세대가 바뀌면 로컬 기억 fallback을 게시하지 않는다', async () => {
const context = typingContext()
const service = getSuggestionService() as unknown as {
_generationToken: number
_lastContext: typeof context | null
_lastContextAt: number
_generate: (prefix: string, context: typeof context, maxCandidates: number, maxChars: number) => Promise<void>
}
harness.config.set('llmModelId', 'local-model')
harness.config.set('suggestionRequestTimeoutMs', 20)
harness.model.available = true
harness.graph.continuations = ['다음 조치를 정리하겠습니다']
harness.model.streamGenerate = (_prompt, options) => ({
async *[Symbol.asyncIterator]() {
await new Promise<void>((_resolve, reject) => {
options.signal?.addEventListener('abort', () => reject(new Error('timeout abort')), { once: true })
})
}
})
service._lastContext = context
service._lastContextAt = Date.now()
const changed = typingContext({ prefix: '다른 문맥입니다', appName: 'Slack.exe', windowTitle: '채널' })
const invalidate = setTimeout(() => {
service._generationToken += 1
service._lastContext = changed
service._lastContextAt = Date.now()
}, 1)
await service._generate(context.prefix, context, 3, 160)
clearTimeout(invalidate)
expect(getSuggestionService().getState().visible).toBe(false)
expect(harness.rows.suggestions).toHaveLength(0)
})
it('세대가 달라진 로컬 후보는 게시하지 않아 명시 취소 뒤 되살아나지 않는다', () => {
const service = getSuggestionService() as unknown as {
_generationToken: number
_publishLocalMemory: (
prefix: string,
context: Parameters<ReturnType<typeof getSuggestionService>['handleTypingContext']>[0],
maxCandidates: number,
maxChars: number,
latencyMs: number,
token: number
) => boolean
}
service._generationToken = 2
expect(
service._publishLocalMemory(
'오늘 회의 결과를',
{
prefix: '오늘 회의 결과를', fullText: '', caretOffset: null, anchor: null, anchorKind: null, isPassword: false,
isEditable: true, isComposing: false, hasSelection: false, available: true, appName: 'Notion.exe', windowTitle: null,
idleMs: 300, capturedAt: now, editedSinceFocus: true, typedRecently: true
},
3,
160,
0,
1
)
).toBe(false)
expect(getSuggestionService().getState().visible).toBe(false)
})
})