379 lines
12 KiB
TypeScript
379 lines
12 KiB
TypeScript
// src/main/services/VoiceActionService.ts
|
|
// Phase 13.3: OS 자동화 — 음성 → LLM이 JSON 액션 플랜 생성 → 실행
|
|
// 사전 정의 명령 + LLM 자유 해석. 위험 액션은 차단.
|
|
|
|
import { EventEmitter } from 'events'
|
|
import { exec } from 'child_process'
|
|
import { shell } from 'electron'
|
|
import { getLogger } from './LoggerService'
|
|
import { getPremiumLLMService } from './PremiumLLMService'
|
|
import { getMainWindow } from '../windows/WindowManager'
|
|
import { IPC_CHANNELS } from '@d3ro/core/ipc-channels'
|
|
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
|
|
import type {
|
|
VoiceActionPlan,
|
|
VoiceActionPreset,
|
|
VoiceActionHistoryEntry,
|
|
VoiceActionPlannedEvent,
|
|
VoiceActionExecutedEvent,
|
|
VoiceActionErrorEvent,
|
|
} from '@d3ro/core/types'
|
|
|
|
const logger = getLogger('VoiceActionService')
|
|
|
|
/** 위험 명령어 블랙리스트 (실행 차단) */
|
|
const BLOCKED_COMMANDS = [
|
|
'rm ', 'del ', 'format ', 'rmdir', 'rd ',
|
|
'shutdown', 'restart', 'taskkill',
|
|
'reg delete', 'reg add',
|
|
'net user', 'net localgroup',
|
|
]
|
|
|
|
/** LLM에 보낼 시스템 프롬프트 */
|
|
const ACTION_SYSTEM_PROMPT = `You are an OS automation assistant. Parse the user's voice command and output a JSON action plan.
|
|
|
|
Output EXACTLY one JSON object (no markdown, no explanation):
|
|
{
|
|
"action": "open_app" | "open_url" | "open_file" | "keyboard_shortcut" | "type_text" | "system_command",
|
|
"target": "<the target value>",
|
|
"description": "<brief description of what this does>",
|
|
"safe": true | false
|
|
}
|
|
|
|
Rules:
|
|
- "open_app": target = app name (e.g., "notepad", "chrome", "code")
|
|
- "open_url": target = full URL (e.g., "https://google.com")
|
|
- "open_file": target = file path
|
|
- "keyboard_shortcut": target = key combo (e.g., "ctrl+c", "alt+tab")
|
|
- "type_text": target = text to type
|
|
- "system_command": target = shell command
|
|
- Set safe=false for destructive operations (delete, format, shutdown, etc.)
|
|
|
|
If the command is unclear, output: {"action": "type_text", "target": "", "description": "Could not parse command", "safe": true}`
|
|
|
|
/** 프리셋 명령어 (LLM 없이 바로 실행) */
|
|
const PRESETS: VoiceActionPreset[] = [
|
|
{
|
|
keywords: ['크롬 열어', '크롬', 'chrome', 'open chrome'],
|
|
action: { action: 'open_app', target: 'chrome', description: 'Open Chrome browser', safe: true },
|
|
},
|
|
{
|
|
keywords: ['메모장 열어', '메모장', 'notepad', 'open notepad'],
|
|
action: { action: 'open_app', target: 'notepad', description: 'Open Notepad', safe: true },
|
|
},
|
|
{
|
|
keywords: ['탐색기 열어', '탐색기', 'explorer', 'open explorer'],
|
|
action: { action: 'open_app', target: 'explorer', description: 'Open File Explorer', safe: true },
|
|
},
|
|
{
|
|
keywords: ['볼륨 올려', '소리 올려', 'volume up'],
|
|
action: { action: 'keyboard_shortcut', target: 'volumeup', description: 'Volume up', safe: true },
|
|
},
|
|
{
|
|
keywords: ['볼륨 내려', '소리 내려', 'volume down'],
|
|
action: { action: 'keyboard_shortcut', target: 'volumedown', description: 'Volume down', safe: true },
|
|
},
|
|
{
|
|
keywords: ['음소거', 'mute'],
|
|
action: { action: 'keyboard_shortcut', target: 'volumemute', description: 'Toggle mute', safe: true },
|
|
},
|
|
]
|
|
|
|
class VoiceActionService extends EventEmitter {
|
|
private _history: VoiceActionHistoryEntry[] = []
|
|
private _enabled = false
|
|
|
|
get isEnabled(): boolean {
|
|
return this._enabled
|
|
}
|
|
|
|
setEnabled(enabled: boolean): void {
|
|
this._enabled = enabled
|
|
}
|
|
|
|
getPresets(): VoiceActionPreset[] {
|
|
return PRESETS
|
|
}
|
|
|
|
getHistory(): VoiceActionHistoryEntry[] {
|
|
return [...this._history].reverse()
|
|
}
|
|
|
|
clearHistory(): void {
|
|
this._history = []
|
|
}
|
|
|
|
/**
|
|
* 음성 텍스트로부터 액션 계획 → 실행
|
|
*/
|
|
async execute(text: string): Promise<void> {
|
|
// 라이센스 체크
|
|
try {
|
|
const { getLicenseService } = await import('./LicenseService')
|
|
const { Feature } = await import('@d3ro/core/types')
|
|
const license = getLicenseService()
|
|
const access = license.canUse(Feature.OS_AUTOMATION)
|
|
if (!access.allowed) {
|
|
license.promptUpgrade(Feature.OS_AUTOMATION, 'tier_required')
|
|
throw new D3ROError(ErrorCode.FeatureNotAvailable, 'Pro+ required for OS automation')
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof D3ROError) throw err
|
|
}
|
|
|
|
// 1. 프리셋 매칭
|
|
const preset = this._matchPreset(text)
|
|
if (preset) {
|
|
await this._executePlan(preset.action, text)
|
|
return
|
|
}
|
|
|
|
// 2. LLM으로 액션 플랜 생성
|
|
try {
|
|
const llmService = getPremiumLLMService()
|
|
const result = await llmService.generate(text, {
|
|
systemPrompt: ACTION_SYSTEM_PROMPT,
|
|
temperature: 0.1,
|
|
})
|
|
|
|
const plan = this._parsePlan(result.text)
|
|
if (!plan) {
|
|
this._emitError(text, 'Failed to parse action plan from LLM response')
|
|
return
|
|
}
|
|
|
|
// 안전장치
|
|
if (!plan.safe) {
|
|
const entry: VoiceActionHistoryEntry = {
|
|
id: crypto.randomUUID(),
|
|
userText: text,
|
|
plan,
|
|
executed: false,
|
|
timestamp: Date.now(),
|
|
}
|
|
this._history.push(entry)
|
|
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText: text })
|
|
logger.warn(`Voice action blocked (unsafe): ${plan.description}`)
|
|
return
|
|
}
|
|
|
|
await this._executePlan(plan, text)
|
|
} catch (err) {
|
|
this._emitError(text, err instanceof Error ? err.message : String(err))
|
|
}
|
|
}
|
|
|
|
private _matchPreset(text: string): VoiceActionPreset | null {
|
|
const lower = text.toLowerCase().trim()
|
|
for (const preset of PRESETS) {
|
|
for (const keyword of preset.keywords) {
|
|
if (lower.includes(keyword.toLowerCase())) {
|
|
return preset
|
|
}
|
|
}
|
|
}
|
|
return null
|
|
}
|
|
|
|
private _parsePlan(llmOutput: string): VoiceActionPlan | null {
|
|
try {
|
|
// JSON 블록 추출
|
|
const jsonMatch = llmOutput.match(/\{[\s\S]*\}/)
|
|
if (!jsonMatch) return null
|
|
|
|
const parsed = JSON.parse(jsonMatch[0]) as Record<string, unknown>
|
|
if (!parsed.action || !parsed.target) return null
|
|
|
|
return {
|
|
action: parsed.action as VoiceActionPlan['action'],
|
|
target: String(parsed.target),
|
|
description: String(parsed.description ?? ''),
|
|
safe: parsed.safe !== false,
|
|
}
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
private async _executePlan(plan: VoiceActionPlan, userText: string): Promise<void> {
|
|
// 차단된 명령어 체크
|
|
if (plan.action === 'system_command') {
|
|
const targetLower = plan.target.toLowerCase()
|
|
for (const blocked of BLOCKED_COMMANDS) {
|
|
if (targetLower.includes(blocked)) {
|
|
plan.safe = false
|
|
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText })
|
|
logger.warn(`Voice action blocked: ${plan.target}`)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText } as VoiceActionPlannedEvent)
|
|
|
|
try {
|
|
switch (plan.action) {
|
|
case 'open_app':
|
|
await this._openApp(plan.target)
|
|
break
|
|
case 'open_url':
|
|
await shell.openExternal(plan.target)
|
|
break
|
|
case 'open_file':
|
|
await shell.openPath(plan.target)
|
|
break
|
|
case 'keyboard_shortcut':
|
|
await this._simulateKeyboard(plan.target)
|
|
break
|
|
case 'type_text':
|
|
// @nut-tree 사용 (lazy import)
|
|
await this._typeText(plan.target)
|
|
break
|
|
case 'system_command':
|
|
await this._runCommand(plan.target)
|
|
break
|
|
}
|
|
|
|
const entry: VoiceActionHistoryEntry = {
|
|
id: crypto.randomUUID(),
|
|
userText,
|
|
plan,
|
|
executed: true,
|
|
timestamp: Date.now(),
|
|
}
|
|
this._history.push(entry)
|
|
if (this._history.length > 50) this._history.shift()
|
|
|
|
const event: VoiceActionExecutedEvent = { plan, success: true }
|
|
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_EXECUTED, event)
|
|
this.emit('action-executed', event)
|
|
logger.info(`Voice action executed: ${plan.action} → ${plan.target}`)
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err)
|
|
this._emitError(userText, msg)
|
|
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, msg)
|
|
}
|
|
}
|
|
|
|
private _openApp(appName: string): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
const cmd = `start "" "${appName}"`
|
|
exec(cmd, { shell: 'cmd.exe' }, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
})
|
|
}
|
|
|
|
private _simulateKeyboard(combo: string): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
// PowerShell SendKeys 사용
|
|
const keys = combo.toLowerCase()
|
|
let sendKeysStr = ''
|
|
|
|
if (keys === 'volumeup') {
|
|
// nircmd 대안: PowerShell로 볼륨 조절
|
|
const script = `
|
|
$wshell = New-Object -ComObject WScript.Shell
|
|
$wshell.SendKeys([char]175)
|
|
`
|
|
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
return
|
|
}
|
|
if (keys === 'volumedown') {
|
|
const script = `
|
|
$wshell = New-Object -ComObject WScript.Shell
|
|
$wshell.SendKeys([char]174)
|
|
`
|
|
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
return
|
|
}
|
|
if (keys === 'volumemute') {
|
|
const script = `
|
|
$wshell = New-Object -ComObject WScript.Shell
|
|
$wshell.SendKeys([char]173)
|
|
`
|
|
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
return
|
|
}
|
|
|
|
// 일반 키보드 단축키 (ctrl+c 등)
|
|
if (keys.includes('ctrl')) sendKeysStr += '^'
|
|
if (keys.includes('alt')) sendKeysStr += '%'
|
|
if (keys.includes('shift')) sendKeysStr += '+'
|
|
|
|
const key = keys.replace(/ctrl\+|alt\+|shift\+/g, '').trim()
|
|
sendKeysStr += key
|
|
|
|
const script = `
|
|
$wshell = New-Object -ComObject WScript.Shell
|
|
$wshell.SendKeys('${sendKeysStr}')
|
|
`
|
|
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
})
|
|
}
|
|
|
|
private async _typeText(text: string): Promise<void> {
|
|
try {
|
|
const { getTextInsertService } = await import('./TextInsertService')
|
|
await getTextInsertService().insertText(text)
|
|
} catch (err) {
|
|
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, `Type text failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
}
|
|
}
|
|
|
|
private _runCommand(command: string): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
exec(command, { timeout: 10000 }, (err) => {
|
|
if (err) reject(err)
|
|
else resolve()
|
|
})
|
|
})
|
|
}
|
|
|
|
private _emitError(userText: string, message: string): void {
|
|
const event: VoiceActionErrorEvent = { message, userText }
|
|
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_ERROR, event)
|
|
this.emit('error', event)
|
|
}
|
|
|
|
private _sendToRenderer(channel: string, data: unknown): void {
|
|
try {
|
|
const mainWindow = getMainWindow()
|
|
if (mainWindow && !mainWindow.isDestroyed()) {
|
|
mainWindow.webContents.send(channel, data)
|
|
}
|
|
} catch { /* ignore */ }
|
|
}
|
|
|
|
dispose(): void {
|
|
this.removeAllListeners()
|
|
}
|
|
}
|
|
|
|
// ── 싱글톤 ──
|
|
let instance: VoiceActionService | null = null
|
|
|
|
export function resetVoiceActionServiceForTests(): void {
|
|
if (instance) instance.removeAllListeners()
|
|
instance = null
|
|
}
|
|
|
|
export function getVoiceActionService(): VoiceActionService {
|
|
if (!instance) {
|
|
instance = new VoiceActionService()
|
|
}
|
|
return instance
|
|
}
|