Phase 12~13 전체 구현: Pro+ 피처 6종 + 음성 대화 + RAG + OS 자동화
Phase 12: - FileTranscriptionService: ffmpeg PCM 변환 + 30초 청크 순차 STT - MeetingSummaryService: 자막 세션 → LLM 자동 요약 + DB summaryText - DictationTemplateService: 필드별 음성 입력 상태 머신 + 프리셋 3개 Phase 13.1: - VoiceConversationService: STT→Ollama /api/chat→TTS 대화 루프 (10턴) - TTSPlaybackService: Windows SAPI 문장 단위 큐 재생 - LocalLLMService.chatStream: Ollama /api/chat 스트리밍 Phase 13.2: - RAGService: Ollama 임베딩 + SQLite 벡터 + 코사인 유사도 검색 - KnowledgeBasePage: 문서 관리 + 질문/답변 UI - PDF 파서: zlib FlateDecode 해제 + BT/ET 텍스트 추출 Phase 13.3: - VoiceActionService: LLM JSON 액션 플랜 생성 + 실행 - 프리셋 6개 (크롬/메모장/탐색기/볼륨), 위험 명령 차단 공통: IPC ~70채널, 에러코드 780-878, i18n 100+키 버그픽스: 라이선스 로컬 키 우선, i18n featureLabel, DOM 중첩
This commit is contained in:
parent
a31f96bbb8
commit
eb83682269
38 changed files with 5678 additions and 19 deletions
376
src/main/services/VoiceActionService.ts
Normal file
376
src/main/services/VoiceActionService.ts
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
// src/main/services/VoiceActionService.ts
|
||||
// Phase 13.3: OS 자동화 — 음성 → LLM이 JSON 액션 플랜 생성 → 실행
|
||||
// 사전 정의 명령 + LLM 자유 해석. 위험 액션은 차단.
|
||||
|
||||
import { EventEmitter } from 'events'
|
||||
import { exec } from 'child_process'
|
||||
import { shell } from 'electron'
|
||||
import { nanoid } from 'nanoid'
|
||||
import { getLogger } from './LoggerService'
|
||||
import { getLocalLLMService } from './LocalLLMService'
|
||||
import { configGet } from './ConfigService'
|
||||
import { getMainWindow } from '../windows/WindowManager'
|
||||
import { IPC_CHANNELS } from '@shared/ipc-channels'
|
||||
import { D3ROError, ErrorCode } from '@shared/errors'
|
||||
import type {
|
||||
VoiceActionPlan,
|
||||
VoiceActionPreset,
|
||||
VoiceActionHistoryEntry,
|
||||
VoiceActionPlannedEvent,
|
||||
VoiceActionExecutedEvent,
|
||||
VoiceActionErrorEvent,
|
||||
} from '@shared/types'
|
||||
|
||||
const logger = getLogger('VoiceActionService')
|
||||
|
||||
/** 위험 명령어 블랙리스트 (실행 차단) */
|
||||
const BLOCKED_COMMANDS = [
|
||||
'rm ', 'del ', 'format ', 'rmdir', 'rd ',
|
||||
'shutdown', 'restart', 'taskkill',
|
||||
'reg delete', 'reg add',
|
||||
'net user', 'net localgroup',
|
||||
]
|
||||
|
||||
/** LLM에 보낼 시스템 프롬프트 */
|
||||
const ACTION_SYSTEM_PROMPT = `You are an OS automation assistant. Parse the user's voice command and output a JSON action plan.
|
||||
|
||||
Output EXACTLY one JSON object (no markdown, no explanation):
|
||||
{
|
||||
"action": "open_app" | "open_url" | "open_file" | "keyboard_shortcut" | "type_text" | "system_command",
|
||||
"target": "<the target value>",
|
||||
"description": "<brief description of what this does>",
|
||||
"safe": true | false
|
||||
}
|
||||
|
||||
Rules:
|
||||
- "open_app": target = app name (e.g., "notepad", "chrome", "code")
|
||||
- "open_url": target = full URL (e.g., "https://google.com")
|
||||
- "open_file": target = file path
|
||||
- "keyboard_shortcut": target = key combo (e.g., "ctrl+c", "alt+tab")
|
||||
- "type_text": target = text to type
|
||||
- "system_command": target = shell command
|
||||
- Set safe=false for destructive operations (delete, format, shutdown, etc.)
|
||||
|
||||
If the command is unclear, output: {"action": "type_text", "target": "", "description": "Could not parse command", "safe": true}`
|
||||
|
||||
/** 프리셋 명령어 (LLM 없이 바로 실행) */
|
||||
const PRESETS: VoiceActionPreset[] = [
|
||||
{
|
||||
keywords: ['크롬 열어', '크롬', 'chrome', 'open chrome'],
|
||||
action: { action: 'open_app', target: 'chrome', description: 'Open Chrome browser', safe: true },
|
||||
},
|
||||
{
|
||||
keywords: ['메모장 열어', '메모장', 'notepad', 'open notepad'],
|
||||
action: { action: 'open_app', target: 'notepad', description: 'Open Notepad', safe: true },
|
||||
},
|
||||
{
|
||||
keywords: ['탐색기 열어', '탐색기', 'explorer', 'open explorer'],
|
||||
action: { action: 'open_app', target: 'explorer', description: 'Open File Explorer', safe: true },
|
||||
},
|
||||
{
|
||||
keywords: ['볼륨 올려', '소리 올려', 'volume up'],
|
||||
action: { action: 'keyboard_shortcut', target: 'volumeup', description: 'Volume up', safe: true },
|
||||
},
|
||||
{
|
||||
keywords: ['볼륨 내려', '소리 내려', 'volume down'],
|
||||
action: { action: 'keyboard_shortcut', target: 'volumedown', description: 'Volume down', safe: true },
|
||||
},
|
||||
{
|
||||
keywords: ['음소거', 'mute'],
|
||||
action: { action: 'keyboard_shortcut', target: 'volumemute', description: 'Toggle mute', safe: true },
|
||||
},
|
||||
]
|
||||
|
||||
class VoiceActionService extends EventEmitter {
|
||||
private _history: VoiceActionHistoryEntry[] = []
|
||||
private _enabled = false
|
||||
|
||||
get isEnabled(): boolean {
|
||||
return this._enabled
|
||||
}
|
||||
|
||||
setEnabled(enabled: boolean): void {
|
||||
this._enabled = enabled
|
||||
}
|
||||
|
||||
getPresets(): VoiceActionPreset[] {
|
||||
return PRESETS
|
||||
}
|
||||
|
||||
getHistory(): VoiceActionHistoryEntry[] {
|
||||
return [...this._history].reverse()
|
||||
}
|
||||
|
||||
clearHistory(): void {
|
||||
this._history = []
|
||||
}
|
||||
|
||||
/**
|
||||
* 음성 텍스트로부터 액션 계획 → 실행
|
||||
*/
|
||||
async execute(text: string): Promise<void> {
|
||||
// 라이센스 체크
|
||||
try {
|
||||
const { getLicenseService } = await import('./LicenseService')
|
||||
const { Feature } = await import('@shared/types')
|
||||
const license = getLicenseService()
|
||||
const access = license.canUse(Feature.OS_AUTOMATION)
|
||||
if (!access.allowed) {
|
||||
license.promptUpgrade(Feature.OS_AUTOMATION, 'tier_required')
|
||||
throw new D3ROError(ErrorCode.FeatureNotAvailable, 'Pro+ required for OS automation')
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof D3ROError) throw err
|
||||
}
|
||||
|
||||
// 1. 프리셋 매칭
|
||||
const preset = this._matchPreset(text)
|
||||
if (preset) {
|
||||
await this._executePlan(preset.action, text)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. LLM으로 액션 플랜 생성
|
||||
try {
|
||||
const llmService = getLocalLLMService()
|
||||
const result = await llmService.generate(text, {
|
||||
systemPrompt: ACTION_SYSTEM_PROMPT,
|
||||
temperature: 0.1,
|
||||
})
|
||||
|
||||
const plan = this._parsePlan(result.text)
|
||||
if (!plan) {
|
||||
this._emitError(text, 'Failed to parse action plan from LLM response')
|
||||
return
|
||||
}
|
||||
|
||||
// 안전장치
|
||||
if (!plan.safe) {
|
||||
const entry: VoiceActionHistoryEntry = {
|
||||
id: nanoid(),
|
||||
userText: text,
|
||||
plan,
|
||||
executed: false,
|
||||
timestamp: Date.now(),
|
||||
}
|
||||
this._history.push(entry)
|
||||
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText: text })
|
||||
logger.warn(`Voice action blocked (unsafe): ${plan.description}`)
|
||||
return
|
||||
}
|
||||
|
||||
await this._executePlan(plan, text)
|
||||
} catch (err) {
|
||||
this._emitError(text, err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
}
|
||||
|
||||
private _matchPreset(text: string): VoiceActionPreset | null {
|
||||
const lower = text.toLowerCase().trim()
|
||||
for (const preset of PRESETS) {
|
||||
for (const keyword of preset.keywords) {
|
||||
if (lower.includes(keyword.toLowerCase())) {
|
||||
return preset
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private _parsePlan(llmOutput: string): VoiceActionPlan | null {
|
||||
try {
|
||||
// JSON 블록 추출
|
||||
const jsonMatch = llmOutput.match(/\{[\s\S]*\}/)
|
||||
if (!jsonMatch) return null
|
||||
|
||||
const parsed = JSON.parse(jsonMatch[0]) as Record<string, unknown>
|
||||
if (!parsed.action || !parsed.target) return null
|
||||
|
||||
return {
|
||||
action: parsed.action as VoiceActionPlan['action'],
|
||||
target: String(parsed.target),
|
||||
description: String(parsed.description ?? ''),
|
||||
safe: parsed.safe !== false,
|
||||
}
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private async _executePlan(plan: VoiceActionPlan, userText: string): Promise<void> {
|
||||
// 차단된 명령어 체크
|
||||
if (plan.action === 'system_command') {
|
||||
const targetLower = plan.target.toLowerCase()
|
||||
for (const blocked of BLOCKED_COMMANDS) {
|
||||
if (targetLower.includes(blocked)) {
|
||||
plan.safe = false
|
||||
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText })
|
||||
logger.warn(`Voice action blocked: ${plan.target}`)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText } as VoiceActionPlannedEvent)
|
||||
|
||||
try {
|
||||
switch (plan.action) {
|
||||
case 'open_app':
|
||||
await this._openApp(plan.target)
|
||||
break
|
||||
case 'open_url':
|
||||
await shell.openExternal(plan.target)
|
||||
break
|
||||
case 'open_file':
|
||||
await shell.openPath(plan.target)
|
||||
break
|
||||
case 'keyboard_shortcut':
|
||||
await this._simulateKeyboard(plan.target)
|
||||
break
|
||||
case 'type_text':
|
||||
// @nut-tree 사용 (lazy import)
|
||||
await this._typeText(plan.target)
|
||||
break
|
||||
case 'system_command':
|
||||
await this._runCommand(plan.target)
|
||||
break
|
||||
}
|
||||
|
||||
const entry: VoiceActionHistoryEntry = {
|
||||
id: nanoid(),
|
||||
userText,
|
||||
plan,
|
||||
executed: true,
|
||||
timestamp: Date.now(),
|
||||
}
|
||||
this._history.push(entry)
|
||||
if (this._history.length > 50) this._history.shift()
|
||||
|
||||
const event: VoiceActionExecutedEvent = { plan, success: true }
|
||||
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_EXECUTED, event)
|
||||
this.emit('action-executed', event)
|
||||
logger.info(`Voice action executed: ${plan.action} → ${plan.target}`)
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
this._emitError(userText, msg)
|
||||
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, msg)
|
||||
}
|
||||
}
|
||||
|
||||
private _openApp(appName: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const cmd = `start "" "${appName}"`
|
||||
exec(cmd, { shell: 'cmd.exe' }, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private _simulateKeyboard(combo: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// PowerShell SendKeys 사용
|
||||
const keys = combo.toLowerCase()
|
||||
let sendKeysStr = ''
|
||||
|
||||
if (keys === 'volumeup') {
|
||||
// nircmd 대안: PowerShell로 볼륨 조절
|
||||
const script = `
|
||||
$wshell = New-Object -ComObject WScript.Shell
|
||||
$wshell.SendKeys([char]175)
|
||||
`
|
||||
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
return
|
||||
}
|
||||
if (keys === 'volumedown') {
|
||||
const script = `
|
||||
$wshell = New-Object -ComObject WScript.Shell
|
||||
$wshell.SendKeys([char]174)
|
||||
`
|
||||
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
return
|
||||
}
|
||||
if (keys === 'volumemute') {
|
||||
const script = `
|
||||
$wshell = New-Object -ComObject WScript.Shell
|
||||
$wshell.SendKeys([char]173)
|
||||
`
|
||||
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// 일반 키보드 단축키 (ctrl+c 등)
|
||||
if (keys.includes('ctrl')) sendKeysStr += '^'
|
||||
if (keys.includes('alt')) sendKeysStr += '%'
|
||||
if (keys.includes('shift')) sendKeysStr += '+'
|
||||
|
||||
const key = keys.replace(/ctrl\+|alt\+|shift\+/g, '').trim()
|
||||
sendKeysStr += key
|
||||
|
||||
const script = `
|
||||
$wshell = New-Object -ComObject WScript.Shell
|
||||
$wshell.SendKeys('${sendKeysStr}')
|
||||
`
|
||||
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private async _typeText(text: string): Promise<void> {
|
||||
try {
|
||||
const { getTextInsertService } = await import('./TextInsertService')
|
||||
await getTextInsertService().insertText(text)
|
||||
} catch (err) {
|
||||
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, `Type text failed: ${err instanceof Error ? err.message : String(err)}`)
|
||||
}
|
||||
}
|
||||
|
||||
private _runCommand(command: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, { timeout: 10000 }, (err) => {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private _emitError(userText: string, message: string): void {
|
||||
const event: VoiceActionErrorEvent = { message, userText }
|
||||
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_ERROR, event)
|
||||
this.emit('error', event)
|
||||
}
|
||||
|
||||
private _sendToRenderer(channel: string, data: unknown): void {
|
||||
try {
|
||||
const mainWindow = getMainWindow()
|
||||
if (mainWindow && !mainWindow.isDestroyed()) {
|
||||
mainWindow.webContents.send(channel, data)
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.removeAllListeners()
|
||||
}
|
||||
}
|
||||
|
||||
// ── 싱글톤 ──
|
||||
let instance: VoiceActionService | null = null
|
||||
|
||||
export function getVoiceActionService(): VoiceActionService {
|
||||
if (!instance) {
|
||||
instance = new VoiceActionService()
|
||||
}
|
||||
return instance
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue