Phase 12~13 전체 구현: Pro+ 피처 6종 + 음성 대화 + RAG + OS 자동화

Phase 12:
- FileTranscriptionService: ffmpeg PCM 변환 + 30초 청크 순차 STT
- MeetingSummaryService: 자막 세션 → LLM 자동 요약 + DB summaryText
- DictationTemplateService: 필드별 음성 입력 상태 머신 + 프리셋 3개

Phase 13.1:
- VoiceConversationService: STT→Ollama /api/chat→TTS 대화 루프 (10턴)
- TTSPlaybackService: Windows SAPI 문장 단위 큐 재생
- LocalLLMService.chatStream: Ollama /api/chat 스트리밍

Phase 13.2:
- RAGService: Ollama 임베딩 + SQLite 벡터 + 코사인 유사도 검색
- KnowledgeBasePage: 문서 관리 + 질문/답변 UI
- PDF 파서: zlib FlateDecode 해제 + BT/ET 텍스트 추출

Phase 13.3:
- VoiceActionService: LLM JSON 액션 플랜 생성 + 실행
- 프리셋 6개 (크롬/메모장/탐색기/볼륨), 위험 명령 차단

공통: IPC ~70채널, 에러코드 780-878, i18n 100+키
버그픽스: 라이선스 로컬 키 우선, i18n featureLabel, DOM 중첩
This commit is contained in:
Yun Chan 2026-04-05 23:52:14 +09:00
parent a31f96bbb8
commit eb83682269
38 changed files with 5678 additions and 19 deletions

View file

@ -0,0 +1,376 @@
// src/main/services/VoiceActionService.ts
// Phase 13.3: OS 자동화 — 음성 → LLM이 JSON 액션 플랜 생성 → 실행
// 사전 정의 명령 + LLM 자유 해석. 위험 액션은 차단.
import { EventEmitter } from 'events'
import { exec } from 'child_process'
import { shell } from 'electron'
import { nanoid } from 'nanoid'
import { getLogger } from './LoggerService'
import { getLocalLLMService } from './LocalLLMService'
import { configGet } from './ConfigService'
import { getMainWindow } from '../windows/WindowManager'
import { IPC_CHANNELS } from '@shared/ipc-channels'
import { D3ROError, ErrorCode } from '@shared/errors'
import type {
VoiceActionPlan,
VoiceActionPreset,
VoiceActionHistoryEntry,
VoiceActionPlannedEvent,
VoiceActionExecutedEvent,
VoiceActionErrorEvent,
} from '@shared/types'
const logger = getLogger('VoiceActionService')
/** 위험 명령어 블랙리스트 (실행 차단) */
const BLOCKED_COMMANDS = [
'rm ', 'del ', 'format ', 'rmdir', 'rd ',
'shutdown', 'restart', 'taskkill',
'reg delete', 'reg add',
'net user', 'net localgroup',
]
/** LLM에 보낼 시스템 프롬프트 */
const ACTION_SYSTEM_PROMPT = `You are an OS automation assistant. Parse the user's voice command and output a JSON action plan.
Output EXACTLY one JSON object (no markdown, no explanation):
{
"action": "open_app" | "open_url" | "open_file" | "keyboard_shortcut" | "type_text" | "system_command",
"target": "<the target value>",
"description": "<brief description of what this does>",
"safe": true | false
}
Rules:
- "open_app": target = app name (e.g., "notepad", "chrome", "code")
- "open_url": target = full URL (e.g., "https://google.com")
- "open_file": target = file path
- "keyboard_shortcut": target = key combo (e.g., "ctrl+c", "alt+tab")
- "type_text": target = text to type
- "system_command": target = shell command
- Set safe=false for destructive operations (delete, format, shutdown, etc.)
If the command is unclear, output: {"action": "type_text", "target": "", "description": "Could not parse command", "safe": true}`
/** 프리셋 명령어 (LLM 없이 바로 실행) */
const PRESETS: VoiceActionPreset[] = [
{
keywords: ['크롬 열어', '크롬', 'chrome', 'open chrome'],
action: { action: 'open_app', target: 'chrome', description: 'Open Chrome browser', safe: true },
},
{
keywords: ['메모장 열어', '메모장', 'notepad', 'open notepad'],
action: { action: 'open_app', target: 'notepad', description: 'Open Notepad', safe: true },
},
{
keywords: ['탐색기 열어', '탐색기', 'explorer', 'open explorer'],
action: { action: 'open_app', target: 'explorer', description: 'Open File Explorer', safe: true },
},
{
keywords: ['볼륨 올려', '소리 올려', 'volume up'],
action: { action: 'keyboard_shortcut', target: 'volumeup', description: 'Volume up', safe: true },
},
{
keywords: ['볼륨 내려', '소리 내려', 'volume down'],
action: { action: 'keyboard_shortcut', target: 'volumedown', description: 'Volume down', safe: true },
},
{
keywords: ['음소거', 'mute'],
action: { action: 'keyboard_shortcut', target: 'volumemute', description: 'Toggle mute', safe: true },
},
]
class VoiceActionService extends EventEmitter {
private _history: VoiceActionHistoryEntry[] = []
private _enabled = false
get isEnabled(): boolean {
return this._enabled
}
setEnabled(enabled: boolean): void {
this._enabled = enabled
}
getPresets(): VoiceActionPreset[] {
return PRESETS
}
getHistory(): VoiceActionHistoryEntry[] {
return [...this._history].reverse()
}
clearHistory(): void {
this._history = []
}
/**
*
*/
async execute(text: string): Promise<void> {
// 라이센스 체크
try {
const { getLicenseService } = await import('./LicenseService')
const { Feature } = await import('@shared/types')
const license = getLicenseService()
const access = license.canUse(Feature.OS_AUTOMATION)
if (!access.allowed) {
license.promptUpgrade(Feature.OS_AUTOMATION, 'tier_required')
throw new D3ROError(ErrorCode.FeatureNotAvailable, 'Pro+ required for OS automation')
}
} catch (err) {
if (err instanceof D3ROError) throw err
}
// 1. 프리셋 매칭
const preset = this._matchPreset(text)
if (preset) {
await this._executePlan(preset.action, text)
return
}
// 2. LLM으로 액션 플랜 생성
try {
const llmService = getLocalLLMService()
const result = await llmService.generate(text, {
systemPrompt: ACTION_SYSTEM_PROMPT,
temperature: 0.1,
})
const plan = this._parsePlan(result.text)
if (!plan) {
this._emitError(text, 'Failed to parse action plan from LLM response')
return
}
// 안전장치
if (!plan.safe) {
const entry: VoiceActionHistoryEntry = {
id: nanoid(),
userText: text,
plan,
executed: false,
timestamp: Date.now(),
}
this._history.push(entry)
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText: text })
logger.warn(`Voice action blocked (unsafe): ${plan.description}`)
return
}
await this._executePlan(plan, text)
} catch (err) {
this._emitError(text, err instanceof Error ? err.message : String(err))
}
}
private _matchPreset(text: string): VoiceActionPreset | null {
const lower = text.toLowerCase().trim()
for (const preset of PRESETS) {
for (const keyword of preset.keywords) {
if (lower.includes(keyword.toLowerCase())) {
return preset
}
}
}
return null
}
private _parsePlan(llmOutput: string): VoiceActionPlan | null {
try {
// JSON 블록 추출
const jsonMatch = llmOutput.match(/\{[\s\S]*\}/)
if (!jsonMatch) return null
const parsed = JSON.parse(jsonMatch[0]) as Record<string, unknown>
if (!parsed.action || !parsed.target) return null
return {
action: parsed.action as VoiceActionPlan['action'],
target: String(parsed.target),
description: String(parsed.description ?? ''),
safe: parsed.safe !== false,
}
} catch {
return null
}
}
private async _executePlan(plan: VoiceActionPlan, userText: string): Promise<void> {
// 차단된 명령어 체크
if (plan.action === 'system_command') {
const targetLower = plan.target.toLowerCase()
for (const blocked of BLOCKED_COMMANDS) {
if (targetLower.includes(blocked)) {
plan.safe = false
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText })
logger.warn(`Voice action blocked: ${plan.target}`)
return
}
}
}
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_PLANNED, { plan, userText } as VoiceActionPlannedEvent)
try {
switch (plan.action) {
case 'open_app':
await this._openApp(plan.target)
break
case 'open_url':
await shell.openExternal(plan.target)
break
case 'open_file':
await shell.openPath(plan.target)
break
case 'keyboard_shortcut':
await this._simulateKeyboard(plan.target)
break
case 'type_text':
// @nut-tree 사용 (lazy import)
await this._typeText(plan.target)
break
case 'system_command':
await this._runCommand(plan.target)
break
}
const entry: VoiceActionHistoryEntry = {
id: nanoid(),
userText,
plan,
executed: true,
timestamp: Date.now(),
}
this._history.push(entry)
if (this._history.length > 50) this._history.shift()
const event: VoiceActionExecutedEvent = { plan, success: true }
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_EXECUTED, event)
this.emit('action-executed', event)
logger.info(`Voice action executed: ${plan.action}${plan.target}`)
} catch (err) {
const msg = err instanceof Error ? err.message : String(err)
this._emitError(userText, msg)
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, msg)
}
}
private _openApp(appName: string): Promise<void> {
return new Promise((resolve, reject) => {
const cmd = `start "" "${appName}"`
exec(cmd, { shell: 'cmd.exe' }, (err) => {
if (err) reject(err)
else resolve()
})
})
}
private _simulateKeyboard(combo: string): Promise<void> {
return new Promise((resolve, reject) => {
// PowerShell SendKeys 사용
const keys = combo.toLowerCase()
let sendKeysStr = ''
if (keys === 'volumeup') {
// nircmd 대안: PowerShell로 볼륨 조절
const script = `
$wshell = New-Object -ComObject WScript.Shell
$wshell.SendKeys([char]175)
`
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
if (err) reject(err)
else resolve()
})
return
}
if (keys === 'volumedown') {
const script = `
$wshell = New-Object -ComObject WScript.Shell
$wshell.SendKeys([char]174)
`
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
if (err) reject(err)
else resolve()
})
return
}
if (keys === 'volumemute') {
const script = `
$wshell = New-Object -ComObject WScript.Shell
$wshell.SendKeys([char]173)
`
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
if (err) reject(err)
else resolve()
})
return
}
// 일반 키보드 단축키 (ctrl+c 등)
if (keys.includes('ctrl')) sendKeysStr += '^'
if (keys.includes('alt')) sendKeysStr += '%'
if (keys.includes('shift')) sendKeysStr += '+'
const key = keys.replace(/ctrl\+|alt\+|shift\+/g, '').trim()
sendKeysStr += key
const script = `
$wshell = New-Object -ComObject WScript.Shell
$wshell.SendKeys('${sendKeysStr}')
`
exec(`powershell -NoProfile -Command "${script}"`, (err) => {
if (err) reject(err)
else resolve()
})
})
}
private async _typeText(text: string): Promise<void> {
try {
const { getTextInsertService } = await import('./TextInsertService')
await getTextInsertService().insertText(text)
} catch (err) {
throw new D3ROError(ErrorCode.VoiceActionExecutionFailed, `Type text failed: ${err instanceof Error ? err.message : String(err)}`)
}
}
private _runCommand(command: string): Promise<void> {
return new Promise((resolve, reject) => {
exec(command, { timeout: 10000 }, (err) => {
if (err) reject(err)
else resolve()
})
})
}
private _emitError(userText: string, message: string): void {
const event: VoiceActionErrorEvent = { message, userText }
this._sendToRenderer(IPC_CHANNELS.VOICE_ACTION.ACTION_ERROR, event)
this.emit('error', event)
}
private _sendToRenderer(channel: string, data: unknown): void {
try {
const mainWindow = getMainWindow()
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send(channel, data)
}
} catch { /* ignore */ }
}
dispose(): void {
this.removeAllListeners()
}
}
// ── 싱글톤 ──
let instance: VoiceActionService | null = null
export function getVoiceActionService(): VoiceActionService {
if (!instance) {
instance = new VoiceActionService()
}
return instance
}