d3ro-voice/apps/desktop/src/main/services/AudioCaptureService.ts
Yun Chan 2d585bfc29 feat(desktop): make local speech transcription work end to end
Local dictation had never produced a transcript on an installed build. The
engine itself was healthy; every connection to it was broken.

Installed builds shipped no speech engine at all: the packaging config had no
entry for the faster-whisper sidecar and no pipeline step built one, so the app
always fell back to a system Python without the runtime. Development was broken
too, because the sidecar and SoX paths were resolved against the Vite output
directory instead of the app root, which also meant recording failed with a SoX
ENOENT. On hosts where localhost resolves only to IPv6, every local request was
refused outright, which silently disabled both local transcription and the local
LLM.

The sidecar is now built and bundled (including the Silero VAD data it needs),
gated by a packaging check that fails when the engine or its data is missing.
Paths are discovered from the app root and fail loudly when the engine is
absent. Local engine URLs are normalized to the IPv4 loopback, decoding is tuned
so repeated hallucinations cannot compound (the same transcript now takes about
a fifth of the time), the engine is warmed up at startup, and holding the hotkey
now shows the text forming live in the recording tip.
2026-09-18 00:48:47 +09:00

562 lines
18 KiB
TypeScript

// src/main/services/AudioCaptureService.ts
// 마이크 PCM 캡처 서비스. 설계서 01의 IAudioCaptureService 구현.
// Windows: SoX 직접 spawn (-t waveaudio). 기타: node-record-lpcm16.
import { EventEmitter } from 'events'
import { existsSync } from 'fs'
import { spawn, type ChildProcess } from 'child_process'
import type { Readable } from 'stream'
import { getLogger } from './LoggerService'
import { configGet } from './ConfigService'
import { getSoxPath } from '../utils/paths'
import type { AudioDevice } from '@d3ro/core/types'
import { AUDIO_FORMAT, TIMING } from '@d3ro/core/constants'
import { D3ROError, ErrorCode } from '@d3ro/core/errors'
const logger = getLogger('AudioCaptureService')
/** 60ms 프레임 크기 (바이트): 16000 * 2 * 0.06 = 1920 */
const FRAME_SIZE_BYTES = (AUDIO_FORMAT.SAMPLE_RATE * AUDIO_FORMAT.BYTES_PER_SAMPLE * 60) / 1000
export type CaptureState = 'idle' | 'starting' | 'capturing' | 'stopping' | 'error'
interface AudioCaptureEvents {
'audio-data': (payload: { buffer: Buffer; timestamp: number }) => void
'audio-level': (payload: { level: number; timestamp: number }) => void
'device-changed': (payload: {
previous: AudioDevice | null
current: AudioDevice
}) => void
started: (payload: { deviceId: string }) => void
stopped: (payload: { reason: 'manual' | 'device-lost' | 'error' }) => void
error: (payload: { error: D3ROError }) => void
}
/**
* PCM16 버퍼에서 RMS(Root Mean Square) 오디오 레벨을 계산한다.
* 반환값은 0.0 ~ 1.0 범위로 정규화된다.
*/
export function calculateRMS(buffer: Buffer): number {
const samples = buffer.length / AUDIO_FORMAT.BYTES_PER_SAMPLE
if (samples === 0) return 0
let sumSquares = 0
for (let i = 0; i < buffer.length; i += AUDIO_FORMAT.BYTES_PER_SAMPLE) {
const sample = buffer.readInt16LE(i)
sumSquares += sample * sample
}
const rms = Math.sqrt(sumSquares / samples)
// PCM16 최대값 32768로 나눠서 0.0~1.0 범위로 정규화
return Math.min(1.0, rms / 32768)
}
class AudioCaptureService extends EventEmitter {
private _state: CaptureState = 'idle'
private _currentDevice: AudioDevice | null = null
private _refCount = 0
private _cachedDevices: AudioDevice[] | null = null
private _soxProcess: ChildProcess | null = null
private _stream: Readable | null = null
private _levelInterval: ReturnType<typeof setInterval> | null = null
/** 프레임 조립용 잔여 바이트 버퍼 */
private _residualBuffer: Buffer = Buffer.alloc(0)
/** RMS 계산용 누적 버퍼 (100ms 간격 emit) */
private _levelAccumulator: Buffer[] = []
get state(): CaptureState {
return this._state
}
get currentDevice(): AudioDevice | null {
return this._currentDevice
}
async start(deviceId?: string): Promise<void> {
if (this._state === 'capturing') {
this._refCount++
logger.debug(`Reference count increased to ${this._refCount}`)
return
}
if (this._state !== 'idle' && this._state !== 'error') {
logger.warn(`Cannot start capture in state: ${this._state}`)
return
}
this._state = 'starting'
const selectedDeviceId = deviceId ?? configGet('selectedDeviceId')
try {
logger.info(
`Starting audio capture (device: ${selectedDeviceId ?? 'default'}, ` +
`format: ${AUDIO_FORMAT.SAMPLE_RATE}Hz ${AUDIO_FORMAT.CHANNELS}ch ${AUDIO_FORMAT.BIT_DEPTH}bit)`
)
// SoX 직접 spawn (Windows: -t waveaudio 필수)
const soxExe = getSoxPath()
logger.info(`Using SoX: ${soxExe}`)
if (!existsSync(soxExe) && soxExe !== 'sox') {
throw new D3ROError(
ErrorCode.AudioCaptureStartFailed,
`SoX 실행 파일을 찾을 수 없습니다: ${soxExe}`
)
}
const deviceArg = selectedDeviceId && selectedDeviceId !== 'default'
? selectedDeviceId
: 'default'
// Windows: sox -t waveaudio <device> --no-show-progress -r 16000 -c 1 -b 16 -e signed-integer -t raw -
// Linux/Mac: sox -d --no-show-progress ...
const soxArgs = process.platform === 'win32'
? [
'-t', 'waveaudio', deviceArg,
'--no-show-progress',
'-r', String(AUDIO_FORMAT.SAMPLE_RATE),
'-c', String(AUDIO_FORMAT.CHANNELS),
'-b', '16',
'-e', 'signed-integer',
'-t', 'raw',
'-',
]
: [
'-d',
'--no-show-progress',
'-r', String(AUDIO_FORMAT.SAMPLE_RATE),
'-c', String(AUDIO_FORMAT.CHANNELS),
'-b', '16',
'-e', 'signed-integer',
'-t', 'raw',
'-',
]
logger.info(`SoX args: ${soxArgs.join(' ')}`)
this._soxProcess = spawn(soxExe, soxArgs, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true })
this._stream = this._soxProcess.stdout
this._residualBuffer = Buffer.alloc(0)
this._levelAccumulator = []
if (!this._stream) {
throw new D3ROError(ErrorCode.AudioCaptureStartFailed, 'SoX stdout stream is null')
}
// 스트림 데이터 수신: 60ms 프레임 단위로 잘라 emit
this._stream.on('data', (chunk: Buffer) => {
this._onAudioChunk(chunk)
})
// SoX stderr → 로그
this._soxProcess.stderr?.on('data', (chunk: Buffer) => {
const msg = chunk.toString().trim()
if (msg) logger.warn(`SoX stderr: ${msg}`)
})
// SoX 프로세스 종료 감지
this._soxProcess.on('close', (code) => {
if (this._state === 'capturing') {
logger.warn(`SoX process exited unexpectedly (code: ${code})`)
this._handleError(
new D3ROError(ErrorCode.AudioCaptureFailed, `SoX process exited with code ${code}`),
'device-lost'
)
}
})
this._soxProcess.on('error', (err: Error) => {
logger.error(`SoX process spawn error: ${err.message}`)
const hint =
soxExe === 'sox' && (err as NodeJS.ErrnoException).code === 'ENOENT'
? ' 번들된 SoX(resources/sox/sox.exe)도, 시스템 PATH의 sox도 없습니다. `npm --prefix apps/desktop run setup:sox`로 내려받으세요.'
: ''
this._handleError(
new D3ROError(ErrorCode.AudioCaptureStartFailed, `SoX 프로세스 시작 실패: ${err.message}.${hint}`),
'error'
)
})
this._currentDevice = {
deviceId: selectedDeviceId ?? 'default',
label: 'Default Microphone',
isDefault: !selectedDeviceId || selectedDeviceId === 'default'
}
this._state = 'capturing'
this._refCount = 1
// 100ms 간격으로 audio-level 이벤트 emit
this._levelInterval = setInterval(() => {
this._emitAudioLevel()
}, TIMING.AUDIO_LEVEL_INTERVAL)
this.emit('started', { deviceId: this._currentDevice.deviceId })
logger.info('Audio capture started')
} catch (error) {
this._state = 'error'
const d3roError = new D3ROError(
ErrorCode.AudioCaptureStartFailed,
`Failed to start audio capture: ${error instanceof Error ? error.message : String(error)}`
)
this.emit('error', { error: d3roError })
throw d3roError
}
}
async stop(): Promise<void> {
if (this._state !== 'capturing') {
return
}
this._refCount--
if (this._refCount > 0) {
logger.debug(`Reference count decreased to ${this._refCount}`)
return
}
this._state = 'stopping'
this._cleanup()
this._state = 'idle'
this._currentDevice = null
this.emit('stopped', { reason: 'manual' })
logger.info('Audio capture stopped')
}
async getDevices(): Promise<AudioDevice[]> {
// 캐싱: 네이티브 프로세스 호출이 느리므로 한번만 실행
if (this._cachedDevices) return this._cachedDevices
const devices: AudioDevice[] = [
{ deviceId: 'default', label: '시스템 기본 마이크', isDefault: true },
]
try {
if (process.platform === 'win32') {
devices.push(...(await this._getDevicesWindows()))
} else if (process.platform === 'darwin') {
devices.push(...(await this._getDevicesMac()))
}
// linux: default 1개만 (pulseaudio/alsa 열거는 sox fallback -d 로 충분)
} catch (err) {
logger.warn(
`Failed to enumerate audio devices: ${err instanceof Error ? err.message : String(err)}`,
)
}
this._cachedDevices = devices
logger.debug(`Found ${devices.length} audio device(s) on ${process.platform}`)
return devices
}
/**
* Windows: PowerShell로 AudioEndpoint 열거 (비차단 비동기).
* SoX -t waveaudio는 이름으로 매칭하므로 FriendlyName을 deviceId로 사용.
*/
private async _getDevicesWindows(): Promise<AudioDevice[]> {
const { exec } = await import('child_process')
const psCommand = `[Console]::OutputEncoding = [Text.Encoding]::UTF8; Get-PnpDevice -Class AudioEndpoint -Status OK | Select-Object InstanceId, FriendlyName | ConvertTo-Json -Compress`
return new Promise<AudioDevice[]>((resolve) => {
exec(
`powershell -NoProfile -Command "${psCommand}"`,
{ encoding: 'utf8', timeout: 3000, env: { ...process.env, PYTHONIOENCODING: 'utf-8' } },
(err, stdout) => {
if (err || !stdout?.trim()) {
return resolve([
{ deviceId: 'default', label: '기본 마이크 (Default)', isDefault: true }
])
}
try {
const output = stdout.trim()
const parsed: unknown = JSON.parse(output.startsWith('[') ? output : `[${output}]`)
if (!Array.isArray(parsed) || parsed.length === 0) {
return resolve([
{ deviceId: 'default', label: '기본 마이크 (Default)', isDefault: true }
])
}
const result: AudioDevice[] = []
for (const dev of parsed) {
const d = dev as { InstanceId?: string; FriendlyName?: string }
if (d.InstanceId && d.FriendlyName) {
result.push({
deviceId: d.FriendlyName,
label: d.FriendlyName,
isDefault: result.length === 0,
})
}
}
resolve(result.length > 0 ? result : [
{ deviceId: 'default', label: '기본 마이크 (Default)', isDefault: true }
])
} catch {
resolve([
{ deviceId: 'default', label: '기본 마이크 (Default)', isDefault: true }
])
}
}
)
})
}
/**
* macOS: system_profiler SPAudioDataType -json 으로 CoreAudio 디바이스 열거.
* input 전용 디바이스만 필터 (coreaudio_device_input > 0).
* macOS SoX 빌드는 실제 디바이스 선택에 -d(default)만 지원하는 경우가 대부분이므로
* label은 UI 표시 용도, deviceId는 'default'로 통일해 실제 캡처 경로를 단순화한다.
*/
private async _getDevicesMac(): Promise<AudioDevice[]> {
const { execFile } = await import('child_process')
const output: string = await new Promise((resolve, reject) => {
execFile(
'/usr/sbin/system_profiler',
['SPAudioDataType', '-json'],
{ encoding: 'utf8', timeout: 5000, maxBuffer: 1024 * 1024 },
(err, stdout) => {
if (err) reject(err)
else resolve(stdout)
},
)
})
if (!output) return []
interface MacAudioItem {
_name?: string
coreaudio_device_input?: number
coreaudio_default_audio_input_device?: string
coreaudio_input_source?: string
}
interface MacAudioGroup {
_items?: MacAudioItem[]
}
interface MacAudioJson {
SPAudioDataType?: MacAudioGroup[]
}
const parsed = JSON.parse(output) as MacAudioJson
const groups = parsed.SPAudioDataType ?? []
const result: AudioDevice[] = []
for (const group of groups) {
const items = group._items ?? []
for (const item of items) {
const hasInput =
typeof item.coreaudio_device_input === 'number' && item.coreaudio_device_input > 0
const name = item._name ?? item.coreaudio_input_source
if (!hasInput || !name) continue
// 중복 방지 (name 기준)
if (result.some((d) => d.label === name)) continue
result.push({
// macOS SoX record 경로는 -d(default) 고정이므로 선택 시 default로 매핑.
// 라벨만 사용자 구분용으로 노출.
deviceId: 'default',
label: name,
isDefault: item.coreaudio_default_audio_input_device === 'spaudio_yes',
})
}
}
return result
}
getCurrentDevice(): AudioDevice | null {
return this._currentDevice
}
dispose(): void {
this._cleanup()
this._state = 'idle'
this._currentDevice = null
this._refCount = 0
this.removeAllListeners()
logger.info('AudioCaptureService disposed')
}
/**
* 지정 시간(ms) 동안 마이크 캡처 후 평균/피크 레벨을 반환한다.
*/
async testCapture(durationMs: number): Promise<{ averageLevel: number; peakLevel: number; hasAudio: boolean }> {
const soxExe = getSoxPath()
const selectedDeviceId = configGet('selectedDeviceId')
const deviceArg = selectedDeviceId && selectedDeviceId !== 'default'
? selectedDeviceId
: 'default'
const soxArgs = process.platform === 'win32'
? ['-t', 'waveaudio', deviceArg, '--no-show-progress',
'-r', String(AUDIO_FORMAT.SAMPLE_RATE), '-c', String(AUDIO_FORMAT.CHANNELS),
'-b', '16', '-e', 'signed-integer', '-t', 'raw', '-']
: ['-d', '--no-show-progress',
'-r', String(AUDIO_FORMAT.SAMPLE_RATE), '-c', String(AUDIO_FORMAT.CHANNELS),
'-b', '16', '-e', 'signed-integer', '-t', 'raw', '-']
return new Promise((resolve, reject) => {
const proc = spawn(soxExe, soxArgs, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
const buffers: Buffer[] = []
let peakRms = 0
proc.stdout?.on('data', (chunk: Buffer) => {
buffers.push(chunk)
const rms = calculateRMS(chunk)
if (rms > peakRms) peakRms = rms
})
proc.on('error', (err) => {
reject(new D3ROError(ErrorCode.AudioCaptureStartFailed, `SoX 실행 실패: ${err.message}`))
})
setTimeout(() => {
proc.kill()
}, durationMs)
proc.on('close', () => {
if (buffers.length === 0) {
resolve({ averageLevel: 0, peakLevel: 0, hasAudio: false })
return
}
const combined = Buffer.concat(buffers)
const avgRms = calculateRMS(combined)
const avgLevel = avgRms > 0 ? Math.min(1.0, Math.pow(avgRms, 0.28)) : 0
const peakLevel = peakRms > 0 ? Math.min(1.0, Math.pow(peakRms, 0.28)) : 0
resolve({
averageLevel: avgLevel,
peakLevel: peakLevel,
hasAudio: avgRms > 0.001,
})
})
})
}
/**
* 수신된 오디오 청크를 60ms 프레임(1920 bytes) 단위로 분할하여 emit한다.
* 잔여 바이트는 다음 청크와 결합한다.
*/
private _onAudioChunk(chunk: Buffer): void {
// 잔여 버퍼와 새 청크를 결합
const combined = this._residualBuffer.length > 0
? Buffer.concat([this._residualBuffer, chunk])
: chunk
let offset = 0
// 60ms 프레임 단위로 분할하여 emit
while (offset + FRAME_SIZE_BYTES <= combined.length) {
const frame = combined.subarray(offset, offset + FRAME_SIZE_BYTES)
offset += FRAME_SIZE_BYTES
this.emit('audio-data', {
buffer: Buffer.from(frame), // 방어적 복사
timestamp: Date.now()
})
// RMS 계산용 누적
this._levelAccumulator.push(frame)
}
// 남은 바이트는 잔여 버퍼에 보관
if (offset < combined.length) {
this._residualBuffer = Buffer.from(combined.subarray(offset))
} else {
this._residualBuffer = Buffer.alloc(0)
}
}
/**
* 100ms 간격으로 누적된 오디오 데이터의 RMS 레벨을 계산하여 emit한다.
*/
private _emitAudioLevel(): void {
if (this._levelAccumulator.length === 0) {
this.emit('audio-level', { level: 0, timestamp: Date.now() })
return
}
// 누적된 프레임들을 하나로 합쳐서 RMS 계산
const combined = Buffer.concat(this._levelAccumulator)
this._levelAccumulator = []
const rawLevel = calculateRMS(combined)
// 로그 스케일: 작은 소리도 크게, 큰 소리는 압축 (DAW 미터 방식)
// pow(x, 0.28) → 0.001→0.04, 0.01→0.14, 0.05→0.35, 0.1→0.52, 0.3→0.80
const level = rawLevel > 0 ? Math.min(1.0, Math.pow(rawLevel, 0.28)) : 0
this.emit('audio-level', { level, timestamp: Date.now() })
}
/**
* 에러 발생 시 리소스 정리 + 에러 이벤트 emit.
*/
private _handleError(error: D3ROError, stopReason: 'device-lost' | 'error'): void {
if (this._state === 'idle' || this._state === 'stopping') {
return
}
this._cleanup()
this._state = 'error'
this._currentDevice = null
this.emit('error', { error })
this.emit('stopped', { reason: stopReason })
}
/**
* 녹음 프로세스와 타이머를 정리한다.
*/
private _cleanup(): void {
if (this._levelInterval) {
clearInterval(this._levelInterval)
this._levelInterval = null
}
if (this._stream) {
this._stream.removeAllListeners()
this._stream = null
}
if (this._soxProcess) {
try {
this._soxProcess.kill()
} catch {
// 이미 종료된 프로세스 kill 시 에러 무시
}
this._soxProcess = null
}
this._residualBuffer = Buffer.alloc(0)
this._levelAccumulator = []
}
// EventEmitter 타입 오버라이드
override on<K extends keyof AudioCaptureEvents>(
event: K,
listener: AudioCaptureEvents[K]
): this {
return super.on(event, listener)
}
override off<K extends keyof AudioCaptureEvents>(
event: K,
listener: AudioCaptureEvents[K]
): this {
return super.off(event, listener)
}
override emit<K extends keyof AudioCaptureEvents>(
event: K,
...args: Parameters<AudioCaptureEvents[K]>
): boolean {
return super.emit(event, ...args)
}
}
// 싱글톤
let instance: AudioCaptureService | null = null
export function getAudioCaptureService(): AudioCaptureService {
if (!instance) {
instance = new AudioCaptureService()
}
return instance
}