diff --git a/CLAUDE.md b/CLAUDE.md index cc76887..4f6b661 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ npm run typecheck # tsc --noEmit - 녹음 UI: 9개 웨이브 바, cos 분포, 100ms ## 현재 상태 -Phase 1~15 전체 완료 + 랜딩 페이지(site/). +Phase 1~15.5 전체 완료 + 랜딩 페이지(site/). 차단 이슈: @nut-tree-fork/nut-js 포크 사용 ## 설계 문서 (구현 시 Read 도구로 참조) @@ -52,6 +52,7 @@ Phase13+: VoiceConversationService, RAGService, VoiceActionService Phase14: MeetingModeService Phase14.5: MeetingDocTemplateService + 다중 문서 생성/편집/내보내기 Phase15: Auto Polish + AI 채팅 + 마인드맵 + 공유 +Phase15.5: 화자 구분 (LLM 추정 + pyannote 준비) ## DS 컴포넌트 (src/renderer/components/ds/) CrtDisplay, InstrumentPanel, Led, PhysicalButton, MetalCard, diff --git a/src/main/services/MeetingModeService.ts b/src/main/services/MeetingModeService.ts index fa50869..d5bde69 100644 --- a/src/main/services/MeetingModeService.ts +++ b/src/main/services/MeetingModeService.ts @@ -65,6 +65,10 @@ class MeetingModeService extends EventEmitter { /** CaptionService session-saved 방지 플래그 */ private _meetingModeActive = false + /** Phase 15.5-2: 녹음 중 오디오 WAV 파일 보존 */ + private _audioBuffersForFile: Buffer[] = [] + private _audioFilePath: string | null = null + // Phase 15: Meeting AI Chat private _chatHistory: Array<{ role: 'user' | 'assistant'; content: string }> = [] private _chatAbortController: AbortController | null = null @@ -179,8 +183,11 @@ class MeetingModeService extends EventEmitter { // 오디오 레벨 모니터링 시작 const { getAudioCaptureService, calculateRMS } = await import('./AudioCaptureService') const audioCaptureService = getAudioCaptureService() + this._audioBuffersForFile = [] this._audioDataHandler = (payload: { buffer: Buffer; timestamp: number }) => { this._lastRms = calculateRMS(payload.buffer) + // Phase 15.5-2: 오디오 버퍼 축적 (WAV 저장용) + this._audioBuffersForFile.push(Buffer.from(payload.buffer)) } audioCaptureService.on('audio-data', this._audioDataHandler) @@ -305,6 +312,23 @@ class MeetingModeService extends EventEmitter { updatedAt: now, }).where(eq(meetingSessions.id, sessionId)).run() + // Step 2.5: 오디오 WAV 파일 저장 (화자 구분용) + if (this._audioBuffersForFile.length > 0) { + try { + const { app } = await import('electron') + const path = await import('path') + const audioDir = path.join(app.getPath('userData'), 'meeting-audio') + if (!fs.existsSync(audioDir)) fs.mkdirSync(audioDir, { recursive: true }) + const wavPath = path.join(audioDir, `${sessionId}.wav`) + this._saveWav(wavPath, Buffer.concat(this._audioBuffersForFile)) + this._audioFilePath = wavPath + logger.info(`회의 오디오 저장: ${wavPath}`) + } catch (err) { + logger.warn(`오디오 저장 실패: ${err instanceof Error ? err.message : String(err)}`) + } + } + this._audioBuffersForFile = [] + // Step 3: 알림 this._sendProgress(sessionId, 'notifying', 95) new Notification({ @@ -487,6 +511,35 @@ class MeetingModeService extends EventEmitter { this._segments = [] this._memos = [] this._meetingModeActive = false + this._audioBuffersForFile = [] + this._audioFilePath = null + } + + /** PCM16 버퍼를 WAV 파일로 저장 (16kHz mono 16-bit) */ + private _saveWav(filePath: string, pcmBuffer: Buffer): void { + const sampleRate = 16000 + const numChannels = 1 + const bitsPerSample = 16 + const byteRate = sampleRate * numChannels * (bitsPerSample / 8) + const blockAlign = numChannels * (bitsPerSample / 8) + const dataSize = pcmBuffer.length + + const header = Buffer.alloc(44) + header.write('RIFF', 0) + header.writeUInt32LE(36 + dataSize, 4) + header.write('WAVE', 8) + header.write('fmt ', 12) + header.writeUInt32LE(16, 16) // fmt chunk size + header.writeUInt16LE(1, 20) // PCM + header.writeUInt16LE(numChannels, 22) + header.writeUInt32LE(sampleRate, 24) + header.writeUInt32LE(byteRate, 28) + header.writeUInt16LE(blockAlign, 32) + header.writeUInt16LE(bitsPerSample, 34) + header.write('data', 36) + header.writeUInt32LE(dataSize, 40) + + fs.writeFileSync(filePath, Buffer.concat([header, pcmBuffer])) } private _sendToRenderer(channel: string, data: unknown): void { @@ -941,18 +994,105 @@ ${transcript}` if (!row) throw new D3ROError(ErrorCode.MeetingSessionNotFound, `세션 없음: ${sessionId}`) if (!row.rawTranscript) throw new D3ROError(ErrorCode.DiarizationFailed, '전사 텍스트가 없습니다') - const hfToken = configGet('hfToken') as string - if (!hfToken) throw new D3ROError(ErrorCode.DiarizationTokenRequired, 'HuggingFace 토큰이 설정되지 않았습니다') - this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 10 }) - // 실제 오디오가 없으므로 LLM 기반 화자 추정 수행 + // 오디오 파일 존재 확인 + const { app } = await import('electron') + const path = await import('path') + const audioPath = path.join(app.getPath('userData'), 'meeting-audio', `${sessionId}.wav`) + const hasAudioFile = fs.existsSync(audioPath) + + const hfToken = configGet('hfToken') as string + + if (hasAudioFile && hfToken) { + // Phase 15.5-2: pyannote 실제 화자 구분 + logger.info(`pyannote 화자 구분 시작: ${audioPath}`) + this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 20 }) + + try { + const audioBuffer = fs.readFileSync(audioPath) + const serverUrl = `http://127.0.0.1:18765` + + const formData = new FormData() + formData.append('audio', new Blob([audioBuffer]), 'audio.wav') + formData.append('hf_token', hfToken) + if (numSpeakers && numSpeakers > 0) { + formData.append('num_speakers', String(numSpeakers)) + } + + this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 40 }) + + const response = await fetch(`${serverUrl}/diarize`, { + method: 'POST', + body: formData, + }) + + if (!response.ok) { + const errData = await response.json() as { message?: string } + throw new Error(errData.message ?? `Diarization failed: ${response.status}`) + } + + const diarResult = await response.json() as { + segments: Array<{ speaker: string; start: number; end: number }> + num_speakers: number + } + + this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 70 }) + + // 전사 세그먼트와 화자 세그먼트 매칭 + const transcript = row.editedTranscript ?? row.rawTranscript ?? '' + const lines = transcript.split('\n').filter((l) => l.trim()) + const diarSegments = diarResult.segments + + const labeledLines = lines.map((line) => { + const timeMatch = /^\[(\d{2}):(\d{2})\]/.exec(line) + if (!timeMatch) return line + + const timeSec = parseInt(timeMatch[1], 10) * 60 + parseInt(timeMatch[2], 10) + // 해당 시간에 가장 가까운 화자 세그먼트 찾기 + const matchedSeg = diarSegments.find( + (s) => timeSec >= s.start && timeSec <= s.end, + ) + if (matchedSeg) { + const speakerLabel = matchedSeg.speaker.replace('SPEAKER_', '화자 ') + return line.replace(/^\[(\d{2}:\d{2})\]/, `[$1] [${speakerLabel}]`) + } + return line + }) + + this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 85 }) + + db.update(meetingSessions).set({ + editedTranscript: labeledLines.join('\n'), + updatedAt: Date.now(), + }).where(eq(meetingSessions.id, sessionId)).run() + + logger.info(`pyannote 화자 구분 완료: ${diarResult.num_speakers}명, sessionId=${sessionId}`) + } catch (err) { + logger.warn(`pyannote 실패, LLM 폴백: ${err instanceof Error ? err.message : String(err)}`) + // pyannote 실패 시 LLM 폴백 + await this._diarizeLLMFallback(sessionId, row, numSpeakers) + } + } else { + // LLM 기반 화자 추정 (오디오 없거나 HF 토큰 없음) + await this._diarizeLLMFallback(sessionId, row, numSpeakers) + } + + this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 100 }) + logger.info(`화자 구분 완료: sessionId=${sessionId}`) + } + + private async _diarizeLLMFallback( + sessionId: string, + row: { rawTranscript: string | null; editedTranscript: string | null }, + numSpeakers?: number, + ): Promise { this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 30 }) const { getLocalLLMService } = await import('./LocalLLMService') const llm = getLocalLLMService() - const transcript = row.editedTranscript ?? row.rawTranscript + const transcript = row.editedTranscript ?? row.rawTranscript ?? '' const speakerHint = numSpeakers && numSpeakers > 0 ? `회의에는 총 ${numSpeakers}명의 화자가 있습니다.` : '화자 수는 문맥에서 추정하세요.' @@ -970,13 +1110,11 @@ ${speakerHint} this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 80 }) + const db = getDatabase() db.update(meetingSessions).set({ editedTranscript: result.text, updatedAt: Date.now(), }).where(eq(meetingSessions.id, sessionId)).run() - - this._sendToRenderer(IPC_CHANNELS.MEETING_MODE.DIARIZATION_PROGRESS, { sessionId, percent: 100 }) - logger.info(`화자 구분 완료: sessionId=${sessionId}`) } private _buildPdfHtml(session: MeetingSessionDetail): string {