diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee00c2..9f3e5e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **Live captions stream as you listen.** Words appear about a second after they are + spoken and keep updating while the sentence is in progress; the part that is no + longer changing is shown solid and the rest dimmed. A pause finishes the line, and + long unbroken speech is split at natural boundaries instead of waiting six seconds. +- **Captions are polished in context.** Each finished line is corrected by the local + model using the lines before it (spacing, punctuation, misheard words); rewrites + that change too much are ignored. It can be turned off in Settings. + ### Planned - macOS / Linux support - Additional Whisper model management UI diff --git a/apps/desktop/src/main/services/CaptionService.ts b/apps/desktop/src/main/services/CaptionService.ts index 7d204a9..e1169b3 100644 --- a/apps/desktop/src/main/services/CaptionService.ts +++ b/apps/desktop/src/main/services/CaptionService.ts @@ -1,6 +1,8 @@ // src/main/services/CaptionService.ts // Phase 10.1: Live Caption — 실시간 자막 서비스 -// 3초 청크 기반 스트리밍 전사. 싱글톤 + EventEmitter 패턴. +// 스트리밍 전사: 1초마다 중간 결과를 갱신하고, 말이 멈추면 문장을 확정한다 +// (StreamingCaptionTrack). 확정된 줄은 로컬 LLM이 문맥에 맞게 다듬는다. +// 싱글톤 + EventEmitter 패턴. import { EventEmitter } from 'events' import { app } from 'electron' @@ -8,9 +10,12 @@ import { getLogger } from './LoggerService' import { getAudioCaptureService, calculateRMS } from './AudioCaptureService' import { getSoundEffectService } from './SoundEffectService' import { getLocalSTTService } from './LocalSTTService' -import { getSTTManager } from './stt/STTManager' +import { getLocalLLMService } from './LocalLLMService' import { getHistoryService } from './HistoryService' import { configGet } from './ConfigService' +import { buildCaptionRefinePrompt } from './llm-prompts' +import { StreamingCaptionTrack, type CaptionTranscription } from './caption/StreamingCaptionTrack' +import { acceptCaptionRefinement } from '@d3ro/core/caption-streaming' import { showCaptionOverlay, hideCaptionOverlay, @@ -28,17 +33,16 @@ import { getMainWindow } from '../windows/WindowManager' const logger = getLogger('CaptionService') -/** 청크 수집 간격 (ms) — 6초로 충분한 컨텍스트 확보 */ -const CHUNK_INTERVAL_MS = 6000 +/** 트랙 판단 주기 (ms) — 인식 주기는 트랙이 정한다(중간 결과 1초) */ +const TICK_INTERVAL_MS = 250 /** RMS 무음 임계값 — 이하면 무음으로 판정 (SoX 캡처 레벨이 낮으므로 0.003 사용) */ const SILENCE_RMS_THRESHOLD = 0.003 -/** 유성음 프레임 비율 — 이 비율 미만이면 청크 스킵 (환각 방지) */ -const VOICED_FRAME_RATIO = 0.03 - -/** initialPrompt 컨텍스트 윈도우 (자) */ -const CONTEXT_WINDOW_SIZE = 300 +/** 문맥 다듬기 대기열 상한 — 밀리면 오래된 줄은 다듬지 않고 넘긴다 */ +const REFINE_QUEUE_LIMIT = 3 +const REFINE_TIMEOUT_MS = 6000 +const REFINE_KEEP_ALIVE = '10m' /** 기본 자막 설정 */ const DEFAULT_CONFIG: CaptionConfig = { @@ -63,21 +67,18 @@ class CaptionService extends EventEmitter { private _sessionId: string | null = null private _sessionStartedAt: number | null = null private _segments: CaptionSegment[] = [] - private _audioBuffers: Buffer[] = [] - private _chunkTimer: ReturnType | null = null - private _isProcessingChunk = false - private _previousContext = '' + private _tickTimer: ReturnType | null = null private _disposed = false - /** 현재 청크 내 유성음 프레임 수 */ - private _voicedFrameCount = 0 - /** 현재 청크 내 총 프레임 수 */ - private _totalFrameCount = 0 + /** 마이크 트랙 (audioSource='mic' 또는 'both') */ + private _micTrack: StreamingCaptionTrack | null = null + /** 시스템 소리 트랙 (audioSource='system' 또는 'both') */ + private _systemTrack: StreamingCaptionTrack | null = null + /** 'auto' 언어일 때 첫 확정 결과로 고정한다 — 중간 결과마다 언어가 흔들리지 않게 */ + private _sessionLanguage: string | null = null + private _refineQueue: CaptionSegment[] = [] + private _refining = false private _audioDataHandler: ((payload: { buffer: Buffer; timestamp: number }) => void) | null = null - /** 시스템 오디오용 별도 버퍼 (audioSource='system' 또는 'both') */ - private _systemAudioBuffers: Buffer[] = [] - private _systemVoicedFrameCount = 0 - private _systemTotalFrameCount = 0 // ── 공개 접근자 ── @@ -130,13 +131,11 @@ class CaptionService extends EventEmitter { this._sessionId = crypto.randomUUID() this._sessionStartedAt = Date.now() this._segments = [] - this._audioBuffers = [] + this._refineQueue = [] + this._sessionLanguage = null // 초기 컨텍스트 힌트 — Whisper 첫 청크 환각 방지 const lang = configGet('sttLanguage') as string | undefined - this._previousContext = lang === 'ko' ? '다음은 한국어 대화입니다.' : '' - this._isProcessingChunk = false - this._voicedFrameCount = 0 - this._totalFrameCount = 0 + const initialContext = lang === 'ko' ? '다음은 한국어 대화입니다.' : '' // ConfigService에서 저장된 오디오 소스 읽기 const savedSource = configGet('captionAudioSource' as keyof import('@d3ro/core/types').AppConfig) as unknown as string @@ -148,6 +147,7 @@ class CaptionService extends EventEmitter { // 마이크 캡처 (mic 또는 both) if (audioSource === 'mic' || audioSource === 'both') { + this._micTrack = this._createTrack('mic', initialContext) const audioCaptureService = getAudioCaptureService() this._audioDataHandler = (payload) => { this._onAudioData(payload.buffer) @@ -158,45 +158,26 @@ class CaptionService extends EventEmitter { // 시스템 오디오 캡처 요청 (system 또는 both) — 렌더러에 시작 요청 if (audioSource === 'system' || audioSource === 'both') { + this._systemTrack = this._createTrack('system', initialContext) this._sendToMainWindow(IPC_CHANNELS.CAPTION.START_SYSTEM_AUDIO, {}) - this._systemAudioBuffers = [] - this._systemVoicedFrameCount = 0 - this._systemTotalFrameCount = 0 } - // 청크 타이머 시작 - this._chunkTimer = setInterval(() => { - const promises: Promise[] = [] - - // 마이크 청크 처리 - if (audioSource === 'mic' || audioSource === 'both') { - promises.push( - this._processChunk().catch((err: unknown) => { - logger.error(`마이크 청크 처리 실패: ${err instanceof Error ? err.message : String(err)}`) - }) - ) - } - - // 시스템 오디오 청크 처리 - if (audioSource === 'system' || audioSource === 'both') { - promises.push( - this._processSystemChunk().catch((err: unknown) => { - logger.error(`시스템 오디오 청크 처리 실패: ${err instanceof Error ? err.message : String(err)}`) - }) - ) - } - - Promise.all(promises).catch(() => { /* 개별 에러는 이미 로깅됨 */ }) - }, CHUNK_INTERVAL_MS) + // 트랙 판단 타이머 — 각 트랙은 한 번에 인식 한 건만 돌린다 + this._tickTimer = setInterval(() => { + void this._micTrack?.tick() + void this._systemTrack?.tick() + }, TICK_INTERVAL_MS) this._setState('active') getSoundEffectService().play('recording-start') logger.info(`Live Caption 시작: sessionId=${this._sessionId}`) } catch (err) { - if (this._chunkTimer) { - clearInterval(this._chunkTimer) - this._chunkTimer = null + if (this._tickTimer) { + clearInterval(this._tickTimer) + this._tickTimer = null } + this._micTrack = null + this._systemTrack = null if (this._audioDataHandler) { const audioCaptureService = getAudioCaptureService() audioCaptureService.off('audio-data', this._audioDataHandler) @@ -228,19 +209,15 @@ class CaptionService extends EventEmitter { this._setState('stopping') // 타이머 정리 - if (this._chunkTimer) { - clearInterval(this._chunkTimer) - this._chunkTimer = null + if (this._tickTimer) { + clearInterval(this._tickTimer) + this._tickTimer = null } - // 잔여 오디오 처리 - try { - await this._processChunk() - } catch (err) { - logger.warn( - `잔여 오디오 처리 실패: ${err instanceof Error ? err.message : String(err)}`, - ) - } + // 잔여 오디오 처리 — 말하던 중이면 마지막 문장을 확정한다 + await Promise.all([this._micTrack?.flush(), this._systemTrack?.flush()]) + this._micTrack = null + this._systemTrack = null // 마이크 캡처 정리 if (this._audioDataHandler) { @@ -252,7 +229,6 @@ class CaptionService extends EventEmitter { // 시스템 오디오 캡처 중지 요청 this._sendToMainWindow(IPC_CHANNELS.CAPTION.STOP_SYSTEM_AUDIO, {}) - this._systemAudioBuffers = [] // 오버레이 숨김 hideCaptionOverlay() @@ -262,8 +238,8 @@ class CaptionService extends EventEmitter { this._sessionId = null this._sessionStartedAt = null - this._audioBuffers = [] - this._previousContext = '' + this._refineQueue = [] + this._sessionLanguage = null this._setState('inactive') getSoundEffectService().play('recording-stop') @@ -293,14 +269,7 @@ class CaptionService extends EventEmitter { private _onAudioData(buffer: Buffer): void { if (this._state !== 'active') return - this._audioBuffers.push(buffer) - - // RMS 기반 유성음 감지 (환각 방지) - this._totalFrameCount++ - const rms = calculateRMS(buffer) - if (rms >= SILENCE_RMS_THRESHOLD) { - this._voicedFrameCount++ - } + this._micTrack?.push(buffer) } /** @@ -309,147 +278,128 @@ class CaptionService extends EventEmitter { */ onSystemAudioData(buffer: Buffer): void { if (this._state !== 'active') return - this._systemAudioBuffers.push(buffer) + this._systemTrack?.push(buffer) + } - this._systemTotalFrameCount++ - const rms = calculateRMS(buffer) - if (rms >= SILENCE_RMS_THRESHOLD) { - this._systemVoicedFrameCount++ + // ── 내부: 스트리밍 트랙 ── + + private _createTrack(source: 'mic' | 'system', initialContext: string): StreamingCaptionTrack { + return new StreamingCaptionTrack( + { + transcribe: (audio, opts) => this._transcribe(audio, opts), + isVoiced: (chunk) => calculateRMS(chunk) >= SILENCE_RMS_THRESHOLD, + onPartial: (text, stable) => this._emitPartial(text, stable), + onFinal: (text) => this._emitFinal(text, source), + onError: (err) => { + const d3roErr = + err instanceof D3ROError + ? err + : new D3ROError( + ErrorCode.CaptionSTTFailed, + `자막 전사 실패(${source}): ${err instanceof Error ? err.message : String(err)}`, + ) + logger.error(d3roErr.message) + this.emit('error', d3roErr) + }, + }, + initialContext, + ) + } + + /** 자막은 항상 로컬 엔진으로 인식한다 — 1초마다 다시 인식하므로 클라우드로는 보낼 수 없다. */ + private async _transcribe( + audio: Buffer, + opts: { initialPrompt: string; partial: boolean }, + ): Promise { + const configured = (configGet('sttLanguage') as string | undefined) ?? 'auto' + const language = configured === 'auto' ? (this._sessionLanguage ?? 'auto') : configured + const result = await getLocalSTTService().transcribe(audio, { + language, + initialPrompt: opts.initialPrompt, + vadFilter: true, + partial: opts.partial, + }) + if (!opts.partial && configured === 'auto' && !this._sessionLanguage && result.text.trim() && result.language) { + this._sessionLanguage = result.language + } + return { + text: result.text, + segments: result.segments.map((segment) => ({ + text: segment.text, + start: segment.start, + end: segment.end, + })), } } - // ── 내부: 3초 청크 처리 ── + private _emitPartial(text: string, stable: string): void { + if (this._state !== 'active') return + const delta = { text, stable, isFinal: false } + this.emit('delta', delta) + sendToCaptionOverlay(IPC_CHANNELS.CAPTION.DELTA, delta) + this._sendToMainWindow(IPC_CHANNELS.CAPTION.DELTA, delta) + } - private async _processChunk(): Promise { - if (this._audioBuffers.length === 0) return - if (this._isProcessingChunk) return + private _emitFinal(text: string, source: 'mic' | 'system'): void { + const segment: CaptionSegment = { + id: crypto.randomUUID(), + text, + timestamp: Date.now(), + isFinal: true, + } + this._segments.push(segment) + this.emit('segment', segment) + sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment) + this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment) + logger.debug(`자막 확정(${source}): "${segment.text.substring(0, 50)}"`) + this._enqueueRefine(segment) + } - this._isProcessingChunk = true + // ── 내부: 문맥 다듬기 ── - // VAD 카운터 리셋 (로깅용) - const voicedRatio = this._totalFrameCount > 0 - ? this._voicedFrameCount / this._totalFrameCount - : 0 - this._voicedFrameCount = 0 - this._totalFrameCount = 0 - logger.debug(`청크 처리: 유성음 비율 ${(voicedRatio * 100).toFixed(1)}%`) - - // VAD 필터링은 faster-whisper 사이드카에서 처리 (vadFilter: true) - // SoX의 마이크 캡처 레벨이 매우 낮아 로컬 RMS 게이트는 신뢰 불가 + private _enqueueRefine(segment: CaptionSegment): void { + if (configGet('captionRefineEnabled') === false) return + if (!getLocalLLMService().isAvailable()) return + this._refineQueue.push(segment) + // 모델이 밀리면 오래된 줄은 포기한다 — 이미 지나간 자막을 늦게 고쳐 봐야 소용없다. + while (this._refineQueue.length > REFINE_QUEUE_LIMIT) this._refineQueue.shift() + if (!this._refining) void this._drainRefineQueue() + } + private async _drainRefineQueue(): Promise { + this._refining = true try { - const merged = Buffer.concat(this._audioBuffers) - this._audioBuffers = [] - - // 최소 오디오 크기 확인 (100ms 분량 이상) - const minBytes = 16000 * 2 * 0.1 // 100ms @ 16kHz 16bit mono - if (merged.length < minBytes) { - return + for (let next = this._refineQueue.shift(); next; next = this._refineQueue.shift()) { + await this._refine(next) } - - const sttService = getSTTManager() - const language = configGet('sttLanguage') as string | undefined - - const result = await sttService.transcribe(merged, { - language: language ?? 'auto', - initialPrompt: this._previousContext, - vadFilter: true, - }) - - if (!result.text || result.text.trim().length === 0) { - return - } - - const segment: CaptionSegment = { - id: crypto.randomUUID(), - text: result.text.trim(), - timestamp: Date.now(), - isFinal: true, - } - - this._segments.push(segment) - - // 다음 청크를 위한 컨텍스트 업데이트 (연속성 유지) - this._previousContext = result.text.trim().slice(-CONTEXT_WINDOW_SIZE) - - // 이벤트 emit - this.emit('segment', segment) - - // 오버레이 윈도우에 전송 - sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment) - - // 메인 윈도우에 전송 (Dashboard 등에서 사용) - this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment) - - logger.debug(`자막 세그먼트: "${segment.text.substring(0, 50)}"`) - } catch (err) { - const d3roErr = - err instanceof D3ROError - ? err - : new D3ROError( - ErrorCode.CaptionSTTFailed, - `자막 전사 실패: ${err instanceof Error ? err.message : String(err)}`, - ) - logger.error(`자막 전사 에러: ${d3roErr.message}`) - this.emit('error', d3roErr) } finally { - this._isProcessingChunk = false + this._refining = false } } - // ── 내부: 시스템 오디오 청크 처리 ── - - private async _processSystemChunk(): Promise { - if (this._systemAudioBuffers.length === 0) return - - // VAD 게이트 - const voicedRatio = this._systemTotalFrameCount > 0 - ? this._systemVoicedFrameCount / this._systemTotalFrameCount - : 0 - this._systemVoicedFrameCount = 0 - this._systemTotalFrameCount = 0 - - if (voicedRatio < VOICED_FRAME_RATIO) { - this._systemAudioBuffers = [] - logger.debug(`시스템 오디오 청크 스킵: 유성음 비율 ${(voicedRatio * 100).toFixed(1)}%`) - return - } - - const merged = Buffer.concat(this._systemAudioBuffers) - this._systemAudioBuffers = [] - - const minBytes = 16000 * 2 * 0.1 - if (merged.length < minBytes) return - + private async _refine(segment: CaptionSegment): Promise { + const index = this._segments.findIndex((item) => item.id === segment.id) + if (index === -1) return + const previous = this._segments.slice(Math.max(0, index - 2), index).map((item) => item.text) + const { systemPrompt, text } = buildCaptionRefinePrompt({ text: segment.text, previous }) try { - const sttService = getLocalSTTService() - const language = configGet('sttLanguage') as string | undefined - - const result = await sttService.transcribe(merged, { - language: language ?? 'auto', - initialPrompt: this._previousContext, - vadFilter: true, + const result = await getLocalLLMService().generate(text, { + systemPrompt, + temperature: 0, + maxTokens: Math.min(256, segment.text.length * 2 + 32), + timeoutMs: REFINE_TIMEOUT_MS, + keepAlive: REFINE_KEEP_ALIVE, }) - - if (!result.text || result.text.trim().length === 0) return - - const segment: CaptionSegment = { - id: crypto.randomUUID(), - text: result.text.trim(), - timestamp: Date.now(), - isFinal: true, - } - - this._segments.push(segment) - this._previousContext = result.text.trim().slice(-CONTEXT_WINDOW_SIZE) - - this.emit('segment', segment) - sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment) - this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment) - - logger.debug(`시스템 자막: "${segment.text.substring(0, 50)}"`) + const refined = acceptCaptionRefinement(segment.text, result.text) + if (!refined) return + const current = this._segments.find((item) => item.id === segment.id) + if (!current) return + current.text = refined + const update = { id: segment.id, text: refined } + sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, update) + this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, update) } catch (err) { - logger.error(`시스템 오디오 전사 실패: ${err instanceof Error ? err.message : String(err)}`) + logger.debug(`자막 다듬기 건너뜀: ${err instanceof Error ? err.message : String(err)}`) } } diff --git a/apps/desktop/src/main/services/ConfigService.ts b/apps/desktop/src/main/services/ConfigService.ts index 399d321..4180f52 100644 --- a/apps/desktop/src/main/services/ConfigService.ts +++ b/apps/desktop/src/main/services/ConfigService.ts @@ -186,6 +186,7 @@ const CONFIG_DEFAULTS: AppConfig = { activeChainId: null, captionAudioSource: 'mic', captionOverlayPosition: null, + captionRefineEnabled: true, updateChannel: 'latest', updateDeviceId: '', skippedUpdateVersion: null, diff --git a/apps/desktop/src/main/services/caption/StreamingCaptionTrack.ts b/apps/desktop/src/main/services/caption/StreamingCaptionTrack.ts new file mode 100644 index 0000000..e25fd18 --- /dev/null +++ b/apps/desktop/src/main/services/caption/StreamingCaptionTrack.ts @@ -0,0 +1,160 @@ +// src/main/services/caption/StreamingCaptionTrack.ts +// +// 오디오 한 줄기(마이크 또는 시스템 소리)의 실시간 자막 트랙. +// 판단은 core `decideCaptionTick` 이 하고, 이 클래스는 오디오 버퍼와 인식 호출만 맡는다. + +import { + CAPTION_STREAMING_DEFAULTS, + agreedCaptionPrefix, + decideCaptionTick, + planForcedCommit, + splitCaptionWords, + type CaptionTimedSegment +} from '@d3ro/core/caption-streaming' + +/** 16kHz · 16bit · mono PCM → 1ms = 32 bytes */ +const BYTES_PER_MS = 32 + +export interface CaptionTranscription { + text: string + segments: CaptionTimedSegment[] +} + +export interface StreamingCaptionTrackDeps { + transcribe: (audio: Buffer, opts: { initialPrompt: string; partial: boolean }) => Promise + /** 이 조각에 소리가 있는가 (RMS 등) */ + isVoiced: (chunk: Buffer) => boolean + /** 확정되지 않은 현재 문장 — stable 은 두 번 연속 같게 나온 앞부분 */ + onPartial: (text: string, stable: string) => void + onFinal: (text: string) => void + onError: (error: unknown) => void + now?: () => number +} + +export class StreamingCaptionTrack { + private _buffer: Buffer = Buffer.alloc(0) + private _busy = false + private _lastVoiceAt = 0 + private _lastPartialAt = 0 + private _previousWords: string[] = [] + private _context = '' + /** 버퍼를 자를 때마다 바뀐다 — 자르기 전에 시작한 인식 결과는 버린다 */ + private _epoch = 0 + private readonly _now: () => number + + constructor(private readonly _deps: StreamingCaptionTrackDeps, initialContext = '') { + this._now = _deps.now ?? Date.now + this._context = initialContext + } + + get bufferMs(): number { + return this._buffer.length / BYTES_PER_MS + } + + push(chunk: Buffer): void { + this._buffer = this._buffer.length === 0 ? chunk : Buffer.concat([this._buffer, chunk]) + if (this._deps.isVoiced(chunk)) this._lastVoiceAt = this._now() + } + + /** 주기적으로 부른다. 한 번에 인식 한 건만 돈다. */ + async tick(): Promise { + const now = this._now() + const action = decideCaptionTick({ + bufferMs: this.bufferMs, + hasVoice: this._lastVoiceAt > 0, + sinceVoiceMs: this._lastVoiceAt > 0 ? now - this._lastVoiceAt : Number.MAX_SAFE_INTEGER, + sincePartialMs: now - this._lastPartialAt, + busy: this._busy + }) + + if (action === 'trim-idle') { + this._keepTail(CAPTION_STREAMING_DEFAULTS.idleKeepMs) + return + } + if (action === 'partial') return this._run(() => this._partial()) + if (action === 'finalize') return this._run(() => this._finalize()) + if (action === 'force-commit') { + // 말이 끊기지 않고 너무 길어지면(배경 음악 등으로 무음이 없는 경우 포함) 앞쪽만 확정한다. + // 확정할 구간 경계가 끝내 안 나오면 두 배 길이에서 통째로 확정한다. + const hardLimit = this.bufferMs >= CAPTION_STREAMING_DEFAULTS.maxBufferMs * 2 + return this._run(() => (hardLimit ? this._finalize() : this._forceCommit())) + } + } + + /** 세션 종료 — 남은 말을 확정한다. */ + async flush(): Promise { + if (this._lastVoiceAt === 0 || this.bufferMs < CAPTION_STREAMING_DEFAULTS.minAudioMs) return + while (this._busy) await new Promise((resolve) => setTimeout(resolve, 20)) + await this._run(() => this._finalize()) + } + + private async _run(job: () => Promise): Promise { + this._busy = true + try { + await job() + } catch (error) { + this._deps.onError(error) + } finally { + this._busy = false + } + } + + private async _partial(): Promise { + this._lastPartialAt = this._now() + const epoch = this._epoch + const result = await this._deps.transcribe(this._buffer, { initialPrompt: this._context, partial: true }) + if (epoch !== this._epoch) return + const words = splitCaptionWords(result.text) + if (words.length === 0) return + const stable = agreedCaptionPrefix(this._previousWords, words) + this._previousWords = words + this._deps.onPartial(words.join(' '), stable.join(' ')) + } + + private async _finalize(): Promise { + const snapshotBytes = this._buffer.length + const takenAt = this._now() + const result = await this._deps.transcribe(this._buffer.subarray(0, snapshotBytes), { + initialPrompt: this._context, + partial: false + }) + this._cut(snapshotBytes) + // 인식하는 동안 들어온 오디오에 소리가 있었으면 그 소리는 다음 문장이다. + if (this._lastVoiceAt <= takenAt) this._lastVoiceAt = 0 + this._emitFinal(result.text) + } + + private async _forceCommit(): Promise { + const snapshotBytes = this._buffer.length + const result = await this._deps.transcribe(this._buffer.subarray(0, snapshotBytes), { + initialPrompt: this._context, + partial: false + }) + const plan = planForcedCommit( + result.segments, + snapshotBytes / BYTES_PER_MS / 1000, + CAPTION_STREAMING_DEFAULTS.keepTailMs / 1000 + ) + if (!plan) return + this._cut(Math.min(snapshotBytes, Math.round(plan.cutAtSec * 1000) * BYTES_PER_MS)) + this._emitFinal(plan.text) + } + + private _emitFinal(raw: string): void { + const text = raw.trim() + this._previousWords = [] + if (!text) return + this._context = `${this._context} ${text}`.trim().slice(-CAPTION_STREAMING_DEFAULTS.contextChars) + this._deps.onFinal(text) + } + + private _cut(bytes: number): void { + this._buffer = this._buffer.subarray(bytes - (bytes % 2)) + this._epoch += 1 + } + + private _keepTail(ms: number): void { + const keep = ms * BYTES_PER_MS + if (this._buffer.length > keep) this._buffer = this._buffer.subarray(this._buffer.length - keep) + } +} diff --git a/apps/desktop/src/main/services/llm-prompts.ts b/apps/desktop/src/main/services/llm-prompts.ts index 9de97c9..0d77cc3 100644 --- a/apps/desktop/src/main/services/llm-prompts.ts +++ b/apps/desktop/src/main/services/llm-prompts.ts @@ -253,3 +253,32 @@ export function buildSuggestionPrompt(input: SuggestionPromptInput): { } export { SUGGESTION_SYSTEM_PROMPT } + +/** + * 실시간 자막 문맥 다듬기 — 확정된 자막 한 줄을 앞 문맥에 맞게 고친다. + * + * 받아 적기의 오류(띄어쓰기·문장부호·잘못 들은 단어)만 고친다. 요약·의역·추가는 + * 금지이며, 호출자는 `acceptCaptionRefinement` 로 변화량을 다시 검사한다. + */ +const CAPTION_REFINE_SYSTEM_PROMPT = `실시간 음성 인식으로 받아 적은 자막 한 줄을 다듬습니다. + +규칙: +- 띄어쓰기, 문장부호, 앞 문맥으로 볼 때 명백하게 잘못 들은 단어만 고치세요. +- 뜻을 바꾸거나, 요약하거나, 말을 더하거나 빼지 마세요. +- 원문과 같은 언어, 같은 말투(존댓말/반말, 구어체)를 유지하세요. +- 고친 문장 한 줄만 출력하세요. 설명, 따옴표, 번호를 붙이지 마세요. +- 고칠 것이 없으면 원문을 그대로 출력하세요.` + +export function buildCaptionRefinePrompt(input: { + text: string + previous: readonly string[] +}): { systemPrompt: string; text: string } { + const sections: string[] = [] + const previous = input.previous.filter((line) => line.trim().length > 0).slice(-2) + if (previous.length > 0) sections.push('[앞 문맥]', ...previous, '') + sections.push('[다듬을 자막]', input.text.trim()) + return { + systemPrompt: `${SUGGESTION_NO_THINK_PREFIX}\n${CAPTION_REFINE_SYSTEM_PROMPT}`, + text: sections.join('\n') + } +} diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index d69816b..f048229 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -520,7 +520,9 @@ const electronAPI = { getConfig: () => invoke(IPC_CHANNELS.CAPTION.GET_CONFIG), onSegment: (cb: (data: CaptionSegment) => void): Unsubscribe => on(IPC_CHANNELS.CAPTION.SEGMENT, cb), - onDelta: (cb: (data: { text: string }) => void): Unsubscribe => + onSegmentUpdated: (cb: (data: { id: string; text: string }) => void): Unsubscribe => + on(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, cb), + onDelta: (cb: (data: { text: string; stable: string; isFinal: boolean }) => void): Unsubscribe => on(IPC_CHANNELS.CAPTION.DELTA, cb), onStateChanged: (cb: (data: { state: CaptionState }) => void): Unsubscribe => on(IPC_CHANNELS.CAPTION.STATE_CHANGED, cb), diff --git a/apps/desktop/src/renderer/components/SettingsModal.tsx b/apps/desktop/src/renderer/components/SettingsModal.tsx index 49f95c1..f67f72a 100644 --- a/apps/desktop/src/renderer/components/SettingsModal.tsx +++ b/apps/desktop/src/renderer/components/SettingsModal.tsx @@ -481,6 +481,27 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr + updateConfig('captionRefineEnabled', e.target.checked)} + /> + } + label={ + + + {t('settings.captionRefine')} + + + {t('settings.captionRefine.desc')} + + + } + /> + diff --git a/apps/desktop/src/renderer/popups/caption-overlay/script.js b/apps/desktop/src/renderer/popups/caption-overlay/script.js index 25bce1a..5e99082 100644 --- a/apps/desktop/src/renderer/popups/caption-overlay/script.js +++ b/apps/desktop/src/renderer/popups/caption-overlay/script.js @@ -75,6 +75,9 @@ return } + // 첫 중간 결과가 오면 준비 안내를 걷는다 — 이제 실제로 듣고 있다. + removeStatusLine() + if (!deltaLine) { deltaLine = document.createElement('div') deltaLine.className = 'caption-line delta' @@ -82,7 +85,35 @@ linesContainer.appendChild(deltaLine) } - deltaLine.textContent = data.text + // 두 번 연속 같게 들린 앞부분은 또렷하게, 아직 바뀔 수 있는 뒷부분은 흐리게. + var text = data.text || '' + var stable = data.stable || '' + if (stable && text.indexOf(stable) !== 0) stable = '' + deltaLine.textContent = '' + if (stable) deltaLine.appendChild(document.createTextNode(stable)) + var rest = text.slice(stable.length) + if (rest) { + var tentative = document.createElement('span') + tentative.className = 'tentative' + tentative.textContent = rest + deltaLine.appendChild(tentative) + } + } + + /** + * 문맥 다듬기로 고쳐진 줄을 제자리에서 바꾼다. + * @param {{id: string, text: string}} update + */ + function updateSegment(update) { + for (var i = 0; i < lines.length; i++) { + if (lines[i].id !== update.id) continue + lines[i].el.textContent = update.text + lines[i].el.classList.remove('refined') + // 다시 그려야 애니메이션이 재시작된다 + void lines[i].el.offsetWidth + lines[i].el.classList.add('refined') + return + } } /** @@ -163,6 +194,11 @@ updateDelta(data) }) + // 문맥 다듬기 결과 + window.popupAPI.on('caption:segmentUpdated', function (update) { + updateSegment(update) + }) + // 설정 업데이트 window.popupAPI.on('caption:config', function (newConfig) { applyConfig(newConfig) diff --git a/apps/desktop/src/renderer/popups/caption-overlay/style.css b/apps/desktop/src/renderer/popups/caption-overlay/style.css index de2bdd9..b286316 100644 --- a/apps/desktop/src/renderer/popups/caption-overlay/style.css +++ b/apps/desktop/src/renderer/popups/caption-overlay/style.css @@ -144,6 +144,21 @@ html, body { 50% { opacity: 0; } } +/* 아직 바뀔 수 있는 뒷부분 (중간 결과) */ +.caption-line .tentative { + opacity: 0.6; +} + +/* 문맥 다듬기로 고쳐진 줄 — 잠깐 강조 */ +.caption-line.refined { + animation: refined-flash 0.9s ease-out; +} + +@keyframes refined-flash { + 0% { border-color: var(--d3-accent-main); box-shadow: 0 0 0 2px var(--d3-accent-glow-dim); } + 100% { box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5); } +} + /* 로딩 중 표시 — 점멸 애니메이션 */ .caption-line.loading { opacity: 0.6; diff --git a/apps/desktop/tests/main/services/caption-streaming.test.ts b/apps/desktop/tests/main/services/caption-streaming.test.ts new file mode 100644 index 0000000..8b68e2c --- /dev/null +++ b/apps/desktop/tests/main/services/caption-streaming.test.ts @@ -0,0 +1,184 @@ +// tests/main/services/caption-streaming.test.ts +// 실시간 자막 스트리밍: 판단 로직(core) + 트랙(오디오 버퍼·인식 호출) 흐름. + +import { describe, it, expect } from 'vitest' +import { + CAPTION_STREAMING_DEFAULTS, + acceptCaptionRefinement, + agreedCaptionPrefix, + captionChangeRatio, + decideCaptionTick, + planForcedCommit +} from '@d3ro/core/caption-streaming' +import { + StreamingCaptionTrack, + type CaptionTranscription +} from '../../../src/main/services/caption/StreamingCaptionTrack' + +const BYTES_PER_MS = 32 + +function chunk(ms: number, voiced: boolean): Buffer { + const buffer = Buffer.alloc(ms * BYTES_PER_MS) + buffer[0] = voiced ? 1 : 0 + return buffer +} + +describe('decideCaptionTick', () => { + const base = { bufferMs: 2000, hasVoice: true, sinceVoiceMs: 100, sincePartialMs: 1500, busy: false } + + it('말하는 중이면 1초마다 중간 결과를 만든다', () => { + expect(decideCaptionTick(base)).toBe('partial') + expect(decideCaptionTick({ ...base, sincePartialMs: 400 })).toBe('wait') + }) + + it('말이 멈추면 문장을 확정한다', () => { + expect(decideCaptionTick({ ...base, sinceVoiceMs: CAPTION_STREAMING_DEFAULTS.silenceCommitMs })).toBe('finalize') + }) + + it('멈추지 않고 길어지면 앞쪽을 강제로 확정한다', () => { + expect(decideCaptionTick({ ...base, bufferMs: CAPTION_STREAMING_DEFAULTS.maxBufferMs })).toBe('force-commit') + }) + + it('소리가 없는 오디오는 들고 있지 않는다', () => { + expect(decideCaptionTick({ ...base, hasVoice: false, bufferMs: 5000 })).toBe('trim-idle') + expect(decideCaptionTick({ ...base, hasVoice: false, bufferMs: 500 })).toBe('wait') + }) + + it('인식 중이거나 오디오가 짧으면 기다린다', () => { + expect(decideCaptionTick({ ...base, busy: true })).toBe('wait') + expect(decideCaptionTick({ ...base, bufferMs: 300 })).toBe('wait') + }) +}) + +describe('LocalAgreement / 강제 확정 / 다듬기 검사', () => { + it('두 가설이 앞에서부터 같은 단어만 합의한다', () => { + expect(agreedCaptionPrefix(['오늘', '회의는', '세시에'], ['오늘', '회의는', '네시에', '해요'])).toEqual(['오늘', '회의는']) + expect(agreedCaptionPrefix([], ['오늘'])).toEqual([]) + }) + + it('끝부분(keepTail) 안쪽 구간은 남기고 앞쪽만 확정한다', () => { + const plan = planForcedCommit( + [ + { text: '첫 문장', start: 0, end: 4 }, + { text: '둘째 문장', start: 4, end: 9 }, + { text: '진행 중', start: 9, end: 12 } + ], + 12, + 1.5 + ) + expect(plan).toEqual({ text: '첫 문장 둘째 문장', cutAtSec: 9 }) + expect(planForcedCommit([{ text: '긴 한 문장', start: 0, end: 12 }], 12, 1.5)).toBeNull() + }) + + it('띄어쓰기·문장부호 수준의 수정은 받고, 뜻을 바꾼 문장은 버린다', () => { + expect(acceptCaptionRefinement('오늘회의는 세시에 시작합니다', '오늘 회의는 세 시에 시작합니다.')).toBe( + '오늘 회의는 세 시에 시작합니다.' + ) + expect(acceptCaptionRefinement('오늘 회의는 세 시에 시작합니다', '회의 일정이 변경되었다는 공지입니다')).toBeNull() + expect(acceptCaptionRefinement('그대로', '"그대로"')).toBeNull() + expect(captionChangeRatio('abc', 'abc')).toBe(0) + }) +}) + +describe('StreamingCaptionTrack', () => { + function setup(results: CaptionTranscription[]) { + let now = 1_000 + const partials: Array<{ text: string; stable: string }> = [] + const finals: string[] = [] + const calls: Array<{ bytes: number; partial: boolean; prompt: string }> = [] + const track = new StreamingCaptionTrack( + { + transcribe: async (audio, opts) => { + calls.push({ bytes: audio.length, partial: opts.partial, prompt: opts.initialPrompt }) + return results.shift() ?? { text: '', segments: [] } + }, + isVoiced: (c) => c[0] === 1, + onPartial: (text, stable) => partials.push({ text, stable }), + onFinal: (text) => finals.push(text), + onError: (error) => { + throw error + }, + now: () => now + }, + '앞 문맥' + ) + return { + track, + partials, + finals, + calls, + advance: (ms: number) => { + now += ms + } + } + } + + it('말하는 동안 단어가 갱신되고, 멈추면 한 문장으로 확정된다', async () => { + const t = setup([ + { text: '오늘 회의는', segments: [] }, + { text: '오늘 회의는 세시에', segments: [] }, + { text: '오늘 회의는 세 시에 시작합니다', segments: [] } + ]) + + t.track.push(chunk(1000, true)) + await t.track.tick() + t.advance(1000) + t.track.push(chunk(1000, true)) + await t.track.tick() + + expect(t.partials.map((p) => p.text)).toEqual(['오늘 회의는', '오늘 회의는 세시에']) + expect(t.partials[1].stable).toBe('오늘 회의는') + expect(t.calls.every((c) => c.partial)).toBe(true) + + t.advance(800) + t.track.push(chunk(800, false)) + await t.track.tick() + + expect(t.finals).toEqual(['오늘 회의는 세 시에 시작합니다']) + expect(t.calls[2].partial).toBe(false) + expect(t.calls[2].prompt).toBe('앞 문맥') + expect(t.track.bufferMs).toBe(0) + }) + + it('확정된 문장은 다음 인식의 앞 문맥이 된다', async () => { + const t = setup([{ text: '첫 문장', segments: [] }, { text: '둘째', segments: [] }]) + t.track.push(chunk(1000, true)) + t.advance(800) + await t.track.tick() + t.track.push(chunk(1000, true)) + await t.track.tick() + expect(t.calls[1].prompt).toBe('앞 문맥 첫 문장') + }) + + it('끊기지 않는 긴 소리는 앞쪽만 확정하고 그만큼 오디오를 잘라낸다', async () => { + const t = setup([ + { + text: '첫 문장 둘째 문장 진행 중', + segments: [ + { text: '첫 문장', start: 0, end: 5 }, + { text: '둘째 문장', start: 5, end: 9 }, + { text: '진행 중', start: 9, end: 12 } + ] + } + ]) + t.track.push(chunk(CAPTION_STREAMING_DEFAULTS.maxBufferMs, true)) + await t.track.tick() + expect(t.finals).toEqual(['첫 문장 둘째 문장']) + expect(t.track.bufferMs).toBe(CAPTION_STREAMING_DEFAULTS.maxBufferMs - 9000) + }) + + it('소리가 없으면 오디오를 짧게만 남기고 인식하지 않는다', async () => { + const t = setup([]) + t.track.push(chunk(5000, false)) + await t.track.tick() + expect(t.calls).toHaveLength(0) + expect(t.track.bufferMs).toBe(CAPTION_STREAMING_DEFAULTS.idleKeepMs) + }) + + it('종료할 때 말하던 문장을 확정한다', async () => { + const t = setup([{ text: '마지막 말', segments: [] }]) + t.track.push(chunk(1200, true)) + await t.track.flush() + expect(t.finals).toEqual(['마지막 말']) + }) +}) diff --git a/apps/desktop/tests/main/services/llm-prompts.test.ts b/apps/desktop/tests/main/services/llm-prompts.test.ts index b1cb68a..c7dfd4d 100644 --- a/apps/desktop/tests/main/services/llm-prompts.test.ts +++ b/apps/desktop/tests/main/services/llm-prompts.test.ts @@ -26,6 +26,7 @@ import { SUGGESTION_NO_THINK_PREFIX, SUGGESTION_SYSTEM_PROMPT, } from '../../../src/main/services/llm-prompts' +import { buildCaptionRefinePrompt } from '../../../src/main/services/llm-prompts' beforeEach(() => { vi.clearAllMocks() @@ -248,3 +249,21 @@ describe('buildSuggestionPrompt', () => { expect(text).not.toContain('이미 제안한 문장') }) }) + +describe('buildCaptionRefinePrompt', () => { + it('지시문은 시스템 프롬프트에만 두고, 본문에는 앞 문맥 2줄과 다듬을 자막만 넣는다', () => { + const { systemPrompt, text } = buildCaptionRefinePrompt({ + text: '오늘회의는 세시에', + previous: ['첫 줄', '둘째 줄', '셋째 줄'] + }) + expect(systemPrompt).toContain('뜻을 바꾸거나') + expect(text).not.toContain('규칙') + expect(text).toContain(['둘째 줄', '셋째 줄'].join('\n')) + expect(text).not.toContain('첫 줄') + expect(text.endsWith('오늘회의는 세시에')).toBe(true) + }) + + it('앞 문맥이 없으면 다듬을 자막만 넣는다', () => { + expect(buildCaptionRefinePrompt({ text: '안녕', previous: [] }).text).toBe(['[다듬을 자막]', '안녕'].join('\n')) + }) +}) diff --git a/docs/map/10-feature-catalog.md b/docs/map/10-feature-catalog.md index b5a9c20..994a9e0 100644 --- a/docs/map/10-feature-catalog.md +++ b/docs/map/10-feature-catalog.md @@ -26,7 +26,7 @@ Status quick-reference: `[x]` done+verified · `[~]` partial/unverified · `[ ]` | CAP-10 | STT model download/management UI | [x] | [-] | [~] | [-] | Desktop model manager + onboarding; mobile bundled model | | CAP-11 | File transcription (audio/video) | [x] | [ ] | [x] | [~] | Desktop ffmpeg chunking; mobile import picker; web deferred | | CAP-12 | Audio import from other apps (share intent) | [-] | [-] | [x] | [-] | Mobile Android `ACTION_SEND`/`ACTION_VIEW` (SSOT R-016 GREEN) | -| CAP-13 | Live captions overlay | [x] | [-] | [-] | [-] | Desktop `CaptionService` + caption-overlay popup; popup assets verified in the packaged build (`scripts/ci/verify-desktop-renderer-bundles.mjs`); GAP-INFRA-05 | +| CAP-13 | Live captions overlay | [x] | [-] | [-] | [-] | Desktop `CaptionService` + caption-overlay popup; popup assets verified in the packaged build (`scripts/ci/verify-desktop-renderer-bundles.mjs`); GAP-INFRA-05 **2026-09-24 (unreleased, 1.7.0 candidate):** fixed 6 s batches replaced by streaming (`StreamingCaptionTrack` + core `caption-streaming.ts`): the uncommitted buffer is re-recognised every 1 s (greedy `partial`) and sent as `caption:delta` {text, stable} where `stable` is the LocalAgreement-2 prefix; 0.7 s of silence finalises the line with a full pass; unbroken audio over 12 s commits segments ending before the last 1.5 s using Whisper segment times; idle audio is trimmed to 1.5 s. Captions always use the local engine; `auto` language is pinned after the first final. Finished lines are refined by the local LLM (`buildCaptionRefinePrompt`, `acceptCaptionRefinement` rejects >35 % change) and replaced via `caption:segmentUpdated`; toggle `captionRefineEnabled` (Settings). Overlay: draggable handle with remembered position, waiting notice until the first caption. | | CAP-14 | Recording persistence / crash recovery | [x] | [ ] | [x] | [-] | Desktop WAV persist; mobile durable queue + process-kill WAV recovery | | CAP-15 | Android foreground recording service | [-] | [-] | [x] | [-] | Mobile API 34 FGS + persistent notification (SSOT R-005 GREEN) | | CAP-16 | Rebindable global key bindings (keyboard + mouse) | [x] | [-] | [-] | [-] | Contract SSOT `packages/core/src/keybinding.ts`: `KEY_CATALOG` (10 groups, `:615`), `KEYBINDING_ACTIONS` (6 actions, `:719`), `validateBinding` (`:953`), `detectBindingConflicts` (`:1016`). Multiple bindings per action persist as one `AppConfig.keyBindings` map (`packages/core/src/types.ts:459`), replacing the four singular `*Shortcut` fields; `ConfigService` migrates legacy values once (`ConfigService.ts:142`). `KeyBindingService` hooks keyboard **and** mouse via uiohook (`KeyBindingService.ts:387`) — MB1 is not bindable, MB2/MB3 need a modifier, MB4/MB5 are free, and no mouse button can be suppressed, so the original click still fires (warning surfaced in the UI). Selection is either key-recording or a searchable grouped dropdown (`KeyBindingPicker.tsx:536`). `history-popup`/`command-popup` were hardcoded in `bootstrap.ts` and are now rebindable actions (`bootstrap.ts:159`). **Verified 2026-09-21 on Windows by a manual run** (`%APPDATA%/d3ro-voice/logs/main.log`, 12:53–13:06): `ConfigService` migrated the four legacy shortcuts with the user's non-default values preserved exactly, `KeyBindingService` loaded 6 bindings for 6 actions and started the uiohook keyboard **and** mouse hook with zero boot errors, and keyboard plus mouse (MB4/MB5) bindings were exercised through the UI. A `Loaded 7 key binding(s) … for 6 action(s)` line later in the same session shows multi-binding working end to end. The migrated map was read back from `d3ro-voice-config.json`: legacy `*Shortcut` fields gone, no `displayLabel` left. Contract evidence: `packages/core` 117 tests GREEN, no renderer type errors in the key-binding files. **Still open:** `KeyBindingService` has no unit test of its own, macOS/Linux mouse behavior is unconfirmed (`11` GAP-KEY-02), and `command` still falls back to the dictation pipeline (GAP-KEY-03). W/M `[-]`: no OS-level global binding surface exists there (browser sandbox; mobile has no global hotkey, see CAP-01). B `[-]`: device-local setting, nothing server-side. See `11` GAP-KEY-02/03 (open), GAP-KEY-01 (`[x]`), and `11` §7 CONSTRAINT-I18N-01. | diff --git a/packages/core/package.json b/packages/core/package.json index f7ef9ea..b8b8e49 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -38,6 +38,10 @@ "types": "./src/input-intelligence.ts", "default": "./src/input-intelligence.ts" }, + "./caption-streaming": { + "types": "./src/caption-streaming.ts", + "default": "./src/caption-streaming.ts" + }, "./entitlement": { "types": "./src/entitlement.ts", "default": "./src/entitlement.ts" diff --git a/packages/core/src/caption-streaming.ts b/packages/core/src/caption-streaming.ts new file mode 100644 index 0000000..3a6dcef --- /dev/null +++ b/packages/core/src/caption-streaming.ts @@ -0,0 +1,152 @@ +// packages/core/src/caption-streaming.ts +// +// 실시간 자막 스트리밍의 순수 판단 로직 — 오디오/모델/IPC 없이 테스트한다. +// +// 방식 (whisper_streaming 계열, Macháček et al. 2023 의 LocalAgreement 를 단순화): +// - 확정되지 않은 오디오 구간 전체를 짧은 주기로 다시 인식해 "지금까지 들은 문장" 을 +// 중간 결과로 내보낸다 → 단어가 실시간으로 갱신된다. +// - 말이 멈추면(무음) 그 구간을 정밀하게 한 번 더 인식해 문장을 확정한다. +// - 멈추지 않고 길어지면 모델이 준 구간 시간으로 앞쪽만 확정하고 오디오를 잘라낸다. +// - 소리가 없는 오디오는 오래 들고 있지 않는다 — Whisper 는 무음에서 문장을 지어낸다. + +export const CAPTION_STREAMING_DEFAULTS = { + /** 중간 결과 재인식 주기 */ + partialIntervalMs: 1000, + /** 인식을 시작할 최소 오디오 길이 */ + minAudioMs: 800, + /** 말이 이만큼 멈추면 문장을 확정한다 */ + silenceCommitMs: 700, + /** 멈추지 않는 말이 이 길이를 넘으면 앞쪽을 강제로 확정한다 */ + maxBufferMs: 12000, + /** 강제 확정 때 뒤쪽은 이만큼 남긴다 (끝 구간은 아직 바뀔 수 있다) */ + keepTailMs: 1500, + /** 소리가 없을 때 들고 있을 오디오 */ + idleKeepMs: 1500, + /** 다음 인식에 넘기는 앞 문맥 길이 (자) */ + contextChars: 200, + /** 문맥 다듬기 결과가 원문에서 이 비율 이상 바뀌면 버린다 */ + refineMaxChangeRatio: 0.35 +} as const + +export type CaptionTickAction = 'finalize' | 'force-commit' | 'partial' | 'trim-idle' | 'wait' + +export interface CaptionTickInput { + /** 확정되지 않은 오디오 길이 */ + bufferMs: number + /** 이 구간에 소리가 있었는가 */ + hasVoice: boolean + /** 마지막 소리 이후 지난 시간 */ + sinceVoiceMs: number + /** 마지막 중간 인식 이후 지난 시간 */ + sincePartialMs: number + /** 인식이 진행 중인가 (한 트랙에 한 건만) */ + busy: boolean +} + +/** 매 틱마다 무엇을 할지 정한다. */ +export function decideCaptionTick( + input: CaptionTickInput, + opts: Partial = {} +): CaptionTickAction { + const o = { ...CAPTION_STREAMING_DEFAULTS, ...opts } + if (input.busy) return 'wait' + if (!input.hasVoice) return input.bufferMs > o.idleKeepMs ? 'trim-idle' : 'wait' + if (input.bufferMs < o.minAudioMs) return 'wait' + if (input.sinceVoiceMs >= o.silenceCommitMs) return 'finalize' + if (input.bufferMs >= o.maxBufferMs) return 'force-commit' + if (input.sincePartialMs >= o.partialIntervalMs) return 'partial' + return 'wait' +} + +/** 공백 기준 단어 (한국어도 어절 단위로 띄어 쓴다). */ +export function splitCaptionWords(text: string): string[] { + return text.trim().split(/\s+/u).filter((word) => word.length > 0) +} + +/** + * 연속된 두 가설이 앞에서부터 합의한 단어 (LocalAgreement-2). + * + * 두 번 연속 같게 나온 앞부분은 이후 오디오가 더 와도 거의 바뀌지 않는다 — + * 중간 결과에서 이 부분은 흔들리지 않게 보여 줄 수 있다. + */ +export function agreedCaptionPrefix(previous: readonly string[], current: readonly string[]): string[] { + const agreed: string[] = [] + const length = Math.min(previous.length, current.length) + for (let i = 0; i < length; i += 1) { + if (previous[i] !== current[i]) break + agreed.push(current[i]) + } + return agreed +} + +export interface CaptionTimedSegment { + text: string + /** 초 */ + start: number + /** 초 */ + end: number +} + +export interface ForcedCommitPlan { + /** 확정할 텍스트 */ + text: string + /** 이 시점(초)까지의 오디오를 잘라낸다 */ + cutAtSec: number +} + +/** + * 길어진 구간에서 앞쪽을 확정할 계획. + * + * 끝에서 keepTail 안쪽에 걸친 구간은 아직 문장이 이어질 수 있으니 남긴다. 확정할 + * 구간이 없으면 null — 호출자는 다음 틱에 다시 본다. + */ +export function planForcedCommit( + segments: readonly CaptionTimedSegment[], + bufferSec: number, + keepTailSec: number +): ForcedCommitPlan | null { + const limit = bufferSec - keepTailSec + const committed = segments.filter((segment) => segment.end <= limit && segment.text.trim().length > 0) + if (committed.length === 0) return null + return { + text: committed.map((segment) => segment.text.trim()).join(' '), + cutAtSec: committed[committed.length - 1].end + } +} + +/** 문자 단위 편집 거리 비율 (0 = 같음, 1 = 전혀 다름). 공백은 무시한다. */ +export function captionChangeRatio(original: string, revised: string): number { + const a = [...original.replace(/\s+/gu, '')] + const b = [...revised.replace(/\s+/gu, '')] + if (a.length === 0 && b.length === 0) return 0 + let prev = Array.from({ length: b.length + 1 }, (_, j) => j) + for (let i = 1; i <= a.length; i += 1) { + const row = [i] + for (let j = 1; j <= b.length; j += 1) { + row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)) + } + prev = row + } + return prev[b.length] / Math.max(a.length, b.length) +} + +/** + * 모델이 다듬은 문장을 쓸지 정한다. + * + * 자막 다듬기는 띄어쓰기·문장부호·명백한 오인식만 고쳐야 한다. 많이 바뀌었다면 + * 모델이 요약하거나 지어낸 것이다 — 원문을 지킨다(null). + */ +export function acceptCaptionRefinement( + original: string, + refined: string, + maxChangeRatio: number = CAPTION_STREAMING_DEFAULTS.refineMaxChangeRatio +): string | null { + const line = refined + .split(/\r?\n/u) + .map((part) => part.trim()) + .find((part) => part.length > 0) + if (!line) return null + const cleaned = line.replace(/^["'“”‘’「」『』]+|["'“”‘’「」『』]+$/gu, '').trim() + if (!cleaned || cleaned === original.trim()) return null + return captionChangeRatio(original, cleaned) <= maxChangeRatio ? cleaned : null +} diff --git a/packages/core/src/ipc-channels.ts b/packages/core/src/ipc-channels.ts index 19b6907..41a984c 100644 --- a/packages/core/src/ipc-channels.ts +++ b/packages/core/src/ipc-channels.ts @@ -249,6 +249,8 @@ export const IPC_CHANNELS = { SYSTEM_AUDIO_DATA: 'caption:systemAudioData', // Main → Renderer events SEGMENT: 'caption:segment', + /** 확정된 줄을 문맥 다듬기로 고친 결과 { id, text } */ + SEGMENT_UPDATED: 'caption:segmentUpdated', DELTA: 'caption:delta', STATE_CHANGED: 'caption:stateChanged', SESSION_SAVED: 'caption:sessionSaved', diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 8e6098b..993842f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -497,6 +497,8 @@ export interface AppConfig { captionAudioSource: import('@d3ro/core/types').CaptionAudioSource /** 사용자가 끌어다 놓은 자막 창 위치 (스크린 좌표). null 이면 화면 아래 가운데 */ captionOverlayPosition: { x: number; y: number } | null + /** 확정된 자막 줄을 로컬 LLM으로 문맥에 맞게 다듬는다 */ + captionRefineEnabled: boolean /** Auto-update 채널 (latest=stable / beta / alpha). UpdateService */ updateChannel: 'latest' | 'beta' | 'alpha' /** staged rollout용 영구 기기 식별자 (개인정보 아님, 최초 실행 시 생성) */ diff --git a/packages/i18n/src/locales/de.json b/packages/i18n/src/locales/de.json index 759d078..c2c0fba 100644 --- a/packages/i18n/src/locales/de.json +++ b/packages/i18n/src/locales/de.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "{{app}} konnte nicht zu Ausschlüssen hinzugefügt werden.", "popup.caption.loading": "Sprachmodell wird vorbereitet…", "popup.caption.waiting": "Hört zu… der erste Untertitel kann einige Sekunden dauern", - "popup.caption.dragHint": "Ziehen zum Verschieben · Doppelklick setzt zurück" + "popup.caption.dragHint": "Ziehen zum Verschieben · Doppelklick setzt zurück", + "settings.captionRefine": "Untertitel im Kontext glätten", + "settings.captionRefine.desc": "Die lokale KI korrigiert Leerzeichen, Satzzeichen und falsch verstandene Wörter anhand der umgebenden Zeilen." } diff --git a/packages/i18n/src/locales/en.json b/packages/i18n/src/locales/en.json index b730282..c2dc21f 100644 --- a/packages/i18n/src/locales/en.json +++ b/packages/i18n/src/locales/en.json @@ -1903,5 +1903,7 @@ "input.feedback.excludedFailed": "Exclusions could not be saved.", "input.feedback.recommendationFailed": "Could not add {{app}} to exclusions.", "popup.caption.waiting": "Listening… the first caption can take a few seconds", - "popup.caption.dragHint": "Drag to move · double-click to reset" + "popup.caption.dragHint": "Drag to move · double-click to reset", + "settings.captionRefine": "Polish captions with context", + "settings.captionRefine.desc": "The local AI fixes spacing, punctuation and misheard words in finished captions using the surrounding lines." } diff --git a/packages/i18n/src/locales/es.json b/packages/i18n/src/locales/es.json index 47ccb98..985db3d 100644 --- a/packages/i18n/src/locales/es.json +++ b/packages/i18n/src/locales/es.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "No se pudo añadir {{app}} a las exclusiones.", "popup.caption.loading": "Preparando el modelo de voz…", "popup.caption.waiting": "Escuchando… el primer subtítulo puede tardar unos segundos", - "popup.caption.dragHint": "Arrastra para mover · doble clic para restablecer" + "popup.caption.dragHint": "Arrastra para mover · doble clic para restablecer", + "settings.captionRefine": "Pulir subtítulos con contexto", + "settings.captionRefine.desc": "La IA local corrige espacios, puntuación y palabras mal oídas usando las líneas cercanas." } diff --git a/packages/i18n/src/locales/fr.json b/packages/i18n/src/locales/fr.json index 2e4ca75..fb0f936 100644 --- a/packages/i18n/src/locales/fr.json +++ b/packages/i18n/src/locales/fr.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "Impossible d’ajouter {{app}} aux exclusions.", "popup.caption.loading": "Préparation du modèle vocal…", "popup.caption.waiting": "Écoute… le premier sous-titre peut prendre quelques secondes", - "popup.caption.dragHint": "Glisser pour déplacer · double-clic pour réinitialiser" + "popup.caption.dragHint": "Glisser pour déplacer · double-clic pour réinitialiser", + "settings.captionRefine": "Affiner les sous-titres selon le contexte", + "settings.captionRefine.desc": "L’IA locale corrige les espaces, la ponctuation et les mots mal entendus d’après les lignes voisines." } diff --git a/packages/i18n/src/locales/ja.json b/packages/i18n/src/locales/ja.json index 1dcd8fa..65da06e 100644 --- a/packages/i18n/src/locales/ja.json +++ b/packages/i18n/src/locales/ja.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "{{app}} を除外に追加できませんでした。", "popup.caption.loading": "音声モデルを準備中…", "popup.caption.waiting": "聞き取り中… 最初の字幕まで数秒かかることがあります", - "popup.caption.dragHint": "ドラッグで移動 · ダブルクリックで元の位置" + "popup.caption.dragHint": "ドラッグで移動 · ダブルクリックで元の位置", + "settings.captionRefine": "文脈で字幕を整える", + "settings.captionRefine.desc": "確定した字幕の区切り・句読点・聞き間違いを、ローカルAIが前後の文脈に合わせて直します。" } diff --git a/packages/i18n/src/locales/ko.json b/packages/i18n/src/locales/ko.json index fe9a9b2..c473cb9 100644 --- a/packages/i18n/src/locales/ko.json +++ b/packages/i18n/src/locales/ko.json @@ -1910,5 +1910,7 @@ "input.feedback.excludedFailed": "제외 목록을 저장하지 못했습니다.", "input.feedback.recommendationFailed": "{{app}}을(를) 제외 목록에 추가하지 못했습니다.", "popup.caption.waiting": "듣는 중… 첫 자막까지 몇 초 걸릴 수 있어요", - "popup.caption.dragHint": "끌어서 이동 · 더블클릭하면 원위치" + "popup.caption.dragHint": "끌어서 이동 · 더블클릭하면 원위치", + "settings.captionRefine": "자막 문맥 다듬기", + "settings.captionRefine.desc": "확정된 자막을 로컬 AI가 앞뒤 문맥에 맞게 띄어쓰기·문장부호·잘못 들은 단어를 고칩니다." } diff --git a/packages/i18n/src/locales/pt.json b/packages/i18n/src/locales/pt.json index 6331de7..c825a19 100644 --- a/packages/i18n/src/locales/pt.json +++ b/packages/i18n/src/locales/pt.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "Não foi possível adicionar {{app}} às exclusões.", "popup.caption.loading": "Preparando o modelo de voz…", "popup.caption.waiting": "Ouvindo… a primeira legenda pode levar alguns segundos", - "popup.caption.dragHint": "Arraste para mover · clique duplo para restaurar" + "popup.caption.dragHint": "Arraste para mover · clique duplo para restaurar", + "settings.captionRefine": "Refinar legendas pelo contexto", + "settings.captionRefine.desc": "A IA local corrige espaços, pontuação e palavras mal ouvidas usando as linhas próximas." } diff --git a/packages/i18n/src/locales/ru.json b/packages/i18n/src/locales/ru.json index 4407497..be3d53a 100644 --- a/packages/i18n/src/locales/ru.json +++ b/packages/i18n/src/locales/ru.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "Не удалось добавить {{app}} в исключения.", "popup.caption.loading": "Подготовка речевой модели…", "popup.caption.waiting": "Слушаю… первый субтитр может появиться через несколько секунд", - "popup.caption.dragHint": "Перетащите, чтобы переместить · двойной щелчок — сброс" + "popup.caption.dragHint": "Перетащите, чтобы переместить · двойной щелчок — сброс", + "settings.captionRefine": "Уточнять субтитры по контексту", + "settings.captionRefine.desc": "Локальный ИИ исправляет пробелы, пунктуацию и ослышки в готовых субтитрах по соседним строкам." } diff --git a/packages/i18n/src/locales/th.json b/packages/i18n/src/locales/th.json index 0ac1e89..bd55485 100644 --- a/packages/i18n/src/locales/th.json +++ b/packages/i18n/src/locales/th.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "ไม่สามารถเพิ่ม {{app}} ในรายการยกเว้นได้", "popup.caption.loading": "กำลังเตรียมโมเดลเสียง…", "popup.caption.waiting": "กำลังฟัง… คำบรรยายแรกอาจใช้เวลาสักครู่", - "popup.caption.dragHint": "ลากเพื่อย้าย · ดับเบิลคลิกเพื่อรีเซ็ต" + "popup.caption.dragHint": "ลากเพื่อย้าย · ดับเบิลคลิกเพื่อรีเซ็ต", + "settings.captionRefine": "ขัดเกลาคำบรรยายตามบริบท", + "settings.captionRefine.desc": "AI ในเครื่องจะแก้เว้นวรรค เครื่องหมายวรรคตอน และคำที่ได้ยินผิด โดยดูจากบรรทัดรอบข้าง" } diff --git a/packages/i18n/src/locales/vi.json b/packages/i18n/src/locales/vi.json index 3cb839f..eb35a60 100644 --- a/packages/i18n/src/locales/vi.json +++ b/packages/i18n/src/locales/vi.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "Không thể thêm {{app}} vào danh sách loại trừ.", "popup.caption.loading": "Đang chuẩn bị mô hình giọng nói…", "popup.caption.waiting": "Đang nghe… phụ đề đầu tiên có thể mất vài giây", - "popup.caption.dragHint": "Kéo để di chuyển · nhấp đúp để đặt lại" + "popup.caption.dragHint": "Kéo để di chuyển · nhấp đúp để đặt lại", + "settings.captionRefine": "Chỉnh phụ đề theo ngữ cảnh", + "settings.captionRefine.desc": "AI cục bộ sửa khoảng trắng, dấu câu và từ nghe nhầm dựa trên các dòng xung quanh." } diff --git a/packages/i18n/src/locales/zh-TW.json b/packages/i18n/src/locales/zh-TW.json index 8139c3c..b0e0668 100644 --- a/packages/i18n/src/locales/zh-TW.json +++ b/packages/i18n/src/locales/zh-TW.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "無法將 {{app}} 加入排除清單。", "popup.caption.loading": "正在準備語音模型…", "popup.caption.waiting": "聆聽中… 第一則字幕可能需要幾秒鐘", - "popup.caption.dragHint": "拖曳以移動 · 按兩下還原" + "popup.caption.dragHint": "拖曳以移動 · 按兩下還原", + "settings.captionRefine": "依上下文潤飾字幕", + "settings.captionRefine.desc": "本機 AI 會依上下文修正已確定字幕的空格、標點與聽錯的詞。" } diff --git a/packages/i18n/src/locales/zh.json b/packages/i18n/src/locales/zh.json index 2581963..e2f2883 100644 --- a/packages/i18n/src/locales/zh.json +++ b/packages/i18n/src/locales/zh.json @@ -522,5 +522,7 @@ "input.feedback.recommendationFailed": "无法将 {{app}} 添加到排除列表。", "popup.caption.loading": "正在准备语音模型…", "popup.caption.waiting": "正在聆听… 第一条字幕可能需要几秒钟", - "popup.caption.dragHint": "拖动以移动 · 双击复位" + "popup.caption.dragHint": "拖动以移动 · 双击复位", + "settings.captionRefine": "按上下文润色字幕", + "settings.captionRefine.desc": "本地 AI 会根据上下文修正已确定字幕的空格、标点和听错的词。" }