feat(caption): stream live captions and polish finished lines in context

Replaces the fixed six-second batches with a streaming track per audio source: the uncommitted audio is re-recognised every second and sent as a partial with its agreed (stable) prefix, a short pause finalises the line, and long unbroken speech is committed at Whisper segment boundaries. Idle audio is trimmed so silence cannot produce invented sentences. Finished lines are corrected by the local model against the previous lines and replaced in place; edits that change too much are rejected. The behaviour can be switched off in Settings.
2026-09-24 20:22:21 +09:00 · 2026-09-24 20:22:21 +09:00 · 39b8e7448e
commit 39b8e7448e
parent db8d9448a3
28 changed files with 827 additions and 217 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased]

+### Added
+- **Live captions stream as you listen.** Words appear about a second after they are
+  spoken and keep updating while the sentence is in progress; the part that is no
+  longer changing is shown solid and the rest dimmed. A pause finishes the line, and
+  long unbroken speech is split at natural boundaries instead of waiting six seconds.
+- **Captions are polished in context.** Each finished line is corrected by the local
+  model using the lines before it (spacing, punctuation, misheard words); rewrites
+  that change too much are ignored. It can be turned off in Settings.
+
 ### Planned
 - macOS / Linux support
 - Additional Whisper model management UI
--- a/apps/desktop/src/main/services/CaptionService.ts
+++ b/apps/desktop/src/main/services/CaptionService.ts
@ -1,6 +1,8 @@
 // src/main/services/CaptionService.ts
 // Phase 10.1: Live Caption — 실시간 자막 서비스
-// 3초 청크 기반 스트리밍 전사. 싱글톤 + EventEmitter 패턴.
+// 스트리밍 전사: 1초마다 중간 결과를 갱신하고, 말이 멈추면 문장을 확정한다
+// (StreamingCaptionTrack). 확정된 줄은 로컬 LLM이 문맥에 맞게 다듬는다.
+// 싱글톤 + EventEmitter 패턴.

 import { EventEmitter } from 'events'
 import { app } from 'electron'
@ -8,9 +10,12 @@ import { getLogger } from './LoggerService'
 import { getAudioCaptureService, calculateRMS } from './AudioCaptureService'
 import { getSoundEffectService } from './SoundEffectService'
 import { getLocalSTTService } from './LocalSTTService'
-import { getSTTManager } from './stt/STTManager'
+import { getLocalLLMService } from './LocalLLMService'
 import { getHistoryService } from './HistoryService'
 import { configGet } from './ConfigService'
+import { buildCaptionRefinePrompt } from './llm-prompts'
+import { StreamingCaptionTrack, type CaptionTranscription } from './caption/StreamingCaptionTrack'
+import { acceptCaptionRefinement } from '@d3ro/core/caption-streaming'
 import {
  showCaptionOverlay,
  hideCaptionOverlay,
@ -28,17 +33,16 @@ import { getMainWindow } from '../windows/WindowManager'

 const logger = getLogger('CaptionService')

-/** 청크 수집 간격 (ms) — 6초로 충분한 컨텍스트 확보 */
-const CHUNK_INTERVAL_MS = 6000
+/** 트랙 판단 주기 (ms) — 인식 주기는 트랙이 정한다(중간 결과 1초) */
+const TICK_INTERVAL_MS = 250

 /** RMS 무음 임계값 — 이하면 무음으로 판정 (SoX 캡처 레벨이 낮으므로 0.003 사용) */
 const SILENCE_RMS_THRESHOLD = 0.003

-/** 유성음 프레임 비율 — 이 비율 미만이면 청크 스킵 (환각 방지) */
-const VOICED_FRAME_RATIO = 0.03
-
-/** initialPrompt 컨텍스트 윈도우 (자) */
-const CONTEXT_WINDOW_SIZE = 300
+/** 문맥 다듬기 대기열 상한 — 밀리면 오래된 줄은 다듬지 않고 넘긴다 */
+const REFINE_QUEUE_LIMIT = 3
+const REFINE_TIMEOUT_MS = 6000
+const REFINE_KEEP_ALIVE = '10m'

 /** 기본 자막 설정 */
 const DEFAULT_CONFIG: CaptionConfig = {
@ -63,21 +67,18 @@ class CaptionService extends EventEmitter {
  private _sessionId: string | null = null
  private _sessionStartedAt: number | null = null
  private _segments: CaptionSegment[] = []
-  private _audioBuffers: Buffer[] = []
-  private _chunkTimer: ReturnType<typeof setInterval> | null = null
-  private _isProcessingChunk = false
-  private _previousContext = ''
+  private _tickTimer: ReturnType<typeof setInterval> | null = null
  private _disposed = false
-  /** 현재 청크 내 유성음 프레임 수 */
-  private _voicedFrameCount = 0
-  /** 현재 청크 내 총 프레임 수 */
-  private _totalFrameCount = 0
+  /** 마이크 트랙 (audioSource='mic' 또는 'both') */
+  private _micTrack: StreamingCaptionTrack | null = null
+  /** 시스템 소리 트랙 (audioSource='system' 또는 'both') */
+  private _systemTrack: StreamingCaptionTrack | null = null
+  /** 'auto' 언어일 때 첫 확정 결과로 고정한다 — 중간 결과마다 언어가 흔들리지 않게 */
+  private _sessionLanguage: string | null = null
+  private _refineQueue: CaptionSegment[] = []
+  private _refining = false

  private _audioDataHandler: ((payload: { buffer: Buffer; timestamp: number }) => void) | null = null
-  /** 시스템 오디오용 별도 버퍼 (audioSource='system' 또는 'both') */
-  private _systemAudioBuffers: Buffer[] = []
-  private _systemVoicedFrameCount = 0
-  private _systemTotalFrameCount = 0

  // ── 공개 접근자 ──

@ -130,13 +131,11 @@ class CaptionService extends EventEmitter {
      this._sessionId = crypto.randomUUID()
      this._sessionStartedAt = Date.now()
      this._segments = []
-      this._audioBuffers = []
+      this._refineQueue = []
+      this._sessionLanguage = null
      // 초기 컨텍스트 힌트 — Whisper 첫 청크 환각 방지
      const lang = configGet('sttLanguage') as string | undefined
-      this._previousContext = lang === 'ko' ? '다음은 한국어 대화입니다.' : ''
-      this._isProcessingChunk = false
-      this._voicedFrameCount = 0
-      this._totalFrameCount = 0
+      const initialContext = lang === 'ko' ? '다음은 한국어 대화입니다.' : ''

      // ConfigService에서 저장된 오디오 소스 읽기
      const savedSource = configGet('captionAudioSource' as keyof import('@d3ro/core/types').AppConfig) as unknown as string
@ -148,6 +147,7 @@ class CaptionService extends EventEmitter {

      // 마이크 캡처 (mic 또는 both)
      if (audioSource === 'mic' || audioSource === 'both') {
+        this._micTrack = this._createTrack('mic', initialContext)
        const audioCaptureService = getAudioCaptureService()
        this._audioDataHandler = (payload) => {
          this._onAudioData(payload.buffer)
@ -158,45 +158,26 @@ class CaptionService extends EventEmitter {

      // 시스템 오디오 캡처 요청 (system 또는 both) — 렌더러에 시작 요청
      if (audioSource === 'system' || audioSource === 'both') {
+        this._systemTrack = this._createTrack('system', initialContext)
        this._sendToMainWindow(IPC_CHANNELS.CAPTION.START_SYSTEM_AUDIO, {})
-        this._systemAudioBuffers = []
-        this._systemVoicedFrameCount = 0
-        this._systemTotalFrameCount = 0
      }

-      // 청크 타이머 시작
-      this._chunkTimer = setInterval(() => {
-        const promises: Promise<void>[] = []
-
-        // 마이크 청크 처리
-        if (audioSource === 'mic' || audioSource === 'both') {
-          promises.push(
-            this._processChunk().catch((err: unknown) => {
-              logger.error(`마이크 청크 처리 실패: ${err instanceof Error ? err.message : String(err)}`)
-            })
-          )
-        }
-
-        // 시스템 오디오 청크 처리
-        if (audioSource === 'system' || audioSource === 'both') {
-          promises.push(
-            this._processSystemChunk().catch((err: unknown) => {
-              logger.error(`시스템 오디오 청크 처리 실패: ${err instanceof Error ? err.message : String(err)}`)
-            })
-          )
-        }
-
-        Promise.all(promises).catch(() => { /* 개별 에러는 이미 로깅됨 */ })
-      }, CHUNK_INTERVAL_MS)
+      // 트랙 판단 타이머 — 각 트랙은 한 번에 인식 한 건만 돌린다
+      this._tickTimer = setInterval(() => {
+        void this._micTrack?.tick()
+        void this._systemTrack?.tick()
+      }, TICK_INTERVAL_MS)

      this._setState('active')
      getSoundEffectService().play('recording-start')
      logger.info(`Live Caption 시작: sessionId=${this._sessionId}`)
    } catch (err) {
-      if (this._chunkTimer) {
-        clearInterval(this._chunkTimer)
-        this._chunkTimer = null
+      if (this._tickTimer) {
+        clearInterval(this._tickTimer)
+        this._tickTimer = null
      }
+      this._micTrack = null
+      this._systemTrack = null
      if (this._audioDataHandler) {
        const audioCaptureService = getAudioCaptureService()
        audioCaptureService.off('audio-data', this._audioDataHandler)
@ -228,19 +209,15 @@ class CaptionService extends EventEmitter {
    this._setState('stopping')

    // 타이머 정리
-    if (this._chunkTimer) {
-      clearInterval(this._chunkTimer)
-      this._chunkTimer = null
+    if (this._tickTimer) {
+      clearInterval(this._tickTimer)
+      this._tickTimer = null
    }

-    // 잔여 오디오 처리
-    try {
-      await this._processChunk()
-    } catch (err) {
-      logger.warn(
-        `잔여 오디오 처리 실패: ${err instanceof Error ? err.message : String(err)}`,
-      )
-    }
+    // 잔여 오디오 처리 — 말하던 중이면 마지막 문장을 확정한다
+    await Promise.all([this._micTrack?.flush(), this._systemTrack?.flush()])
+    this._micTrack = null
+    this._systemTrack = null

    // 마이크 캡처 정리
    if (this._audioDataHandler) {
@ -252,7 +229,6 @@ class CaptionService extends EventEmitter {

    // 시스템 오디오 캡처 중지 요청
    this._sendToMainWindow(IPC_CHANNELS.CAPTION.STOP_SYSTEM_AUDIO, {})
-    this._systemAudioBuffers = []

    // 오버레이 숨김
    hideCaptionOverlay()
@ -262,8 +238,8 @@ class CaptionService extends EventEmitter {

    this._sessionId = null
    this._sessionStartedAt = null
-    this._audioBuffers = []
-    this._previousContext = ''
+    this._refineQueue = []
+    this._sessionLanguage = null

    this._setState('inactive')
    getSoundEffectService().play('recording-stop')
@ -293,14 +269,7 @@ class CaptionService extends EventEmitter {

  private _onAudioData(buffer: Buffer): void {
    if (this._state !== 'active') return
-    this._audioBuffers.push(buffer)
-
-    // RMS 기반 유성음 감지 (환각 방지)
-    this._totalFrameCount++
-    const rms = calculateRMS(buffer)
-    if (rms >= SILENCE_RMS_THRESHOLD) {
-      this._voicedFrameCount++
-    }
+    this._micTrack?.push(buffer)
  }

  /**
@ -309,147 +278,128 @@ class CaptionService extends EventEmitter {
   */
  onSystemAudioData(buffer: Buffer): void {
    if (this._state !== 'active') return
-    this._systemAudioBuffers.push(buffer)
+    this._systemTrack?.push(buffer)
+  }

-    this._systemTotalFrameCount++
-    const rms = calculateRMS(buffer)
-    if (rms >= SILENCE_RMS_THRESHOLD) {
-      this._systemVoicedFrameCount++
+  // ── 내부: 스트리밍 트랙 ──
+
+  private _createTrack(source: 'mic' | 'system', initialContext: string): StreamingCaptionTrack {
+    return new StreamingCaptionTrack(
+      {
+        transcribe: (audio, opts) => this._transcribe(audio, opts),
+        isVoiced: (chunk) => calculateRMS(chunk) >= SILENCE_RMS_THRESHOLD,
+        onPartial: (text, stable) => this._emitPartial(text, stable),
+        onFinal: (text) => this._emitFinal(text, source),
+        onError: (err) => {
+          const d3roErr =
+            err instanceof D3ROError
+              ? err
+              : new D3ROError(
+                  ErrorCode.CaptionSTTFailed,
+                  `자막 전사 실패(${source}): ${err instanceof Error ? err.message : String(err)}`,
+                )
+          logger.error(d3roErr.message)
+          this.emit('error', d3roErr)
+        },
+      },
+      initialContext,
+    )
+  }
+
+  /** 자막은 항상 로컬 엔진으로 인식한다 — 1초마다 다시 인식하므로 클라우드로는 보낼 수 없다. */
+  private async _transcribe(
+    audio: Buffer,
+    opts: { initialPrompt: string; partial: boolean },
+  ): Promise<CaptionTranscription> {
+    const configured = (configGet('sttLanguage') as string | undefined) ?? 'auto'
+    const language = configured === 'auto' ? (this._sessionLanguage ?? 'auto') : configured
+    const result = await getLocalSTTService().transcribe(audio, {
+      language,
+      initialPrompt: opts.initialPrompt,
+      vadFilter: true,
+      partial: opts.partial,
+    })
+    if (!opts.partial && configured === 'auto' && !this._sessionLanguage && result.text.trim() && result.language) {
+      this._sessionLanguage = result.language
+    }
+    return {
+      text: result.text,
+      segments: result.segments.map((segment) => ({
+        text: segment.text,
+        start: segment.start,
+        end: segment.end,
+      })),
    }
  }

-  // ── 내부: 3초 청크 처리 ──
+  private _emitPartial(text: string, stable: string): void {
+    if (this._state !== 'active') return
+    const delta = { text, stable, isFinal: false }
+    this.emit('delta', delta)
+    sendToCaptionOverlay(IPC_CHANNELS.CAPTION.DELTA, delta)
+    this._sendToMainWindow(IPC_CHANNELS.CAPTION.DELTA, delta)
+  }

-  private async _processChunk(): Promise<void> {
-    if (this._audioBuffers.length === 0) return
-    if (this._isProcessingChunk) return
+  private _emitFinal(text: string, source: 'mic' | 'system'): void {
+    const segment: CaptionSegment = {
+      id: crypto.randomUUID(),
+      text,
+      timestamp: Date.now(),
+      isFinal: true,
+    }
+    this._segments.push(segment)
+    this.emit('segment', segment)
+    sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment)
+    this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment)
+    logger.debug(`자막 확정(${source}): "${segment.text.substring(0, 50)}"`)
+    this._enqueueRefine(segment)
+  }

-    this._isProcessingChunk = true
+  // ── 내부: 문맥 다듬기 ──

-    // VAD 카운터 리셋 (로깅용)
-    const voicedRatio = this._totalFrameCount > 0
-      ? this._voicedFrameCount / this._totalFrameCount
-      : 0
-    this._voicedFrameCount = 0
-    this._totalFrameCount = 0
-    logger.debug(`청크 처리: 유성음 비율 ${(voicedRatio * 100).toFixed(1)}%`)
-
-    // VAD 필터링은 faster-whisper 사이드카에서 처리 (vadFilter: true)
-    // SoX의 마이크 캡처 레벨이 매우 낮아 로컬 RMS 게이트는 신뢰 불가
+  private _enqueueRefine(segment: CaptionSegment): void {
+    if (configGet('captionRefineEnabled') === false) return
+    if (!getLocalLLMService().isAvailable()) return
+    this._refineQueue.push(segment)
+    // 모델이 밀리면 오래된 줄은 포기한다 — 이미 지나간 자막을 늦게 고쳐 봐야 소용없다.
+    while (this._refineQueue.length > REFINE_QUEUE_LIMIT) this._refineQueue.shift()
+    if (!this._refining) void this._drainRefineQueue()
+  }

+  private async _drainRefineQueue(): Promise<void> {
+    this._refining = true
    try {
-      const merged = Buffer.concat(this._audioBuffers)
-      this._audioBuffers = []
-
-      // 최소 오디오 크기 확인 (100ms 분량 이상)
-      const minBytes = 16000 * 2 * 0.1 // 100ms @ 16kHz 16bit mono
-      if (merged.length < minBytes) {
-        return
+      for (let next = this._refineQueue.shift(); next; next = this._refineQueue.shift()) {
+        await this._refine(next)
      }
-
-      const sttService = getSTTManager()
-      const language = configGet('sttLanguage') as string | undefined
-
-      const result = await sttService.transcribe(merged, {
-        language: language ?? 'auto',
-        initialPrompt: this._previousContext,
-        vadFilter: true,
-      })
-
-      if (!result.text || result.text.trim().length === 0) {
-        return
-      }
-
-      const segment: CaptionSegment = {
-        id: crypto.randomUUID(),
-        text: result.text.trim(),
-        timestamp: Date.now(),
-        isFinal: true,
-      }
-
-      this._segments.push(segment)
-
-      // 다음 청크를 위한 컨텍스트 업데이트 (연속성 유지)
-      this._previousContext = result.text.trim().slice(-CONTEXT_WINDOW_SIZE)
-
-      // 이벤트 emit
-      this.emit('segment', segment)
-
-      // 오버레이 윈도우에 전송
-      sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment)
-
-      // 메인 윈도우에 전송 (Dashboard 등에서 사용)
-      this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment)
-
-      logger.debug(`자막 세그먼트: "${segment.text.substring(0, 50)}"`)
-    } catch (err) {
-      const d3roErr =
-        err instanceof D3ROError
-          ? err
-          : new D3ROError(
-              ErrorCode.CaptionSTTFailed,
-              `자막 전사 실패: ${err instanceof Error ? err.message : String(err)}`,
-            )
-      logger.error(`자막 전사 에러: ${d3roErr.message}`)
-      this.emit('error', d3roErr)
    } finally {
-      this._isProcessingChunk = false
+      this._refining = false
    }
  }

-  // ── 내부: 시스템 오디오 청크 처리 ──
-
-  private async _processSystemChunk(): Promise<void> {
-    if (this._systemAudioBuffers.length === 0) return
-
-    // VAD 게이트
-    const voicedRatio = this._systemTotalFrameCount > 0
-      ? this._systemVoicedFrameCount / this._systemTotalFrameCount
-      : 0
-    this._systemVoicedFrameCount = 0
-    this._systemTotalFrameCount = 0
-
-    if (voicedRatio < VOICED_FRAME_RATIO) {
-      this._systemAudioBuffers = []
-      logger.debug(`시스템 오디오 청크 스킵: 유성음 비율 ${(voicedRatio * 100).toFixed(1)}%`)
-      return
-    }
-
-    const merged = Buffer.concat(this._systemAudioBuffers)
-    this._systemAudioBuffers = []
-
-    const minBytes = 16000 * 2 * 0.1
-    if (merged.length < minBytes) return
-
+  private async _refine(segment: CaptionSegment): Promise<void> {
+    const index = this._segments.findIndex((item) => item.id === segment.id)
+    if (index === -1) return
+    const previous = this._segments.slice(Math.max(0, index - 2), index).map((item) => item.text)
+    const { systemPrompt, text } = buildCaptionRefinePrompt({ text: segment.text, previous })
    try {
-      const sttService = getLocalSTTService()
-      const language = configGet('sttLanguage') as string | undefined
-
-      const result = await sttService.transcribe(merged, {
-        language: language ?? 'auto',
-        initialPrompt: this._previousContext,
-        vadFilter: true,
+      const result = await getLocalLLMService().generate(text, {
+        systemPrompt,
+        temperature: 0,
+        maxTokens: Math.min(256, segment.text.length * 2 + 32),
+        timeoutMs: REFINE_TIMEOUT_MS,
+        keepAlive: REFINE_KEEP_ALIVE,
      })
-
-      if (!result.text || result.text.trim().length === 0) return
-
-      const segment: CaptionSegment = {
-        id: crypto.randomUUID(),
-        text: result.text.trim(),
-        timestamp: Date.now(),
-        isFinal: true,
-      }
-
-      this._segments.push(segment)
-      this._previousContext = result.text.trim().slice(-CONTEXT_WINDOW_SIZE)
-
-      this.emit('segment', segment)
-      sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT, segment)
-      this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT, segment)
-
-      logger.debug(`시스템 자막: "${segment.text.substring(0, 50)}"`)
+      const refined = acceptCaptionRefinement(segment.text, result.text)
+      if (!refined) return
+      const current = this._segments.find((item) => item.id === segment.id)
+      if (!current) return
+      current.text = refined
+      const update = { id: segment.id, text: refined }
+      sendToCaptionOverlay(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, update)
+      this._sendToMainWindow(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, update)
    } catch (err) {
-      logger.error(`시스템 오디오 전사 실패: ${err instanceof Error ? err.message : String(err)}`)
+      logger.debug(`자막 다듬기 건너뜀: ${err instanceof Error ? err.message : String(err)}`)
    }
  }

--- a/apps/desktop/src/main/services/ConfigService.ts
+++ b/apps/desktop/src/main/services/ConfigService.ts
@ -186,6 +186,7 @@ const CONFIG_DEFAULTS: AppConfig = {
  activeChainId: null,
  captionAudioSource: 'mic',
  captionOverlayPosition: null,
+  captionRefineEnabled: true,
  updateChannel: 'latest',
  updateDeviceId: '',
  skippedUpdateVersion: null,
--- a/apps/desktop/src/main/services/caption/StreamingCaptionTrack.ts
+++ b/apps/desktop/src/main/services/caption/StreamingCaptionTrack.ts
@ -0,0 +1,160 @@
+// src/main/services/caption/StreamingCaptionTrack.ts
+//
+// 오디오 한 줄기(마이크 또는 시스템 소리)의 실시간 자막 트랙.
+// 판단은 core `decideCaptionTick` 이 하고, 이 클래스는 오디오 버퍼와 인식 호출만 맡는다.
+
+import {
+  CAPTION_STREAMING_DEFAULTS,
+  agreedCaptionPrefix,
+  decideCaptionTick,
+  planForcedCommit,
+  splitCaptionWords,
+  type CaptionTimedSegment
+} from '@d3ro/core/caption-streaming'
+
+/** 16kHz · 16bit · mono PCM → 1ms = 32 bytes */
+const BYTES_PER_MS = 32
+
+export interface CaptionTranscription {
+  text: string
+  segments: CaptionTimedSegment[]
+}
+
+export interface StreamingCaptionTrackDeps {
+  transcribe: (audio: Buffer, opts: { initialPrompt: string; partial: boolean }) => Promise<CaptionTranscription>
+  /** 이 조각에 소리가 있는가 (RMS 등) */
+  isVoiced: (chunk: Buffer) => boolean
+  /** 확정되지 않은 현재 문장 — stable 은 두 번 연속 같게 나온 앞부분 */
+  onPartial: (text: string, stable: string) => void
+  onFinal: (text: string) => void
+  onError: (error: unknown) => void
+  now?: () => number
+}
+
+export class StreamingCaptionTrack {
+  private _buffer: Buffer = Buffer.alloc(0)
+  private _busy = false
+  private _lastVoiceAt = 0
+  private _lastPartialAt = 0
+  private _previousWords: string[] = []
+  private _context = ''
+  /** 버퍼를 자를 때마다 바뀐다 — 자르기 전에 시작한 인식 결과는 버린다 */
+  private _epoch = 0
+  private readonly _now: () => number
+
+  constructor(private readonly _deps: StreamingCaptionTrackDeps, initialContext = '') {
+    this._now = _deps.now ?? Date.now
+    this._context = initialContext
+  }
+
+  get bufferMs(): number {
+    return this._buffer.length / BYTES_PER_MS
+  }
+
+  push(chunk: Buffer): void {
+    this._buffer = this._buffer.length === 0 ? chunk : Buffer.concat([this._buffer, chunk])
+    if (this._deps.isVoiced(chunk)) this._lastVoiceAt = this._now()
+  }
+
+  /** 주기적으로 부른다. 한 번에 인식 한 건만 돈다. */
+  async tick(): Promise<void> {
+    const now = this._now()
+    const action = decideCaptionTick({
+      bufferMs: this.bufferMs,
+      hasVoice: this._lastVoiceAt > 0,
+      sinceVoiceMs: this._lastVoiceAt > 0 ? now - this._lastVoiceAt : Number.MAX_SAFE_INTEGER,
+      sincePartialMs: now - this._lastPartialAt,
+      busy: this._busy
+    })
+
+    if (action === 'trim-idle') {
+      this._keepTail(CAPTION_STREAMING_DEFAULTS.idleKeepMs)
+      return
+    }
+    if (action === 'partial') return this._run(() => this._partial())
+    if (action === 'finalize') return this._run(() => this._finalize())
+    if (action === 'force-commit') {
+      // 말이 끊기지 않고 너무 길어지면(배경 음악 등으로 무음이 없는 경우 포함) 앞쪽만 확정한다.
+      // 확정할 구간 경계가 끝내 안 나오면 두 배 길이에서 통째로 확정한다.
+      const hardLimit = this.bufferMs >= CAPTION_STREAMING_DEFAULTS.maxBufferMs * 2
+      return this._run(() => (hardLimit ? this._finalize() : this._forceCommit()))
+    }
+  }
+
+  /** 세션 종료 — 남은 말을 확정한다. */
+  async flush(): Promise<void> {
+    if (this._lastVoiceAt === 0 || this.bufferMs < CAPTION_STREAMING_DEFAULTS.minAudioMs) return
+    while (this._busy) await new Promise((resolve) => setTimeout(resolve, 20))
+    await this._run(() => this._finalize())
+  }
+
+  private async _run(job: () => Promise<void>): Promise<void> {
+    this._busy = true
+    try {
+      await job()
+    } catch (error) {
+      this._deps.onError(error)
+    } finally {
+      this._busy = false
+    }
+  }
+
+  private async _partial(): Promise<void> {
+    this._lastPartialAt = this._now()
+    const epoch = this._epoch
+    const result = await this._deps.transcribe(this._buffer, { initialPrompt: this._context, partial: true })
+    if (epoch !== this._epoch) return
+    const words = splitCaptionWords(result.text)
+    if (words.length === 0) return
+    const stable = agreedCaptionPrefix(this._previousWords, words)
+    this._previousWords = words
+    this._deps.onPartial(words.join(' '), stable.join(' '))
+  }
+
+  private async _finalize(): Promise<void> {
+    const snapshotBytes = this._buffer.length
+    const takenAt = this._now()
+    const result = await this._deps.transcribe(this._buffer.subarray(0, snapshotBytes), {
+      initialPrompt: this._context,
+      partial: false
+    })
+    this._cut(snapshotBytes)
+    // 인식하는 동안 들어온 오디오에 소리가 있었으면 그 소리는 다음 문장이다.
+    if (this._lastVoiceAt <= takenAt) this._lastVoiceAt = 0
+    this._emitFinal(result.text)
+  }
+
+  private async _forceCommit(): Promise<void> {
+    const snapshotBytes = this._buffer.length
+    const result = await this._deps.transcribe(this._buffer.subarray(0, snapshotBytes), {
+      initialPrompt: this._context,
+      partial: false
+    })
+    const plan = planForcedCommit(
+      result.segments,
+      snapshotBytes / BYTES_PER_MS / 1000,
+      CAPTION_STREAMING_DEFAULTS.keepTailMs / 1000
+    )
+    if (!plan) return
+    this._cut(Math.min(snapshotBytes, Math.round(plan.cutAtSec * 1000) * BYTES_PER_MS))
+    this._emitFinal(plan.text)
+  }
+
+  private _emitFinal(raw: string): void {
+    const text = raw.trim()
+    this._previousWords = []
+    if (!text) return
+    this._context = `${this._context} ${text}`.trim().slice(-CAPTION_STREAMING_DEFAULTS.contextChars)
+    this._deps.onFinal(text)
+  }
+
+  private _cut(bytes: number): void {
+    this._buffer = this._buffer.subarray(bytes - (bytes % 2))
+    this._epoch += 1
+  }
+
+  private _keepTail(ms: number): void {
+    const keep = ms * BYTES_PER_MS
+    if (this._buffer.length > keep) this._buffer = this._buffer.subarray(this._buffer.length - keep)
+  }
+}
--- a/apps/desktop/src/main/services/llm-prompts.ts
+++ b/apps/desktop/src/main/services/llm-prompts.ts
@ -253,3 +253,32 @@ export function buildSuggestionPrompt(input: SuggestionPromptInput): {
 }

 export { SUGGESTION_SYSTEM_PROMPT }
+
+/**
+ * 실시간 자막 문맥 다듬기 — 확정된 자막 한 줄을 앞 문맥에 맞게 고친다.
+ *
+ * 받아 적기의 오류(띄어쓰기·문장부호·잘못 들은 단어)만 고친다. 요약·의역·추가는
+ * 금지이며, 호출자는 `acceptCaptionRefinement` 로 변화량을 다시 검사한다.
+ */
+const CAPTION_REFINE_SYSTEM_PROMPT = `실시간 음성 인식으로 받아 적은 자막 한 줄을 다듬습니다.
+
+규칙:
+- 띄어쓰기, 문장부호, 앞 문맥으로 볼 때 명백하게 잘못 들은 단어만 고치세요.
+- 뜻을 바꾸거나, 요약하거나, 말을 더하거나 빼지 마세요.
+- 원문과 같은 언어, 같은 말투(존댓말/반말, 구어체)를 유지하세요.
+- 고친 문장 한 줄만 출력하세요. 설명, 따옴표, 번호를 붙이지 마세요.
+- 고칠 것이 없으면 원문을 그대로 출력하세요.`
+
+export function buildCaptionRefinePrompt(input: {
+  text: string
+  previous: readonly string[]
+}): { systemPrompt: string; text: string } {
+  const sections: string[] = []
+  const previous = input.previous.filter((line) => line.trim().length > 0).slice(-2)
+  if (previous.length > 0) sections.push('[앞 문맥]', ...previous, '')
+  sections.push('[다듬을 자막]', input.text.trim())
+  return {
+    systemPrompt: `${SUGGESTION_NO_THINK_PREFIX}\n${CAPTION_REFINE_SYSTEM_PROMPT}`,
+    text: sections.join('\n')
+  }
+}
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@ -520,7 +520,9 @@ const electronAPI = {
    getConfig: () => invoke<CaptionConfig>(IPC_CHANNELS.CAPTION.GET_CONFIG),
    onSegment: (cb: (data: CaptionSegment) => void): Unsubscribe =>
      on(IPC_CHANNELS.CAPTION.SEGMENT, cb),
-    onDelta: (cb: (data: { text: string }) => void): Unsubscribe =>
+    onSegmentUpdated: (cb: (data: { id: string; text: string }) => void): Unsubscribe =>
+      on(IPC_CHANNELS.CAPTION.SEGMENT_UPDATED, cb),
+    onDelta: (cb: (data: { text: string; stable: string; isFinal: boolean }) => void): Unsubscribe =>
      on(IPC_CHANNELS.CAPTION.DELTA, cb),
    onStateChanged: (cb: (data: { state: CaptionState }) => void): Unsubscribe =>
      on(IPC_CHANNELS.CAPTION.STATE_CHANGED, cb),
--- a/apps/desktop/src/renderer/components/SettingsModal.tsx
+++ b/apps/desktop/src/renderer/components/SettingsModal.tsx
@ -481,6 +481,27 @@ export function SettingsModal({ open, initialTab = 0, onClose }: SettingsModalPr
                </Select>
              </FormControl>

+              <FormControlLabel
+                sx={{ mr: 0, alignItems: 'flex-start' }}
+                control={
+                  <Switch
+                    size="small"
+                    checked={config.captionRefineEnabled !== false}
+                    onChange={(e) => updateConfig('captionRefineEnabled', e.target.checked)}
+                  />
+                }
+                label={
+                  <Box sx={{ pt: '2px' }}>
+                    <Typography sx={{ fontSize: d3roTypo.body.size, color: d3roPalette.text.primary }}>
+                      {t('settings.captionRefine')}
+                    </Typography>
+                    <Typography sx={{ fontSize: d3roTypo.meta.size, color: d3roPalette.text.secondary }}>
+                      {t('settings.captionRefine.desc')}
+                    </Typography>
+                  </Box>
+                }
+              />
+
              <Divider sx={{ borderColor: d3roPalette.border.subtle }} />

              <Typography variant="overline" sx={{ color: d3roPalette.text.label, letterSpacing: '1.5px' }}>
--- a/apps/desktop/src/renderer/popups/caption-overlay/script.js
+++ b/apps/desktop/src/renderer/popups/caption-overlay/script.js
@ -75,6 +75,9 @@
      return
    }

+    // 첫 중간 결과가 오면 준비 안내를 걷는다 — 이제 실제로 듣고 있다.
+    removeStatusLine()
+
    if (!deltaLine) {
      deltaLine = document.createElement('div')
      deltaLine.className = 'caption-line delta'
@ -82,7 +85,35 @@
      linesContainer.appendChild(deltaLine)
    }

-    deltaLine.textContent = data.text
+    // 두 번 연속 같게 들린 앞부분은 또렷하게, 아직 바뀔 수 있는 뒷부분은 흐리게.
+    var text = data.text || ''
+    var stable = data.stable || ''
+    if (stable && text.indexOf(stable) !== 0) stable = ''
+    deltaLine.textContent = ''
+    if (stable) deltaLine.appendChild(document.createTextNode(stable))
+    var rest = text.slice(stable.length)
+    if (rest) {
+      var tentative = document.createElement('span')
+      tentative.className = 'tentative'
+      tentative.textContent = rest
+      deltaLine.appendChild(tentative)
+    }
+  }
+
+  /**
+   * 문맥 다듬기로 고쳐진 줄을 제자리에서 바꾼다.
+   * @param {{id: string, text: string}} update
+   */
+  function updateSegment(update) {
+    for (var i = 0; i < lines.length; i++) {
+      if (lines[i].id !== update.id) continue
+      lines[i].el.textContent = update.text
+      lines[i].el.classList.remove('refined')
+      // 다시 그려야 애니메이션이 재시작된다
+      void lines[i].el.offsetWidth
+      lines[i].el.classList.add('refined')
+      return
+    }
  }

  /**
@ -163,6 +194,11 @@
      updateDelta(data)
    })

+    // 문맥 다듬기 결과
+    window.popupAPI.on('caption:segmentUpdated', function (update) {
+      updateSegment(update)
+    })
+
    // 설정 업데이트
    window.popupAPI.on('caption:config', function (newConfig) {
      applyConfig(newConfig)
--- a/apps/desktop/src/renderer/popups/caption-overlay/style.css
+++ b/apps/desktop/src/renderer/popups/caption-overlay/style.css
@ -144,6 +144,21 @@ html, body {
  50% { opacity: 0; }
 }

+/* 아직 바뀔 수 있는 뒷부분 (중간 결과) */
+.caption-line .tentative {
+  opacity: 0.6;
+}
+
+/* 문맥 다듬기로 고쳐진 줄 — 잠깐 강조 */
+.caption-line.refined {
+  animation: refined-flash 0.9s ease-out;
+}
+
+@keyframes refined-flash {
+  0% { border-color: var(--d3-accent-main); box-shadow: 0 0 0 2px var(--d3-accent-glow-dim); }
+  100% { box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5); }
+}
+
 /* 로딩 중 표시 — 점멸 애니메이션 */
 .caption-line.loading {
  opacity: 0.6;
--- a/apps/desktop/tests/main/services/caption-streaming.test.ts
+++ b/apps/desktop/tests/main/services/caption-streaming.test.ts
@ -0,0 +1,184 @@
+// tests/main/services/caption-streaming.test.ts
+// 실시간 자막 스트리밍: 판단 로직(core) + 트랙(오디오 버퍼·인식 호출) 흐름.
+
+import { describe, it, expect } from 'vitest'
+import {
+  CAPTION_STREAMING_DEFAULTS,
+  acceptCaptionRefinement,
+  agreedCaptionPrefix,
+  captionChangeRatio,
+  decideCaptionTick,
+  planForcedCommit
+} from '@d3ro/core/caption-streaming'
+import {
+  StreamingCaptionTrack,
+  type CaptionTranscription
+} from '../../../src/main/services/caption/StreamingCaptionTrack'
+
+const BYTES_PER_MS = 32
+
+function chunk(ms: number, voiced: boolean): Buffer {
+  const buffer = Buffer.alloc(ms * BYTES_PER_MS)
+  buffer[0] = voiced ? 1 : 0
+  return buffer
+}
+
+describe('decideCaptionTick', () => {
+  const base = { bufferMs: 2000, hasVoice: true, sinceVoiceMs: 100, sincePartialMs: 1500, busy: false }
+
+  it('말하는 중이면 1초마다 중간 결과를 만든다', () => {
+    expect(decideCaptionTick(base)).toBe('partial')
+    expect(decideCaptionTick({ ...base, sincePartialMs: 400 })).toBe('wait')
+  })
+
+  it('말이 멈추면 문장을 확정한다', () => {
+    expect(decideCaptionTick({ ...base, sinceVoiceMs: CAPTION_STREAMING_DEFAULTS.silenceCommitMs })).toBe('finalize')
+  })
+
+  it('멈추지 않고 길어지면 앞쪽을 강제로 확정한다', () => {
+    expect(decideCaptionTick({ ...base, bufferMs: CAPTION_STREAMING_DEFAULTS.maxBufferMs })).toBe('force-commit')
+  })
+
+  it('소리가 없는 오디오는 들고 있지 않는다', () => {
+    expect(decideCaptionTick({ ...base, hasVoice: false, bufferMs: 5000 })).toBe('trim-idle')
+    expect(decideCaptionTick({ ...base, hasVoice: false, bufferMs: 500 })).toBe('wait')
+  })
+
+  it('인식 중이거나 오디오가 짧으면 기다린다', () => {
+    expect(decideCaptionTick({ ...base, busy: true })).toBe('wait')
+    expect(decideCaptionTick({ ...base, bufferMs: 300 })).toBe('wait')
+  })
+})
+
+describe('LocalAgreement / 강제 확정 / 다듬기 검사', () => {
+  it('두 가설이 앞에서부터 같은 단어만 합의한다', () => {
+    expect(agreedCaptionPrefix(['오늘', '회의는', '세시에'], ['오늘', '회의는', '네시에', '해요'])).toEqual(['오늘', '회의는'])
+    expect(agreedCaptionPrefix([], ['오늘'])).toEqual([])
+  })
+
+  it('끝부분(keepTail) 안쪽 구간은 남기고 앞쪽만 확정한다', () => {
+    const plan = planForcedCommit(
+      [
+        { text: '첫 문장', start: 0, end: 4 },
+        { text: '둘째 문장', start: 4, end: 9 },
+        { text: '진행 중', start: 9, end: 12 }
+      ],
+      12,
+      1.5
+    )
+    expect(plan).toEqual({ text: '첫 문장 둘째 문장', cutAtSec: 9 })
+    expect(planForcedCommit([{ text: '긴 한 문장', start: 0, end: 12 }], 12, 1.5)).toBeNull()
+  })
+
+  it('띄어쓰기·문장부호 수준의 수정은 받고, 뜻을 바꾼 문장은 버린다', () => {
+    expect(acceptCaptionRefinement('오늘회의는 세시에 시작합니다', '오늘 회의는 세 시에 시작합니다.')).toBe(
+      '오늘 회의는 세 시에 시작합니다.'
+    )
+    expect(acceptCaptionRefinement('오늘 회의는 세 시에 시작합니다', '회의 일정이 변경되었다는 공지입니다')).toBeNull()
+    expect(acceptCaptionRefinement('그대로', '"그대로"')).toBeNull()
+    expect(captionChangeRatio('abc', 'abc')).toBe(0)
+  })
+})
+
+describe('StreamingCaptionTrack', () => {
+  function setup(results: CaptionTranscription[]) {
+    let now = 1_000
+    const partials: Array<{ text: string; stable: string }> = []
+    const finals: string[] = []
+    const calls: Array<{ bytes: number; partial: boolean; prompt: string }> = []
+    const track = new StreamingCaptionTrack(
+      {
+        transcribe: async (audio, opts) => {
+          calls.push({ bytes: audio.length, partial: opts.partial, prompt: opts.initialPrompt })
+          return results.shift() ?? { text: '', segments: [] }
+        },
+        isVoiced: (c) => c[0] === 1,
+        onPartial: (text, stable) => partials.push({ text, stable }),
+        onFinal: (text) => finals.push(text),
+        onError: (error) => {
+          throw error
+        },
+        now: () => now
+      },
+      '앞 문맥'
+    )
+    return {
+      track,
+      partials,
+      finals,
+      calls,
+      advance: (ms: number) => {
+        now += ms
+      }
+    }
+  }
+
+  it('말하는 동안 단어가 갱신되고, 멈추면 한 문장으로 확정된다', async () => {
+    const t = setup([
+      { text: '오늘 회의는', segments: [] },
+      { text: '오늘 회의는 세시에', segments: [] },
+      { text: '오늘 회의는 세 시에 시작합니다', segments: [] }
+    ])
+
+    t.track.push(chunk(1000, true))
+    await t.track.tick()
+    t.advance(1000)
+    t.track.push(chunk(1000, true))
+    await t.track.tick()
+
+    expect(t.partials.map((p) => p.text)).toEqual(['오늘 회의는', '오늘 회의는 세시에'])
+    expect(t.partials[1].stable).toBe('오늘 회의는')
+    expect(t.calls.every((c) => c.partial)).toBe(true)
+
+    t.advance(800)
+    t.track.push(chunk(800, false))
+    await t.track.tick()
+
+    expect(t.finals).toEqual(['오늘 회의는 세 시에 시작합니다'])
+    expect(t.calls[2].partial).toBe(false)
+    expect(t.calls[2].prompt).toBe('앞 문맥')
+    expect(t.track.bufferMs).toBe(0)
+  })
+
+  it('확정된 문장은 다음 인식의 앞 문맥이 된다', async () => {
+    const t = setup([{ text: '첫 문장', segments: [] }, { text: '둘째', segments: [] }])
+    t.track.push(chunk(1000, true))
+    t.advance(800)
+    await t.track.tick()
+    t.track.push(chunk(1000, true))
+    await t.track.tick()
+    expect(t.calls[1].prompt).toBe('앞 문맥 첫 문장')
+  })
+
+  it('끊기지 않는 긴 소리는 앞쪽만 확정하고 그만큼 오디오를 잘라낸다', async () => {
+    const t = setup([
+      {
+        text: '첫 문장 둘째 문장 진행 중',
+        segments: [
+          { text: '첫 문장', start: 0, end: 5 },
+          { text: '둘째 문장', start: 5, end: 9 },
+          { text: '진행 중', start: 9, end: 12 }
+        ]
+      }
+    ])
+    t.track.push(chunk(CAPTION_STREAMING_DEFAULTS.maxBufferMs, true))
+    await t.track.tick()
+    expect(t.finals).toEqual(['첫 문장 둘째 문장'])
+    expect(t.track.bufferMs).toBe(CAPTION_STREAMING_DEFAULTS.maxBufferMs - 9000)
+  })
+
+  it('소리가 없으면 오디오를 짧게만 남기고 인식하지 않는다', async () => {
+    const t = setup([])
+    t.track.push(chunk(5000, false))
+    await t.track.tick()
+    expect(t.calls).toHaveLength(0)
+    expect(t.track.bufferMs).toBe(CAPTION_STREAMING_DEFAULTS.idleKeepMs)
+  })
+
+  it('종료할 때 말하던 문장을 확정한다', async () => {
+    const t = setup([{ text: '마지막 말', segments: [] }])
+    t.track.push(chunk(1200, true))
+    await t.track.flush()
+    expect(t.finals).toEqual(['마지막 말'])
+  })
+})
--- a/apps/desktop/tests/main/services/llm-prompts.test.ts
+++ b/apps/desktop/tests/main/services/llm-prompts.test.ts
@ -26,6 +26,7 @@ import {
  SUGGESTION_NO_THINK_PREFIX,
  SUGGESTION_SYSTEM_PROMPT,
 } from '../../../src/main/services/llm-prompts'
+import { buildCaptionRefinePrompt } from '../../../src/main/services/llm-prompts'

 beforeEach(() => {
  vi.clearAllMocks()
@ -248,3 +249,21 @@ describe('buildSuggestionPrompt', () => {
    expect(text).not.toContain('이미 제안한 문장')
  })
 })
+
+describe('buildCaptionRefinePrompt', () => {
+  it('지시문은 시스템 프롬프트에만 두고, 본문에는 앞 문맥 2줄과 다듬을 자막만 넣는다', () => {
+    const { systemPrompt, text } = buildCaptionRefinePrompt({
+      text: '오늘회의는 세시에',
+      previous: ['첫 줄', '둘째 줄', '셋째 줄']
+    })
+    expect(systemPrompt).toContain('뜻을 바꾸거나')
+    expect(text).not.toContain('규칙')
+    expect(text).toContain(['둘째 줄', '셋째 줄'].join('\n'))
+    expect(text).not.toContain('첫 줄')
+    expect(text.endsWith('오늘회의는 세시에')).toBe(true)
+  })
+
+  it('앞 문맥이 없으면 다듬을 자막만 넣는다', () => {
+    expect(buildCaptionRefinePrompt({ text: '안녕', previous: [] }).text).toBe(['[다듬을 자막]', '안녕'].join('\n'))
+  })
+})
--- a/docs/map/10-feature-catalog.md
+++ b/docs/map/10-feature-catalog.md
@ -26,7 +26,7 @@ Status quick-reference: `[x]` done+verified · `[~]` partial/unverified · `[ ]`
 | CAP-10 | STT model download/management UI | [x] | [-] | [~] | [-] | Desktop model manager + onboarding; mobile bundled model |
 | CAP-11 | File transcription (audio/video) | [x] | [ ] | [x] | [~] | Desktop ffmpeg chunking; mobile import picker; web deferred |
 | CAP-12 | Audio import from other apps (share intent) | [-] | [-] | [x] | [-] | Mobile Android `ACTION_SEND`/`ACTION_VIEW` (SSOT R-016 GREEN) |
-| CAP-13 | Live captions overlay | [x] | [-] | [-] | [-] | Desktop `CaptionService` + caption-overlay popup; popup assets verified in the packaged build (`scripts/ci/verify-desktop-renderer-bundles.mjs`); GAP-INFRA-05 |
+| CAP-13 | Live captions overlay | [x] | [-] | [-] | [-] | Desktop `CaptionService` + caption-overlay popup; popup assets verified in the packaged build (`scripts/ci/verify-desktop-renderer-bundles.mjs`); GAP-INFRA-05 **2026-09-24 (unreleased, 1.7.0 candidate):** fixed 6 s batches replaced by streaming (`StreamingCaptionTrack` + core `caption-streaming.ts`): the uncommitted buffer is re-recognised every 1 s (greedy `partial`) and sent as `caption:delta` {text, stable} where `stable` is the LocalAgreement-2 prefix; 0.7 s of silence finalises the line with a full pass; unbroken audio over 12 s commits segments ending before the last 1.5 s using Whisper segment times; idle audio is trimmed to 1.5 s. Captions always use the local engine; `auto` language is pinned after the first final. Finished lines are refined by the local LLM (`buildCaptionRefinePrompt`, `acceptCaptionRefinement` rejects >35 % change) and replaced via `caption:segmentUpdated`; toggle `captionRefineEnabled` (Settings). Overlay: draggable handle with remembered position, waiting notice until the first caption. |
 | CAP-14 | Recording persistence / crash recovery | [x] | [ ] | [x] | [-] | Desktop WAV persist; mobile durable queue + process-kill WAV recovery |
 | CAP-15 | Android foreground recording service | [-] | [-] | [x] | [-] | Mobile API 34 FGS + persistent notification (SSOT R-005 GREEN) |
 | CAP-16 | Rebindable global key bindings (keyboard + mouse) | [x] | [-] | [-] | [-] | Contract SSOT `packages/core/src/keybinding.ts`: `KEY_CATALOG` (10 groups, `:615`), `KEYBINDING_ACTIONS` (6 actions, `:719`), `validateBinding` (`:953`), `detectBindingConflicts` (`:1016`). Multiple bindings per action persist as one `AppConfig.keyBindings` map (`packages/core/src/types.ts:459`), replacing the four singular `*Shortcut` fields; `ConfigService` migrates legacy values once (`ConfigService.ts:142`). `KeyBindingService` hooks keyboard **and** mouse via uiohook (`KeyBindingService.ts:387`) — MB1 is not bindable, MB2/MB3 need a modifier, MB4/MB5 are free, and no mouse button can be suppressed, so the original click still fires (warning surfaced in the UI). Selection is either key-recording or a searchable grouped dropdown (`KeyBindingPicker.tsx:536`). `history-popup`/`command-popup` were hardcoded in `bootstrap.ts` and are now rebindable actions (`bootstrap.ts:159`). **Verified 2026-09-21 on Windows by a manual run** (`%APPDATA%/d3ro-voice/logs/main.log`, 12:53–13:06): `ConfigService` migrated the four legacy shortcuts with the user's non-default values preserved exactly, `KeyBindingService` loaded 6 bindings for 6 actions and started the uiohook keyboard **and** mouse hook with zero boot errors, and keyboard plus mouse (MB4/MB5) bindings were exercised through the UI. A `Loaded 7 key binding(s) … for 6 action(s)` line later in the same session shows multi-binding working end to end. The migrated map was read back from `d3ro-voice-config.json`: legacy `*Shortcut` fields gone, no `displayLabel` left. Contract evidence: `packages/core` 117 tests GREEN, no renderer type errors in the key-binding files. **Still open:** `KeyBindingService` has no unit test of its own, macOS/Linux mouse behavior is unconfirmed (`11` GAP-KEY-02), and `command` still falls back to the dictation pipeline (GAP-KEY-03). W/M `[-]`: no OS-level global binding surface exists there (browser sandbox; mobile has no global hotkey, see CAP-01). B `[-]`: device-local setting, nothing server-side. See `11` GAP-KEY-02/03 (open), GAP-KEY-01 (`[x]`), and `11` §7 CONSTRAINT-I18N-01. |
--- a/packages/core/package.json
+++ b/packages/core/package.json
@ -38,6 +38,10 @@
      "types": "./src/input-intelligence.ts",
      "default": "./src/input-intelligence.ts"
    },
+    "./caption-streaming": {
+      "types": "./src/caption-streaming.ts",
+      "default": "./src/caption-streaming.ts"
+    },
    "./entitlement": {
      "types": "./src/entitlement.ts",
      "default": "./src/entitlement.ts"
--- a/packages/core/src/caption-streaming.ts
+++ b/packages/core/src/caption-streaming.ts
@ -0,0 +1,152 @@
+// packages/core/src/caption-streaming.ts
+//
+// 실시간 자막 스트리밍의 순수 판단 로직 — 오디오/모델/IPC 없이 테스트한다.
+//
+// 방식 (whisper_streaming 계열, Macháček et al. 2023 의 LocalAgreement 를 단순화):
+//   - 확정되지 않은 오디오 구간 전체를 짧은 주기로 다시 인식해 "지금까지 들은 문장" 을
+//     중간 결과로 내보낸다 → 단어가 실시간으로 갱신된다.
+//   - 말이 멈추면(무음) 그 구간을 정밀하게 한 번 더 인식해 문장을 확정한다.
+//   - 멈추지 않고 길어지면 모델이 준 구간 시간으로 앞쪽만 확정하고 오디오를 잘라낸다.
+//   - 소리가 없는 오디오는 오래 들고 있지 않는다 — Whisper 는 무음에서 문장을 지어낸다.
+
+export const CAPTION_STREAMING_DEFAULTS = {
+  /** 중간 결과 재인식 주기 */
+  partialIntervalMs: 1000,
+  /** 인식을 시작할 최소 오디오 길이 */
+  minAudioMs: 800,
+  /** 말이 이만큼 멈추면 문장을 확정한다 */
+  silenceCommitMs: 700,
+  /** 멈추지 않는 말이 이 길이를 넘으면 앞쪽을 강제로 확정한다 */
+  maxBufferMs: 12000,
+  /** 강제 확정 때 뒤쪽은 이만큼 남긴다 (끝 구간은 아직 바뀔 수 있다) */
+  keepTailMs: 1500,
+  /** 소리가 없을 때 들고 있을 오디오 */
+  idleKeepMs: 1500,
+  /** 다음 인식에 넘기는 앞 문맥 길이 (자) */
+  contextChars: 200,
+  /** 문맥 다듬기 결과가 원문에서 이 비율 이상 바뀌면 버린다 */
+  refineMaxChangeRatio: 0.35
+} as const
+
+export type CaptionTickAction = 'finalize' | 'force-commit' | 'partial' | 'trim-idle' | 'wait'
+
+export interface CaptionTickInput {
+  /** 확정되지 않은 오디오 길이 */
+  bufferMs: number
+  /** 이 구간에 소리가 있었는가 */
+  hasVoice: boolean
+  /** 마지막 소리 이후 지난 시간 */
+  sinceVoiceMs: number
+  /** 마지막 중간 인식 이후 지난 시간 */
+  sincePartialMs: number
+  /** 인식이 진행 중인가 (한 트랙에 한 건만) */
+  busy: boolean
+}
+
+/** 매 틱마다 무엇을 할지 정한다. */
+export function decideCaptionTick(
+  input: CaptionTickInput,
+  opts: Partial<typeof CAPTION_STREAMING_DEFAULTS> = {}
+): CaptionTickAction {
+  const o = { ...CAPTION_STREAMING_DEFAULTS, ...opts }
+  if (input.busy) return 'wait'
+  if (!input.hasVoice) return input.bufferMs > o.idleKeepMs ? 'trim-idle' : 'wait'
+  if (input.bufferMs < o.minAudioMs) return 'wait'
+  if (input.sinceVoiceMs >= o.silenceCommitMs) return 'finalize'
+  if (input.bufferMs >= o.maxBufferMs) return 'force-commit'
+  if (input.sincePartialMs >= o.partialIntervalMs) return 'partial'
+  return 'wait'
+}
+
+/** 공백 기준 단어 (한국어도 어절 단위로 띄어 쓴다). */
+export function splitCaptionWords(text: string): string[] {
+  return text.trim().split(/\s+/u).filter((word) => word.length > 0)
+}
+
+/**
+ * 연속된 두 가설이 앞에서부터 합의한 단어 (LocalAgreement-2).
+ *
+ * 두 번 연속 같게 나온 앞부분은 이후 오디오가 더 와도 거의 바뀌지 않는다 —
+ * 중간 결과에서 이 부분은 흔들리지 않게 보여 줄 수 있다.
+ */
+export function agreedCaptionPrefix(previous: readonly string[], current: readonly string[]): string[] {
+  const agreed: string[] = []
+  const length = Math.min(previous.length, current.length)
+  for (let i = 0; i < length; i += 1) {
+    if (previous[i] !== current[i]) break
+    agreed.push(current[i])
+  }
+  return agreed
+}
+
+export interface CaptionTimedSegment {
+  text: string
+  /** 초 */
+  start: number
+  /** 초 */
+  end: number
+}
+
+export interface ForcedCommitPlan {
+  /** 확정할 텍스트 */
+  text: string
+  /** 이 시점(초)까지의 오디오를 잘라낸다 */
+  cutAtSec: number
+}
+
+/**
+ * 길어진 구간에서 앞쪽을 확정할 계획.
+ *
+ * 끝에서 keepTail 안쪽에 걸친 구간은 아직 문장이 이어질 수 있으니 남긴다. 확정할
+ * 구간이 없으면 null — 호출자는 다음 틱에 다시 본다.
+ */
+export function planForcedCommit(
+  segments: readonly CaptionTimedSegment[],
+  bufferSec: number,
+  keepTailSec: number
+): ForcedCommitPlan | null {
+  const limit = bufferSec - keepTailSec
+  const committed = segments.filter((segment) => segment.end <= limit && segment.text.trim().length > 0)
+  if (committed.length === 0) return null
+  return {
+    text: committed.map((segment) => segment.text.trim()).join(' '),
+    cutAtSec: committed[committed.length - 1].end
+  }
+}
+
+/** 문자 단위 편집 거리 비율 (0 = 같음, 1 = 전혀 다름). 공백은 무시한다. */
+export function captionChangeRatio(original: string, revised: string): number {
+  const a = [...original.replace(/\s+/gu, '')]
+  const b = [...revised.replace(/\s+/gu, '')]
+  if (a.length === 0 && b.length === 0) return 0
+  let prev = Array.from({ length: b.length + 1 }, (_, j) => j)
+  for (let i = 1; i <= a.length; i += 1) {
+    const row = [i]
+    for (let j = 1; j <= b.length; j += 1) {
+      row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1))
+    }
+    prev = row
+  }
+  return prev[b.length] / Math.max(a.length, b.length)
+}
+
+/**
+ * 모델이 다듬은 문장을 쓸지 정한다.
+ *
+ * 자막 다듬기는 띄어쓰기·문장부호·명백한 오인식만 고쳐야 한다. 많이 바뀌었다면
+ * 모델이 요약하거나 지어낸 것이다 — 원문을 지킨다(null).
+ */
+export function acceptCaptionRefinement(
+  original: string,
+  refined: string,
+  maxChangeRatio: number = CAPTION_STREAMING_DEFAULTS.refineMaxChangeRatio
+): string | null {
+  const line = refined
+    .split(/\r?\n/u)
+    .map((part) => part.trim())
+    .find((part) => part.length > 0)
+  if (!line) return null
+  const cleaned = line.replace(/^["'“”‘’「」『』]+|["'“”‘’「」『』]+$/gu, '').trim()
+  if (!cleaned || cleaned === original.trim()) return null
+  return captionChangeRatio(original, cleaned) <= maxChangeRatio ? cleaned : null
+}
--- a/packages/core/src/ipc-channels.ts
+++ b/packages/core/src/ipc-channels.ts
@ -249,6 +249,8 @@ export const IPC_CHANNELS = {
    SYSTEM_AUDIO_DATA: 'caption:systemAudioData',
    // Main → Renderer events
    SEGMENT: 'caption:segment',
+    /** 확정된 줄을 문맥 다듬기로 고친 결과 { id, text } */
+    SEGMENT_UPDATED: 'caption:segmentUpdated',
    DELTA: 'caption:delta',
    STATE_CHANGED: 'caption:stateChanged',
    SESSION_SAVED: 'caption:sessionSaved',
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@ -497,6 +497,8 @@ export interface AppConfig {
  captionAudioSource: import('@d3ro/core/types').CaptionAudioSource
  /** 사용자가 끌어다 놓은 자막 창 위치 (스크린 좌표). null 이면 화면 아래 가운데 */
  captionOverlayPosition: { x: number; y: number } | null
+  /** 확정된 자막 줄을 로컬 LLM으로 문맥에 맞게 다듬는다 */
+  captionRefineEnabled: boolean
  /** Auto-update 채널 (latest=stable / beta / alpha). UpdateService */
  updateChannel: 'latest' | 'beta' | 'alpha'
  /** staged rollout용 영구 기기 식별자 (개인정보 아님, 최초 실행 시 생성) */
--- a/packages/i18n/src/locales/de.json
+++ b/packages/i18n/src/locales/de.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "{{app}} konnte nicht zu Ausschlüssen hinzugefügt werden.",
  "popup.caption.loading": "Sprachmodell wird vorbereitet…",
  "popup.caption.waiting": "Hört zu… der erste Untertitel kann einige Sekunden dauern",
-  "popup.caption.dragHint": "Ziehen zum Verschieben · Doppelklick setzt zurück"
+  "popup.caption.dragHint": "Ziehen zum Verschieben · Doppelklick setzt zurück",
+  "settings.captionRefine": "Untertitel im Kontext glätten",
+  "settings.captionRefine.desc": "Die lokale KI korrigiert Leerzeichen, Satzzeichen und falsch verstandene Wörter anhand der umgebenden Zeilen."
 }
--- a/packages/i18n/src/locales/en.json
+++ b/packages/i18n/src/locales/en.json
@ -1903,5 +1903,7 @@
  "input.feedback.excludedFailed": "Exclusions could not be saved.",
  "input.feedback.recommendationFailed": "Could not add {{app}} to exclusions.",
  "popup.caption.waiting": "Listening… the first caption can take a few seconds",
-  "popup.caption.dragHint": "Drag to move · double-click to reset"
+  "popup.caption.dragHint": "Drag to move · double-click to reset",
+  "settings.captionRefine": "Polish captions with context",
+  "settings.captionRefine.desc": "The local AI fixes spacing, punctuation and misheard words in finished captions using the surrounding lines."
 }
--- a/packages/i18n/src/locales/es.json
+++ b/packages/i18n/src/locales/es.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "No se pudo añadir {{app}} a las exclusiones.",
  "popup.caption.loading": "Preparando el modelo de voz…",
  "popup.caption.waiting": "Escuchando… el primer subtítulo puede tardar unos segundos",
-  "popup.caption.dragHint": "Arrastra para mover · doble clic para restablecer"
+  "popup.caption.dragHint": "Arrastra para mover · doble clic para restablecer",
+  "settings.captionRefine": "Pulir subtítulos con contexto",
+  "settings.captionRefine.desc": "La IA local corrige espacios, puntuación y palabras mal oídas usando las líneas cercanas."
 }
--- a/packages/i18n/src/locales/fr.json
+++ b/packages/i18n/src/locales/fr.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "Impossible d’ajouter {{app}} aux exclusions.",
  "popup.caption.loading": "Préparation du modèle vocal…",
  "popup.caption.waiting": "Écoute… le premier sous-titre peut prendre quelques secondes",
-  "popup.caption.dragHint": "Glisser pour déplacer · double-clic pour réinitialiser"
+  "popup.caption.dragHint": "Glisser pour déplacer · double-clic pour réinitialiser",
+  "settings.captionRefine": "Affiner les sous-titres selon le contexte",
+  "settings.captionRefine.desc": "L’IA locale corrige les espaces, la ponctuation et les mots mal entendus d’après les lignes voisines."
 }
--- a/packages/i18n/src/locales/ja.json
+++ b/packages/i18n/src/locales/ja.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "{{app}} を除外に追加できませんでした。",
  "popup.caption.loading": "音声モデルを準備中…",
  "popup.caption.waiting": "聞き取り中… 最初の字幕まで数秒かかることがあります",
-  "popup.caption.dragHint": "ドラッグで移動 · ダブルクリックで元の位置"
+  "popup.caption.dragHint": "ドラッグで移動 · ダブルクリックで元の位置",
+  "settings.captionRefine": "文脈で字幕を整える",
+  "settings.captionRefine.desc": "確定した字幕の区切り・句読点・聞き間違いを、ローカルAIが前後の文脈に合わせて直します。"
 }
--- a/packages/i18n/src/locales/ko.json
+++ b/packages/i18n/src/locales/ko.json
@ -1910,5 +1910,7 @@
  "input.feedback.excludedFailed": "제외 목록을 저장하지 못했습니다.",
  "input.feedback.recommendationFailed": "{{app}}을(를) 제외 목록에 추가하지 못했습니다.",
  "popup.caption.waiting": "듣는 중… 첫 자막까지 몇 초 걸릴 수 있어요",
-  "popup.caption.dragHint": "끌어서 이동 · 더블클릭하면 원위치"
+  "popup.caption.dragHint": "끌어서 이동 · 더블클릭하면 원위치",
+  "settings.captionRefine": "자막 문맥 다듬기",
+  "settings.captionRefine.desc": "확정된 자막을 로컬 AI가 앞뒤 문맥에 맞게 띄어쓰기·문장부호·잘못 들은 단어를 고칩니다."
 }
--- a/packages/i18n/src/locales/pt.json
+++ b/packages/i18n/src/locales/pt.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "Não foi possível adicionar {{app}} às exclusões.",
  "popup.caption.loading": "Preparando o modelo de voz…",
  "popup.caption.waiting": "Ouvindo… a primeira legenda pode levar alguns segundos",
-  "popup.caption.dragHint": "Arraste para mover · clique duplo para restaurar"
+  "popup.caption.dragHint": "Arraste para mover · clique duplo para restaurar",
+  "settings.captionRefine": "Refinar legendas pelo contexto",
+  "settings.captionRefine.desc": "A IA local corrige espaços, pontuação e palavras mal ouvidas usando as linhas próximas."
 }
--- a/packages/i18n/src/locales/ru.json
+++ b/packages/i18n/src/locales/ru.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "Не удалось добавить {{app}} в исключения.",
  "popup.caption.loading": "Подготовка речевой модели…",
  "popup.caption.waiting": "Слушаю… первый субтитр может появиться через несколько секунд",
-  "popup.caption.dragHint": "Перетащите, чтобы переместить · двойной щелчок — сброс"
+  "popup.caption.dragHint": "Перетащите, чтобы переместить · двойной щелчок — сброс",
+  "settings.captionRefine": "Уточнять субтитры по контексту",
+  "settings.captionRefine.desc": "Локальный ИИ исправляет пробелы, пунктуацию и ослышки в готовых субтитрах по соседним строкам."
 }
--- a/packages/i18n/src/locales/th.json
+++ b/packages/i18n/src/locales/th.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "ไม่สามารถเพิ่ม {{app}} ในรายการยกเว้นได้",
  "popup.caption.loading": "กำลังเตรียมโมเดลเสียง…",
  "popup.caption.waiting": "กำลังฟัง… คำบรรยายแรกอาจใช้เวลาสักครู่",
-  "popup.caption.dragHint": "ลากเพื่อย้าย · ดับเบิลคลิกเพื่อรีเซ็ต"
+  "popup.caption.dragHint": "ลากเพื่อย้าย · ดับเบิลคลิกเพื่อรีเซ็ต",
+  "settings.captionRefine": "ขัดเกลาคำบรรยายตามบริบท",
+  "settings.captionRefine.desc": "AI ในเครื่องจะแก้เว้นวรรค เครื่องหมายวรรคตอน และคำที่ได้ยินผิด โดยดูจากบรรทัดรอบข้าง"
 }
--- a/packages/i18n/src/locales/vi.json
+++ b/packages/i18n/src/locales/vi.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "Không thể thêm {{app}} vào danh sách loại trừ.",
  "popup.caption.loading": "Đang chuẩn bị mô hình giọng nói…",
  "popup.caption.waiting": "Đang nghe… phụ đề đầu tiên có thể mất vài giây",
-  "popup.caption.dragHint": "Kéo để di chuyển · nhấp đúp để đặt lại"
+  "popup.caption.dragHint": "Kéo để di chuyển · nhấp đúp để đặt lại",
+  "settings.captionRefine": "Chỉnh phụ đề theo ngữ cảnh",
+  "settings.captionRefine.desc": "AI cục bộ sửa khoảng trắng, dấu câu và từ nghe nhầm dựa trên các dòng xung quanh."
 }
--- a/packages/i18n/src/locales/zh-TW.json
+++ b/packages/i18n/src/locales/zh-TW.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "無法將 {{app}} 加入排除清單。",
  "popup.caption.loading": "正在準備語音模型…",
  "popup.caption.waiting": "聆聽中… 第一則字幕可能需要幾秒鐘",
-  "popup.caption.dragHint": "拖曳以移動 · 按兩下還原"
+  "popup.caption.dragHint": "拖曳以移動 · 按兩下還原",
+  "settings.captionRefine": "依上下文潤飾字幕",
+  "settings.captionRefine.desc": "本機 AI 會依上下文修正已確定字幕的空格、標點與聽錯的詞。"
 }
--- a/packages/i18n/src/locales/zh.json
+++ b/packages/i18n/src/locales/zh.json
@ -522,5 +522,7 @@
  "input.feedback.recommendationFailed": "无法将 {{app}} 添加到排除列表。",
  "popup.caption.loading": "正在准备语音模型…",
  "popup.caption.waiting": "正在聆听… 第一条字幕可能需要几秒钟",
-  "popup.caption.dragHint": "拖动以移动 · 双击复位"
+  "popup.caption.dragHint": "拖动以移动 · 双击复位",
+  "settings.captionRefine": "按上下文润色字幕",
+  "settings.captionRefine.desc": "本地 AI 会根据上下文修正已确定字幕的空格、标点和听错的词。"
 }