yunchan/d3ro-voice
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

feat(desktop): make local speech transcription work end to end

Browse source 
Local dictation had never produced a transcript on an installed build. The
engine itself was healthy; every connection to it was broken.

Installed builds shipped no speech engine at all: the packaging config had no
entry for the faster-whisper sidecar and no pipeline step built one, so the app
always fell back to a system Python without the runtime. Development was broken
too, because the sidecar and SoX paths were resolved against the Vite output
directory instead of the app root, which also meant recording failed with a SoX
ENOENT. On hosts where localhost resolves only to IPv6, every local request was
refused outright, which silently disabled both local transcription and the local
LLM.

The sidecar is now built and bundled (including the Silero VAD data it needs),
gated by a packaging check that fails when the engine or its data is missing.
Paths are discovered from the app root and fail loudly when the engine is
absent. Local engine URLs are normalized to the IPv4 loopback, decoding is tuned
so repeated hallucinations cannot compound (the same transcript now takes about
a fifth of the time), the engine is warmed up at startup, and holding the hotkey
now shows the text forming live in the recording tip.
This commit is contained in:
Yun Chan 2026-09-18 00:48:47 +09:00
parent 359b244dc9
commit 2d585bfc29
52 changed files with 1450 additions and 3861 deletions
Download patch file Download diff file Expand all files Collapse all files

106
apps/desktop/sidecar/main.py
View file

@ -39,6 +39,14 @@ from fastapi.responses import JSONResponse
# ── 로깅 설정 ──────────────────────────────────────────────
# Windows에서 파이프로 연결되면 Python이 로케일(cp949) 인코딩으로 출력해
# 메인 프로세스의 UTF-8 로그가 깨진다. 명시적으로 UTF-8로 고정한다.
for _stream in (sys.stdout, sys.stderr):
try:
_stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
except (AttributeError, ValueError):
pass
logging.basicConfig(
level=logging.INFO,
format="[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
@ -121,6 +129,11 @@ def _detect_gpu() -> None:
logger.info("GPU 감지 실패, CPU 모드로 동작: %s", exc)
def _cpu_threads() -> int:
"""CPU 추론에 사용할 스레드 수 (과도한 점유 방지 위해 8로 상한)."""
return max(1, min(8, os.cpu_count() or 4))
# ── 모델 다운로드 헬퍼 ─────────────────────────────────────
@ -250,6 +263,52 @@ def _cleanup_partial(model_id: str) -> None:
pass
def _build_transcribe_kwargs(
language: str,
vad_filter: str,
initial_prompt: str,
is_partial: bool,
) -> dict:
"""전사 옵션을 만든다.
받아쓰기 정합성을 위해 컨텍스트 누적(condition_on_previous_text)을 끈다.
Whisper가 앞 세그먼트 오류를 반복 증폭하는 현상(환각 루프)을 막는다.
미리보기(partial)는 지연이 목표이므로 greedy + VAD 없음으로 디코딩한다.
"""
if is_partial:
kwargs: dict = {
"beam_size": 1,
"temperature": 0.0,
"vad_filter": False,
"condition_on_previous_text": False,
"word_timestamps": False,
}
else:
kwargs = {
"beam_size": 5,
# 0.0 단일 온도는 실패 시 재시도가 없어 환각이 남는다.
# 낮은 온도 폴백만 허용하되 컨텍스트를 끊어 반복을 차단한다.
"temperature": [0.0, 0.2, 0.4],
"condition_on_previous_text": False,
"no_speech_threshold": 0.6,
"compression_ratio_threshold": 2.4,
"log_prob_threshold": -1.0,
"vad_filter": vad_filter.lower() == "true",
"word_timestamps": False,
}
if kwargs["vad_filter"]:
# 무음 구간을 촘촘히 잘라 속도를 올린다.
kwargs["vad_parameters"] = {"min_silence_duration_ms": 300}
if language != "auto":
kwargs["language"] = language
if initial_prompt and not is_partial:
kwargs["initial_prompt"] = initial_prompt
return kwargs
# ── 엔드포인트 ─────────────────────────────────────────────
@ -260,7 +319,9 @@ async def health() -> JSONResponse:
content={
"status": "ready" if _model is not None else "no_model",
"model": _model_id,
"model_loaded": _model is not None,
"gpu": _gpu_available,
"device": "cuda" if _gpu_available else "cpu",
}
)
@ -282,6 +343,18 @@ async def load_model(body: dict) -> JSONResponse: # noqa: ANN001
start_time = time.monotonic()
# 같은 모델이 이미 로딩되어 있으면 재사용 (재로딩은 수초 지연을 만든다)
if _model is not None and _model_id == model_id:
logger.info("이미 로딩된 모델 재사용: %s", model_id)
return JSONResponse(
content={
"status": "loaded",
"model_id": model_id,
"load_time_ms": 0,
"reused": True,
}
)
try:
from faster_whisper import WhisperModel
@ -295,10 +368,15 @@ async def load_model(body: dict) -> JSONResponse: # noqa: ANN001
if local_dir:
logger.info("로컬 모델 디렉토리 사용: %s", local_dir)
# 모델 교체 시 이전 모델을 먼저 해제해 VRAM/RAM을 회수한다.
_model = None
_model = WhisperModel(
model_source,
device=device,
compute_type=compute_type,
cpu_threads=_cpu_threads(),
num_workers=1,
)
_model_id = model_id
@ -333,14 +411,16 @@ async def transcribe(
language: str = Form("auto"),
vad_filter: str = Form("true"),
initial_prompt: str = Form(""),
partial: str = Form("false"),
) -> JSONResponse:
"""오디오 파일을 전사한다.
Multipart form:
audio - PCM16 16kHz mono 바이너리 파일
language - 언어 코드 ('auto', 'ko', 'en', ...)
vad_filter - VAD 필터 활성화 ('true' / 'false')
audio - PCM16 16kHz mono 바이너리 파일
language - 언어 코드 ('auto', 'ko', 'en', ...)
vad_filter - VAD 필터 활성화 ('true' / 'false')
initial_prompt - 초기 프롬프트 (컨텍스트 힌트)
partial - 녹음 중 미리보기 모드 ('true'면 greedy 디코딩 + 컨텍스트 미사용)
"""
if _model is None:
return JSONResponse(
@ -348,6 +428,7 @@ async def transcribe(
content={"status": "error", "message": "모델이 로딩되지 않았습니다"},
)
is_partial = partial.lower() == "true"
start_time = time.monotonic()
try:
@ -367,22 +448,19 @@ async def transcribe(
audio_duration = len(audio_array) / sample_rate
logger.info(
"전사 시작: %.1f초 오디오, language=%s, vad=%s",
"전사 시작: %.1f초 오디오, language=%s, vad=%s, partial=%s",
audio_duration,
language,
vad_filter,
is_partial,
)
transcribe_kwargs: dict = {
"vad_filter": vad_filter.lower() == "true",
"beam_size": 5,
}
if language != "auto":
transcribe_kwargs["language"] = language
if initial_prompt:
transcribe_kwargs["initial_prompt"] = initial_prompt
transcribe_kwargs = _build_transcribe_kwargs(
language=language,
vad_filter=vad_filter,
initial_prompt=initial_prompt,
is_partial=is_partial,
)
# VAD가 전체 오디오를 제거하면 max() 에러 발생 → VAD 없이 재시도
try: