feat: ship Wave 5-6 multi-surface hygiene, optimization, and bug fixes
This commit is contained in:
parent
01ef3293cf
commit
66b6b45ba9
399 changed files with 17828 additions and 11330 deletions
|
|
@ -29,9 +29,10 @@ import logging
|
|||
import os
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator
|
||||
|
||||
|
|
@ -599,28 +600,40 @@ async def load_model(body: dict) -> JSONResponse: # noqa: ANN001
|
|||
|
||||
@app.post("/unload")
|
||||
async def unload_model(request: Request) -> JSONResponse:
|
||||
"""보조 자리 모델을 내려 VRAM/RAM 을 돌려준다.
|
||||
"""보조 자리 또는 기본(primary) 자리 모델을 내려 VRAM/RAM 을 돌려준다.
|
||||
|
||||
자막·회의가 끝난 뒤에도 보조 모델(예: large-v3-turbo)이 받아쓰기 모델과 함께 상주하면
|
||||
GPU 여유가 줄어 Ollama 모델 로드가 느려지거나 OOM 이 난다. 기본(primary) 모델은 받아쓰기
|
||||
지연을 막기 위해 내리지 않는다.
|
||||
사이드카 HTTP 프로세스 자체는 UIA 브리지(/uia/focus)가 계속 쓰므로 살려 둔 채,
|
||||
클라우드 STT 전환·장시간 유휴·자막 종료 시 Whisper CTranslate2 모델 메모리만 회수한다.
|
||||
|
||||
Request body: { "slot": "aux" }
|
||||
Request body: { "slot": "aux" | "primary" | "all" }
|
||||
"""
|
||||
global _model, _model_id
|
||||
|
||||
body = await request.json()
|
||||
slot: str = body.get("slot", "aux")
|
||||
if slot != "aux":
|
||||
if slot not in ("aux", "primary", "all"):
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"status": "error", "message": "only the aux slot can be unloaded"},
|
||||
content={"status": "error", "message": "slot must be aux, primary, or all"},
|
||||
)
|
||||
|
||||
released: list[str] = []
|
||||
async with _inference_lock:
|
||||
released = list(_aux_models)
|
||||
_aux_models.clear()
|
||||
if slot in ("primary", "all") and _model is not None:
|
||||
if _model_id:
|
||||
released.append(_model_id)
|
||||
_model_devices.pop(_model_id, None)
|
||||
_model = None
|
||||
_model_id = None
|
||||
if slot in ("aux", "all") and _aux_models:
|
||||
for aux_id in list(_aux_models):
|
||||
released.append(aux_id)
|
||||
_model_devices.pop(aux_id, None)
|
||||
_aux_models.clear()
|
||||
|
||||
if released:
|
||||
gc.collect()
|
||||
logger.info("보조 모델 언로드: %s", ", ".join(released))
|
||||
logger.info("모델 언로드(%s): %s", slot, ", ".join(released))
|
||||
return JSONResponse(content={"status": "unloaded", "models": released})
|
||||
|
||||
|
||||
|
|
@ -708,14 +721,21 @@ async def transcribe(
|
|||
start_time = time.monotonic()
|
||||
|
||||
try:
|
||||
pcm_bytes = await audio.read()
|
||||
try:
|
||||
pcm_bytes = await audio.read()
|
||||
finally:
|
||||
await audio.close()
|
||||
|
||||
if len(pcm_bytes) == 0:
|
||||
if len(pcm_bytes) < 2:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"status": "error", "message": "오디오 데이터가 비어있습니다"},
|
||||
)
|
||||
|
||||
if len(pcm_bytes) % 2 != 0:
|
||||
logger.warning("홀수 바이트 PCM 입력(%dB) — 마지막 1바이트를 버립니다", len(pcm_bytes))
|
||||
pcm_bytes = pcm_bytes[:-1]
|
||||
|
||||
audio_array = (
|
||||
np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
)
|
||||
|
|
@ -756,6 +776,8 @@ async def transcribe(
|
|||
finally:
|
||||
cancel.set()
|
||||
watcher.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await watcher
|
||||
|
||||
processing_time = int((time.monotonic() - start_time) * 1000)
|
||||
|
||||
|
|
@ -872,11 +894,134 @@ async def uia_focus(timeoutMs: int = 1500) -> JSONResponse:
|
|||
return JSONResponse(content=snapshot)
|
||||
|
||||
|
||||
def _run_pyannote_diarization(
|
||||
audio_bytes: bytes,
|
||||
hf_token: str,
|
||||
num_speakers: int | None,
|
||||
) -> dict[str, object]:
|
||||
"""선택 설치된 pyannote.audio 파이프라인으로 화자 구간을 추출한다."""
|
||||
try:
|
||||
from pyannote.audio import Pipeline # type: ignore[import-not-found]
|
||||
except Exception as exc:
|
||||
raise ModuleNotFoundError(str(exc)) from exc
|
||||
|
||||
fd, temp_path_str = tempfile.mkstemp(suffix=".wav", prefix="d3ro-diarize-")
|
||||
temp_path = Path(temp_path_str)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(audio_bytes)
|
||||
|
||||
try:
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
"pyannote/speaker-diarization-3.1",
|
||||
token=hf_token,
|
||||
)
|
||||
except TypeError:
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
"pyannote/speaker-diarization-3.1",
|
||||
use_auth_token=hf_token,
|
||||
)
|
||||
|
||||
call_kwargs: dict[str, object] = {}
|
||||
if num_speakers is not None and num_speakers > 0:
|
||||
call_kwargs["num_speakers"] = num_speakers
|
||||
|
||||
annotation = pipeline(str(temp_path), **call_kwargs)
|
||||
segments: list[dict[str, object]] = []
|
||||
speakers: set[str] = set()
|
||||
for turn, _track, speaker in annotation.itertracks(yield_label=True):
|
||||
speaker_label = str(speaker)
|
||||
speakers.add(speaker_label)
|
||||
segments.append(
|
||||
{
|
||||
"speaker": speaker_label,
|
||||
"start": round(float(turn.start), 3),
|
||||
"end": round(float(turn.end), 3),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"segments": segments,
|
||||
"num_speakers": len(speakers),
|
||||
}
|
||||
finally:
|
||||
try:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@app.post("/diarize")
|
||||
async def diarize(
|
||||
audio: UploadFile = File(...),
|
||||
hf_token: str = Form(""),
|
||||
num_speakers: int | None = Form(None),
|
||||
) -> JSONResponse:
|
||||
"""pyannote.audio 기반 화자 분리 엔드포인트 (미설치 시 503 반환으로 LLM 폴백 유도)."""
|
||||
token = hf_token.strip()
|
||||
if not token:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"status": "error",
|
||||
"code": "missing_hf_token",
|
||||
"message": "Hugging Face 토큰이 비어 있습니다",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
audio_bytes = await audio.read()
|
||||
finally:
|
||||
await audio.close()
|
||||
if not audio_bytes:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"status": "error",
|
||||
"code": "empty_audio",
|
||||
"message": "오디오 파일이 비어 있습니다",
|
||||
},
|
||||
)
|
||||
|
||||
async with _inference_lock:
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
_run_pyannote_diarization,
|
||||
audio_bytes,
|
||||
token,
|
||||
num_speakers,
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
except ModuleNotFoundError:
|
||||
return JSONResponse(
|
||||
status_code=503,
|
||||
content={
|
||||
"status": "unavailable",
|
||||
"code": "pyannote_not_installed",
|
||||
"message": "pyannote.audio 패키지가 설치되어 있지 않습니다",
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("pyannote 화자 분리 실패: %s", exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"status": "error",
|
||||
"code": "diarization_failed",
|
||||
"message": f"화자 분리 실행 실패: {exc}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/shutdown")
|
||||
async def shutdown() -> JSONResponse:
|
||||
"""서버를 graceful하게 종료한다."""
|
||||
logger.info("종료 요청 수신")
|
||||
|
||||
if _download_thread is not None and _download_thread.is_alive():
|
||||
_download_cancel.set()
|
||||
await asyncio.to_thread(_download_thread.join, 2.0)
|
||||
|
||||
try:
|
||||
import uia_bridge
|
||||
|
||||
|
|
|
|||
115
apps/desktop/sidecar/tests/test_diarize_endpoint.py
Normal file
115
apps/desktop/sidecar/tests/test_diarize_endpoint.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
"""
|
||||
사이드카 /diarize 엔드포인트 계약 테스트 (GAP-MTG-02).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
SIDECAR_DIR = Path(__file__).resolve().parent.parent
|
||||
if str(SIDECAR_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SIDECAR_DIR))
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
import main # noqa: E402
|
||||
|
||||
|
||||
class _FakeTurn:
|
||||
def __init__(self, start: float, end: float) -> None:
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
|
||||
class _FakeAnnotation:
|
||||
def itertracks(self, yield_label: bool = False):
|
||||
tracks = [
|
||||
(_FakeTurn(0.2, 4.5), "A", "SPEAKER_00"),
|
||||
(_FakeTurn(5.0, 9.8), "B", "SPEAKER_01"),
|
||||
(_FakeTurn(10.1, 13.0), "C", "SPEAKER_00"),
|
||||
]
|
||||
if yield_label:
|
||||
return tracks
|
||||
return [(t[0], t[1]) for t in tracks]
|
||||
|
||||
|
||||
class DiarizeEndpointTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.client = TestClient(main.app)
|
||||
|
||||
def test_diarize_rejects_empty_hf_token(self) -> None:
|
||||
response = self.client.post(
|
||||
"/diarize",
|
||||
files={"audio": ("audio.wav", b"RIFF0000WAVEfmt ", "audio/wav")},
|
||||
data={"hf_token": " "},
|
||||
)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
body = response.json()
|
||||
self.assertEqual(body["code"], "missing_hf_token")
|
||||
|
||||
def test_diarize_returns_503_when_pyannote_not_installed(self) -> None:
|
||||
with mock.patch.dict(sys.modules, {"pyannote": None, "pyannote.audio": None}):
|
||||
response = self.client.post(
|
||||
"/diarize",
|
||||
files={"audio": ("audio.wav", b"RIFF0000WAVEfmt ", "audio/wav")},
|
||||
data={"hf_token": "hf_test_token"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 503)
|
||||
body = response.json()
|
||||
self.assertEqual(body["code"], "pyannote_not_installed")
|
||||
|
||||
def test_diarize_returns_segments_and_cleans_temp_file(self) -> None:
|
||||
observed_paths: list[Path] = []
|
||||
observed_kwargs: list[dict[str, object]] = []
|
||||
|
||||
class _FakePipeline:
|
||||
@classmethod
|
||||
def from_pretrained(cls, model_id: str, **kwargs: object) -> "_FakePipeline":
|
||||
self_inst = cls()
|
||||
self_inst.model_id = model_id
|
||||
self_inst.init_kwargs = kwargs
|
||||
return self_inst
|
||||
|
||||
def __call__(self, audio_path: str, **kwargs: object) -> _FakeAnnotation:
|
||||
observed_paths.append(Path(audio_path))
|
||||
observed_kwargs.append(dict(kwargs))
|
||||
self.temp_existed_during_call = Path(audio_path).exists()
|
||||
return _FakeAnnotation()
|
||||
|
||||
fake_pyannote = types.ModuleType("pyannote")
|
||||
fake_pyannote_audio = types.ModuleType("pyannote.audio")
|
||||
fake_pyannote_audio.Pipeline = _FakePipeline # type: ignore[attr-defined]
|
||||
|
||||
with mock.patch.dict(
|
||||
sys.modules,
|
||||
{"pyannote": fake_pyannote, "pyannote.audio": fake_pyannote_audio},
|
||||
):
|
||||
response = self.client.post(
|
||||
"/diarize",
|
||||
files={"audio": ("audio.wav", b"RIFF1234WAVEfmt ", "audio/wav")},
|
||||
data={"hf_token": "hf_valid_token", "num_speakers": "2"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
body = response.json()
|
||||
self.assertEqual(body["status"], "ok")
|
||||
self.assertEqual(body["num_speakers"], 2)
|
||||
self.assertEqual(
|
||||
body["segments"],
|
||||
[
|
||||
{"speaker": "SPEAKER_00", "start": 0.2, "end": 4.5},
|
||||
{"speaker": "SPEAKER_01", "start": 5.0, "end": 9.8},
|
||||
{"speaker": "SPEAKER_00", "start": 10.1, "end": 13.0},
|
||||
],
|
||||
)
|
||||
self.assertEqual(observed_kwargs, [{"num_speakers": 2}])
|
||||
self.assertEqual(len(observed_paths), 1)
|
||||
self.assertFalse(observed_paths[0].exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
87
apps/desktop/sidecar/tests/test_transcribe_hygiene.py
Normal file
87
apps/desktop/sidecar/tests/test_transcribe_hygiene.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
"""Sidecar /transcribe & /shutdown hygiene regression tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
SIDECAR_DIR = Path(__file__).resolve().parent.parent
|
||||
if str(SIDECAR_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SIDECAR_DIR))
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
import main # noqa: E402
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
pass
|
||||
|
||||
|
||||
class TranscribeAndShutdownHygieneTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._orig_model = main._model
|
||||
self._orig_model_id = main._model_id
|
||||
self._orig_thread = main._download_thread
|
||||
main._model = _FakeModel() # type: ignore[assignment]
|
||||
main._model_id = "tiny"
|
||||
self.client = TestClient(main.app)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
main._model = self._orig_model
|
||||
main._model_id = self._orig_model_id
|
||||
main._download_thread = self._orig_thread
|
||||
main._download_cancel.clear()
|
||||
|
||||
def test_transcribe_single_byte_pcm_returns_400_without_frombuffer_crash(self) -> None:
|
||||
res = self.client.post(
|
||||
"/transcribe",
|
||||
files={"audio": ("chunk.pcm", b"\x01", "application/octet-stream")},
|
||||
)
|
||||
self.assertEqual(res.status_code, 400)
|
||||
self.assertEqual(res.json()["status"], "error")
|
||||
|
||||
def test_transcribe_odd_byte_pcm_truncates_trailing_byte_and_succeeds(self) -> None:
|
||||
observed_samples: list[int] = []
|
||||
|
||||
def fake_transcribe(_model, audio_array, _kwargs, _resolved_id, _is_primary, _cancel):
|
||||
observed_samples.append(len(audio_array))
|
||||
return [{"text": "ok", "start": 0.0, "end": 0.1}], "ok", "ko"
|
||||
|
||||
with mock.patch.object(main, "_transcribe_blocking", side_effect=fake_transcribe):
|
||||
res = self.client.post(
|
||||
"/transcribe",
|
||||
files={"audio": ("chunk.pcm", b"\x00\x01\x02\x03\x99", "application/octet-stream")},
|
||||
)
|
||||
|
||||
self.assertEqual(res.status_code, 200)
|
||||
self.assertEqual(observed_samples, [2])
|
||||
self.assertEqual(res.json()["text"], "ok")
|
||||
|
||||
def test_shutdown_cancels_active_download_thread(self) -> None:
|
||||
stopped = threading.Event()
|
||||
|
||||
def fake_worker() -> None:
|
||||
main._download_cancel.wait(timeout=5.0)
|
||||
stopped.set()
|
||||
|
||||
main._download_cancel.clear()
|
||||
worker = threading.Thread(target=fake_worker, daemon=True)
|
||||
main._download_thread = worker
|
||||
worker.start()
|
||||
|
||||
with mock.patch.object(main, "_server", None):
|
||||
res = self.client.post("/shutdown")
|
||||
|
||||
self.assertEqual(res.status_code, 200)
|
||||
self.assertEqual(res.json()["status"], "shutting_down")
|
||||
self.assertTrue(main._download_cancel.is_set())
|
||||
self.assertTrue(stopped.is_set())
|
||||
self.assertFalse(worker.is_alive())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue