G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
|
|
@ -1,7 +1,7 @@
|
|||
"""음성 캐스케이드 — OpenAI STT + OpenAI/Higgs TTS 어댑터.
|
||||
"""음성 캐스케이드 — Deepgram/OpenAI STT + OpenAI/Higgs TTS 어댑터.
|
||||
|
||||
MASTERPLAN '음성 필수'(한신대 요구) / DESIGN_CONCEPT §5.2(음성 오브 4상태) / §4.3(립싱크 RMS):
|
||||
STT : OpenAI /v1/audio/transcriptions (gpt-4o-transcribe | whisper-1). 학습자 음성 → 텍스트.
|
||||
STT : Deepgram 실시간 WebSocket 또는 OpenAI 배치 전사. 학습자 음성 → 텍스트.
|
||||
TTS : OpenAI /v1/audio/speech 또는 로컬 Higgs v3. 내담자 텍스트 → 음성.
|
||||
|
||||
설계 원칙(이 모듈의 경계):
|
||||
|
|
@ -18,14 +18,21 @@ PRESET_TO_OPENAI_VOICE 테이블이 흡수. 새 preset 추가는 이 테이블
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Mapping, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
from websockets.asyncio.client import connect as websocket_connect
|
||||
|
||||
from ..config import settings
|
||||
from .voice_runtime import voice_runtime_metrics
|
||||
from ..paths import repo_root, repo_path
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
|
@ -35,6 +42,8 @@ OPENAI_BASE_URL = "https://api.openai.com/v1"
|
|||
STT_ENDPOINT = "/audio/transcriptions"
|
||||
TTS_ENDPOINT = "/audio/speech"
|
||||
HIGGS_TTS_ENDPOINT = "/tts"
|
||||
DEEPGRAM_STT_URL = "wss://api.deepgram.com/v1/listen"
|
||||
DEEPGRAM_STT_MODEL = "nova-3"
|
||||
|
||||
# STT 모델: gpt-4o-transcribe(고품질) — 미가용 폴백은 whisper-1.
|
||||
STT_MODEL = "gpt-4o-transcribe"
|
||||
|
|
@ -159,6 +168,524 @@ class TranscriptResult:
|
|||
model: str = STT_MODEL
|
||||
duration: Optional[float] = None
|
||||
provider_events: list[dict[str, object]] = field(default_factory=list)
|
||||
words: list["TranscriptWord"] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TranscriptWord:
|
||||
"""Provider word timing kept in process until it is privacy-safe hashed."""
|
||||
|
||||
word: str
|
||||
start: float
|
||||
end: float
|
||||
confidence: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StreamingTranscriptEvent:
|
||||
"""Provider-neutral live transcript update emitted by streaming STT."""
|
||||
|
||||
text: str
|
||||
final: bool
|
||||
speech_final: bool
|
||||
confidence: float | None = None
|
||||
|
||||
|
||||
class DeepgramStreamingSession:
|
||||
"""One Deepgram Listen WebSocket, scoped to exactly one learner utterance."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
socket: Any,
|
||||
*,
|
||||
model: str,
|
||||
language: str,
|
||||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||||
keepalive_seconds: float,
|
||||
finalize_timeout_seconds: float,
|
||||
) -> None:
|
||||
self._socket = socket
|
||||
self._model = model
|
||||
self._language = language
|
||||
self._on_event = on_event
|
||||
self._keepalive_seconds = keepalive_seconds
|
||||
self._finalize_timeout_seconds = finalize_timeout_seconds
|
||||
self._send_lock = asyncio.Lock()
|
||||
self._last_audio_sent_at = time.monotonic()
|
||||
self._final_segments: list[str] = []
|
||||
self._words: list[TranscriptWord] = []
|
||||
self._provider_events: list[dict[str, object]] = []
|
||||
self._duration: float | None = None
|
||||
self._error: RuntimeError | None = None
|
||||
self._finishing = False
|
||||
self._closed = False
|
||||
self._runtime_closed = False
|
||||
voice_runtime_metrics.streaming_provider_opened()
|
||||
self._receiver_task = asyncio.create_task(self._receive())
|
||||
self._keepalive_task = asyncio.create_task(self._keepalive())
|
||||
|
||||
async def send_audio(self, audio: bytes) -> None:
|
||||
if not audio:
|
||||
return
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
if self._closed or self._receiver_task.done():
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
raise RuntimeError("Deepgram streaming STT connection closed")
|
||||
try:
|
||||
async with self._send_lock:
|
||||
await self._socket.send(audio)
|
||||
self._last_audio_sent_at = time.monotonic()
|
||||
except Exception as exc:
|
||||
raise RuntimeError("Deepgram streaming STT transport failed") from exc
|
||||
|
||||
async def finish(self) -> TranscriptResult:
|
||||
"""Flush remaining audio with CloseStream and await final Results/Metadata."""
|
||||
|
||||
if self._closed:
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
return self._result()
|
||||
self._finishing = True
|
||||
self._keepalive_task.cancel()
|
||||
try:
|
||||
async with self._send_lock:
|
||||
await self._socket.send(json.dumps({"type": "CloseStream"}))
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(self._receiver_task),
|
||||
timeout=self._finalize_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
await self.abort()
|
||||
raise RuntimeError("Deepgram streaming STT finalization timed out") from exc
|
||||
except Exception as exc:
|
||||
await self.abort()
|
||||
if isinstance(exc, RuntimeError):
|
||||
raise
|
||||
raise RuntimeError("Deepgram streaming STT finalization failed") from exc
|
||||
finally:
|
||||
await self._cancel_keepalive()
|
||||
self._closed = True
|
||||
self._close_runtime_metrics(outcome="finalized")
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
return self._result()
|
||||
|
||||
async def abort(self) -> None:
|
||||
"""Close without asking the provider to process buffered audio."""
|
||||
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._finishing = True
|
||||
self._keepalive_task.cancel()
|
||||
if not self._receiver_task.done():
|
||||
self._receiver_task.cancel()
|
||||
try:
|
||||
await self._socket.close(code=1000, reason="utterance aborted")
|
||||
except TypeError:
|
||||
try:
|
||||
await self._socket.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
await self._cancel_keepalive()
|
||||
if not self._receiver_task.done():
|
||||
try:
|
||||
await self._receiver_task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
self._close_runtime_metrics(outcome="aborted")
|
||||
|
||||
def _close_runtime_metrics(self, *, outcome: str) -> None:
|
||||
if self._runtime_closed:
|
||||
return
|
||||
self._runtime_closed = True
|
||||
voice_runtime_metrics.streaming_provider_closed(outcome=outcome)
|
||||
|
||||
async def _receive(self) -> None:
|
||||
try:
|
||||
async for raw in self._socket:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
message_type = str(payload.get("type") or "")
|
||||
if message_type == "Results":
|
||||
await self._consume_results(payload)
|
||||
elif message_type == "Metadata":
|
||||
duration = _optional_float(payload.get("duration"))
|
||||
if duration is not None and duration >= 0:
|
||||
self._duration = max(self._duration or 0.0, duration)
|
||||
elif message_type in {"Error", "Warning"}:
|
||||
self._error = RuntimeError("Deepgram streaming STT provider failed")
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._error = RuntimeError("Deepgram streaming STT receive failed")
|
||||
self._error.__cause__ = exc
|
||||
|
||||
async def _consume_results(self, payload: dict[str, object]) -> None:
|
||||
channel = payload.get("channel")
|
||||
if not isinstance(channel, dict):
|
||||
return
|
||||
alternatives = channel.get("alternatives")
|
||||
if not isinstance(alternatives, list) or not alternatives:
|
||||
return
|
||||
alternative = alternatives[0]
|
||||
if not isinstance(alternative, dict):
|
||||
return
|
||||
transcript = str(alternative.get("transcript") or "").strip()
|
||||
is_final = bool(payload.get("is_final"))
|
||||
speech_final = bool(payload.get("speech_final"))
|
||||
confidence = _optional_float(alternative.get("confidence"))
|
||||
start = max(0.0, _optional_float(payload.get("start")) or 0.0)
|
||||
duration = max(0.0, _optional_float(payload.get("duration")) or 0.0)
|
||||
if duration:
|
||||
self._duration = max(self._duration or 0.0, start + duration)
|
||||
|
||||
if is_final and transcript:
|
||||
self._final_segments.append(transcript)
|
||||
self._consume_final_words(alternative.get("words"))
|
||||
|
||||
display_parts = list(self._final_segments)
|
||||
if transcript and not is_final:
|
||||
display_parts.append(transcript)
|
||||
display_text = " ".join(part for part in display_parts if part).strip()
|
||||
if not display_text and not speech_final:
|
||||
return
|
||||
|
||||
event_type = "speech_final" if speech_final else (
|
||||
"speech_end" if is_final else "voice_activity"
|
||||
)
|
||||
provider_event: dict[str, object] = {
|
||||
"type": event_type,
|
||||
"provider": "deepgram",
|
||||
"source": "streaming_stt",
|
||||
"start_ms": round(start * 1000),
|
||||
"duration_ms": round(duration * 1000),
|
||||
"is_final": is_final,
|
||||
}
|
||||
if confidence is not None:
|
||||
provider_event["confidence"] = confidence
|
||||
if is_final or speech_final:
|
||||
self._provider_events.append(provider_event)
|
||||
await self._on_event(
|
||||
StreamingTranscriptEvent(
|
||||
text=display_text,
|
||||
final=is_final,
|
||||
speech_final=speech_final,
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
|
||||
def _consume_final_words(self, value: object) -> None:
|
||||
if not isinstance(value, list):
|
||||
return
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
word = str(item.get("punctuated_word") or item.get("word") or "").strip()
|
||||
start = _optional_float(item.get("start"))
|
||||
end = _optional_float(item.get("end"))
|
||||
if not word or start is None or end is None or end <= start:
|
||||
continue
|
||||
confidence = _optional_float(item.get("confidence"))
|
||||
self._words.append(
|
||||
TranscriptWord(
|
||||
word=word,
|
||||
start=max(0.0, start),
|
||||
end=max(0.0, end),
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
timing_event: dict[str, object] = {
|
||||
"type": "stt_word",
|
||||
"provider": "deepgram",
|
||||
"source": "stt_word_timestamps",
|
||||
"start_ms": round(start * 1000),
|
||||
"end_ms": round(end * 1000),
|
||||
"is_final": True,
|
||||
}
|
||||
if confidence is not None:
|
||||
timing_event["confidence"] = confidence
|
||||
self._provider_events.append(timing_event)
|
||||
self._duration = max(self._duration or 0.0, end)
|
||||
|
||||
async def _keepalive(self) -> None:
|
||||
try:
|
||||
while not self._finishing and not self._closed:
|
||||
await asyncio.sleep(self._keepalive_seconds)
|
||||
idle_for = time.monotonic() - self._last_audio_sent_at
|
||||
if idle_for < self._keepalive_seconds:
|
||||
continue
|
||||
async with self._send_lock:
|
||||
await self._socket.send(json.dumps({"type": "KeepAlive"}))
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as exc:
|
||||
if not self._finishing:
|
||||
self._error = RuntimeError("Deepgram streaming STT keepalive failed")
|
||||
self._error.__cause__ = exc
|
||||
|
||||
async def _cancel_keepalive(self) -> None:
|
||||
if self._keepalive_task.done():
|
||||
return
|
||||
self._keepalive_task.cancel()
|
||||
try:
|
||||
await self._keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
def _result(self) -> TranscriptResult:
|
||||
return TranscriptResult(
|
||||
text=" ".join(self._final_segments).strip(),
|
||||
language=self._language,
|
||||
model=self._model,
|
||||
duration=self._duration,
|
||||
provider_events=list(self._provider_events),
|
||||
words=list(self._words),
|
||||
)
|
||||
|
||||
|
||||
class LocalWhisperStreamingSession:
|
||||
"""One loopback faster-whisper stream, scoped to exactly one learner utterance.
|
||||
|
||||
Same public surface as the Deepgram session so the WebSocket route does not
|
||||
branch on provider. Audio never leaves the host: the sidecar keeps only an
|
||||
in-memory utterance buffer and drops it when the utterance ends.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
socket: Any,
|
||||
*,
|
||||
model: str,
|
||||
language: str,
|
||||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||||
finalize_timeout_seconds: float,
|
||||
) -> None:
|
||||
self._socket = socket
|
||||
self._model = model
|
||||
self._language = language
|
||||
self._on_event = on_event
|
||||
self._finalize_timeout_seconds = finalize_timeout_seconds
|
||||
self._send_lock = asyncio.Lock()
|
||||
self._final_segments: list[str] = []
|
||||
self._words: list[TranscriptWord] = []
|
||||
self._provider_events: list[dict[str, object]] = []
|
||||
self._duration: float | None = None
|
||||
# 확정된 발화들의 누적 길이. interim 은 같은 발화라 오프셋을 밀지 않는다.
|
||||
self._utterance_offset = 0.0
|
||||
self._error: RuntimeError | None = None
|
||||
self._finishing = False
|
||||
self._closed = False
|
||||
self._runtime_closed = False
|
||||
voice_runtime_metrics.streaming_provider_opened()
|
||||
self._receiver_task = asyncio.create_task(self._receive())
|
||||
|
||||
async def send_audio(self, audio: bytes) -> None:
|
||||
if not audio:
|
||||
return
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
if self._closed or self._receiver_task.done():
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
raise RuntimeError("Local whisper streaming STT connection closed")
|
||||
try:
|
||||
async with self._send_lock:
|
||||
await self._socket.send(audio)
|
||||
except Exception as exc:
|
||||
self._error = RuntimeError("Local whisper streaming STT transport failed")
|
||||
self._error.__cause__ = exc
|
||||
raise self._error from exc
|
||||
|
||||
async def finish(self) -> TranscriptResult:
|
||||
if self._closed:
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
return self._result()
|
||||
self._finishing = True
|
||||
try:
|
||||
async with self._send_lock:
|
||||
await self._socket.send(json.dumps({"type": "CloseStream"}))
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(self._receiver_task),
|
||||
timeout=self._finalize_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
await self.abort()
|
||||
raise RuntimeError(
|
||||
"Local whisper streaming STT finalization timed out"
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
await self.abort()
|
||||
if isinstance(exc, RuntimeError):
|
||||
raise
|
||||
raise RuntimeError(
|
||||
"Local whisper streaming STT finalization failed"
|
||||
) from exc
|
||||
self._closed = True
|
||||
self._close_runtime_metrics(outcome="finalized")
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
return self._result()
|
||||
|
||||
async def abort(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._finishing = True
|
||||
if not self._receiver_task.done():
|
||||
self._receiver_task.cancel()
|
||||
try:
|
||||
await self._socket.close(code=1000, reason="utterance aborted")
|
||||
except TypeError:
|
||||
try:
|
||||
await self._socket.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
if not self._receiver_task.done():
|
||||
try:
|
||||
await self._receiver_task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
self._close_runtime_metrics(outcome="aborted")
|
||||
|
||||
def _close_runtime_metrics(self, *, outcome: str) -> None:
|
||||
if self._runtime_closed:
|
||||
return
|
||||
self._runtime_closed = True
|
||||
voice_runtime_metrics.streaming_provider_closed(outcome=outcome)
|
||||
|
||||
async def _receive(self) -> None:
|
||||
try:
|
||||
async for message in self._socket:
|
||||
if isinstance(message, (bytes, bytearray)):
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(message)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
kind = str(payload.get("type") or "")
|
||||
if kind == "error":
|
||||
self._error = RuntimeError(
|
||||
"Local whisper streaming STT provider failed"
|
||||
)
|
||||
return
|
||||
if kind == "transcript":
|
||||
await self._consume_transcript(payload)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if not self._finishing:
|
||||
self._error = RuntimeError("Local whisper streaming STT receive failed")
|
||||
self._error.__cause__ = exc
|
||||
|
||||
async def _consume_transcript(self, payload: dict[str, object]) -> None:
|
||||
transcript = str(payload.get("text") or "").strip()
|
||||
is_final = bool(payload.get("is_final"))
|
||||
speech_final = bool(payload.get("speech_final"))
|
||||
confidence = _optional_float(payload.get("confidence"))
|
||||
duration = max(0.0, _optional_float(payload.get("duration")) or 0.0)
|
||||
start = self._utterance_offset
|
||||
if duration:
|
||||
self._duration = max(self._duration or 0.0, start + duration)
|
||||
|
||||
if is_final and transcript:
|
||||
self._final_segments.append(transcript)
|
||||
self._consume_final_words(payload.get("words"), offset=start)
|
||||
if is_final:
|
||||
# 다음 발화는 이 발화가 끝난 지점부터 시작한다.
|
||||
self._utterance_offset = start + duration
|
||||
|
||||
display_parts = list(self._final_segments)
|
||||
if transcript and not is_final:
|
||||
display_parts.append(transcript)
|
||||
display_text = " ".join(part for part in display_parts if part).strip()
|
||||
if not display_text and not speech_final:
|
||||
return
|
||||
|
||||
event_type = "speech_final" if speech_final else (
|
||||
"speech_end" if is_final else "voice_activity"
|
||||
)
|
||||
provider_event: dict[str, object] = {
|
||||
"type": event_type,
|
||||
"provider": "local_whisper",
|
||||
"source": "streaming_stt",
|
||||
"start_ms": round(start * 1000),
|
||||
"duration_ms": round(duration * 1000),
|
||||
"is_final": is_final,
|
||||
}
|
||||
if confidence is not None:
|
||||
provider_event["confidence"] = confidence
|
||||
if is_final or speech_final:
|
||||
self._provider_events.append(provider_event)
|
||||
await self._on_event(
|
||||
StreamingTranscriptEvent(
|
||||
text=display_text,
|
||||
final=is_final,
|
||||
speech_final=speech_final,
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
|
||||
def _consume_final_words(self, value: object, *, offset: float) -> None:
|
||||
if not isinstance(value, list):
|
||||
return
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
word = str(item.get("word") or "").strip()
|
||||
start = _optional_float(item.get("start"))
|
||||
end = _optional_float(item.get("end"))
|
||||
if not word or start is None or end is None or end <= start:
|
||||
continue
|
||||
confidence = _optional_float(item.get("confidence"))
|
||||
absolute_start = max(0.0, offset + start)
|
||||
absolute_end = max(absolute_start, offset + end)
|
||||
self._words.append(
|
||||
TranscriptWord(
|
||||
word=word,
|
||||
start=absolute_start,
|
||||
end=absolute_end,
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
timing_event: dict[str, object] = {
|
||||
"type": "stt_word",
|
||||
"provider": "local_whisper",
|
||||
"source": "stt_word_timestamps",
|
||||
"start_ms": round(absolute_start * 1000),
|
||||
"end_ms": round(absolute_end * 1000),
|
||||
"is_final": True,
|
||||
}
|
||||
if confidence is not None:
|
||||
timing_event["confidence"] = confidence
|
||||
self._provider_events.append(timing_event)
|
||||
self._duration = max(self._duration or 0.0, absolute_end)
|
||||
|
||||
def _result(self) -> TranscriptResult:
|
||||
return TranscriptResult(
|
||||
text=" ".join(self._final_segments).strip(),
|
||||
language=self._language,
|
||||
model=self._model,
|
||||
duration=self._duration,
|
||||
provider_events=list(self._provider_events),
|
||||
words=list(self._words),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -361,7 +888,7 @@ def assess_end_of_turn(
|
|||
# OpenAI 음성 서비스
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
class VoiceService:
|
||||
"""OpenAI STT와 선택형 OpenAI/Higgs TTS 어댑터(httpx 풀 공유)."""
|
||||
"""Deepgram/OpenAI STT와 선택형 OpenAI/Higgs TTS 어댑터."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -374,10 +901,130 @@ class VoiceService:
|
|||
tts_provider: Optional[str] = None,
|
||||
higgs_base_url: Optional[str] = None,
|
||||
higgs_timeout_seconds: Optional[float] = None,
|
||||
stt_provider: Optional[str] = None,
|
||||
deepgram_api_key: Optional[str] = None,
|
||||
deepgram_stt_url: Optional[str] = None,
|
||||
deepgram_stt_model: Optional[str] = None,
|
||||
deepgram_stt_language: Optional[str] = None,
|
||||
deepgram_endpointing_ms: Optional[int] = None,
|
||||
deepgram_utterance_end_ms: Optional[int] = None,
|
||||
deepgram_keepalive_seconds: Optional[float] = None,
|
||||
deepgram_finalize_timeout_seconds: Optional[float] = None,
|
||||
deepgram_mip_opt_out: Optional[bool] = None,
|
||||
deepgram_connect: Optional[Callable[..., Awaitable[Any]]] = None,
|
||||
local_whisper_stt_url: Optional[str] = None,
|
||||
local_whisper_stt_model: Optional[str] = None,
|
||||
local_whisper_stt_language: Optional[str] = None,
|
||||
local_whisper_endpointing_ms: Optional[int] = None,
|
||||
local_whisper_utterance_end_ms: Optional[int] = None,
|
||||
local_whisper_finalize_timeout_seconds: Optional[float] = None,
|
||||
local_whisper_connect: Optional[Callable[..., Awaitable[Any]]] = None,
|
||||
) -> None:
|
||||
self._api_key = (api_key if api_key is not None else settings.openai_api_key) or ""
|
||||
self._base_url = (base_url or settings.openai_base_url or OPENAI_BASE_URL).rstrip("/")
|
||||
self._environment = environment if environment is not None else settings.environment
|
||||
self._stt_provider = (
|
||||
stt_provider if stt_provider is not None else settings.voice_stt_provider
|
||||
).strip().lower()
|
||||
configured_deepgram_key = settings.deepgram_api_key.get_secret_value()
|
||||
self._deepgram_api_key = (
|
||||
deepgram_api_key
|
||||
if deepgram_api_key is not None
|
||||
else configured_deepgram_key
|
||||
).strip()
|
||||
self._deepgram_stt_url = (
|
||||
deepgram_stt_url
|
||||
if deepgram_stt_url is not None
|
||||
else settings.deepgram_stt_url
|
||||
).rstrip("?")
|
||||
self._deepgram_stt_model = (
|
||||
deepgram_stt_model
|
||||
if deepgram_stt_model is not None
|
||||
else settings.deepgram_stt_model
|
||||
).strip() or DEEPGRAM_STT_MODEL
|
||||
self._deepgram_stt_language = (
|
||||
deepgram_stt_language
|
||||
if deepgram_stt_language is not None
|
||||
else settings.deepgram_stt_language
|
||||
).strip() or STT_LANGUAGE
|
||||
self._deepgram_endpointing_ms = max(
|
||||
10,
|
||||
int(
|
||||
deepgram_endpointing_ms
|
||||
if deepgram_endpointing_ms is not None
|
||||
else settings.deepgram_endpointing_ms
|
||||
),
|
||||
)
|
||||
self._deepgram_utterance_end_ms = max(
|
||||
1000,
|
||||
int(
|
||||
deepgram_utterance_end_ms
|
||||
if deepgram_utterance_end_ms is not None
|
||||
else settings.deepgram_utterance_end_ms
|
||||
),
|
||||
)
|
||||
self._deepgram_keepalive_seconds = max(
|
||||
1.0,
|
||||
float(
|
||||
deepgram_keepalive_seconds
|
||||
if deepgram_keepalive_seconds is not None
|
||||
else settings.deepgram_keepalive_seconds
|
||||
),
|
||||
)
|
||||
self._deepgram_finalize_timeout_seconds = max(
|
||||
1.0,
|
||||
float(
|
||||
deepgram_finalize_timeout_seconds
|
||||
if deepgram_finalize_timeout_seconds is not None
|
||||
else settings.deepgram_finalize_timeout_seconds
|
||||
),
|
||||
)
|
||||
self._deepgram_mip_opt_out = (
|
||||
bool(settings.deepgram_mip_opt_out)
|
||||
if deepgram_mip_opt_out is None
|
||||
else bool(deepgram_mip_opt_out)
|
||||
)
|
||||
self._deepgram_connect = deepgram_connect or websocket_connect
|
||||
self._local_whisper_stt_url = (
|
||||
local_whisper_stt_url
|
||||
if local_whisper_stt_url is not None
|
||||
else settings.local_whisper_stt_url
|
||||
).strip().rstrip("?")
|
||||
self._local_whisper_stt_model = (
|
||||
local_whisper_stt_model
|
||||
if local_whisper_stt_model is not None
|
||||
else settings.local_whisper_stt_model
|
||||
).strip() or "large-v3"
|
||||
self._local_whisper_stt_language = (
|
||||
local_whisper_stt_language
|
||||
if local_whisper_stt_language is not None
|
||||
else settings.local_whisper_stt_language
|
||||
).strip() or STT_LANGUAGE
|
||||
self._local_whisper_endpointing_ms = max(
|
||||
10,
|
||||
int(
|
||||
local_whisper_endpointing_ms
|
||||
if local_whisper_endpointing_ms is not None
|
||||
else settings.local_whisper_endpointing_ms
|
||||
),
|
||||
)
|
||||
self._local_whisper_utterance_end_ms = max(
|
||||
1000,
|
||||
int(
|
||||
local_whisper_utterance_end_ms
|
||||
if local_whisper_utterance_end_ms is not None
|
||||
else settings.local_whisper_utterance_end_ms
|
||||
),
|
||||
)
|
||||
self._local_whisper_finalize_timeout_seconds = max(
|
||||
1.0,
|
||||
float(
|
||||
local_whisper_finalize_timeout_seconds
|
||||
if local_whisper_finalize_timeout_seconds is not None
|
||||
else settings.local_whisper_finalize_timeout_seconds
|
||||
),
|
||||
)
|
||||
self._local_whisper_connect = local_whisper_connect or websocket_connect
|
||||
self._tts_provider = (
|
||||
tts_provider if tts_provider is not None else settings.voice_tts_provider
|
||||
).strip().lower()
|
||||
|
|
@ -436,8 +1083,162 @@ class VoiceService:
|
|||
return self.stt_available() and self.tts_available()
|
||||
|
||||
def stt_available(self) -> bool:
|
||||
return self.streaming_stt_enabled() or self.batch_stt_available()
|
||||
|
||||
def batch_stt_available(self) -> bool:
|
||||
return bool(self._api_key)
|
||||
|
||||
def streaming_stt_enabled(self) -> bool:
|
||||
if self._stt_provider == "deepgram":
|
||||
return bool(self._deepgram_api_key)
|
||||
if self._stt_provider == "local_whisper":
|
||||
# 로컬 사이드카는 키가 없다. URL 설정만으로 활성화된다.
|
||||
return bool(self._local_whisper_stt_url)
|
||||
return False
|
||||
|
||||
def stt_provider(self) -> str:
|
||||
if self.streaming_stt_enabled():
|
||||
return self._stt_provider
|
||||
if (
|
||||
self._stt_provider in {"deepgram", "local_whisper"}
|
||||
and self.batch_stt_available()
|
||||
):
|
||||
return "openai-batch-fallback"
|
||||
if self.batch_stt_available():
|
||||
return "openai"
|
||||
return "unavailable"
|
||||
|
||||
def stt_model(self) -> str:
|
||||
if self.streaming_stt_enabled():
|
||||
if self._stt_provider == "local_whisper":
|
||||
return self._local_whisper_stt_model
|
||||
return self._deepgram_stt_model
|
||||
return STT_MODEL
|
||||
|
||||
def can_stream_audio(
|
||||
self,
|
||||
*,
|
||||
fmt: str | None,
|
||||
sample_rate: int | None,
|
||||
channels: int | None,
|
||||
sample_width: int | None,
|
||||
) -> bool:
|
||||
normalized = (fmt or "").strip().lower()
|
||||
return (
|
||||
self.streaming_stt_enabled()
|
||||
and normalized in {"pcm", "s16le", "linear16", "audio/pcm"}
|
||||
and sample_width == 2
|
||||
and sample_rate is not None
|
||||
and 8000 <= sample_rate <= 192000
|
||||
and channels in {1, 2}
|
||||
)
|
||||
|
||||
async def open_streaming_transcription(
|
||||
self,
|
||||
*,
|
||||
fmt: str | None,
|
||||
sample_rate: int | None,
|
||||
channels: int | None,
|
||||
sample_width: int | None,
|
||||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||||
) -> DeepgramStreamingSession | LocalWhisperStreamingSession:
|
||||
if not self.can_stream_audio(
|
||||
fmt=fmt,
|
||||
sample_rate=sample_rate,
|
||||
channels=channels,
|
||||
sample_width=sample_width,
|
||||
):
|
||||
raise VoiceUnavailable(
|
||||
"streaming STT requires linear16 PCM metadata"
|
||||
)
|
||||
assert sample_rate is not None and channels is not None
|
||||
if self._stt_provider == "local_whisper":
|
||||
return await self._open_local_whisper_transcription(
|
||||
sample_rate=sample_rate, channels=channels, on_event=on_event
|
||||
)
|
||||
query = urlencode(
|
||||
{
|
||||
"model": self._deepgram_stt_model,
|
||||
"language": self._deepgram_stt_language,
|
||||
"encoding": "linear16",
|
||||
"sample_rate": sample_rate,
|
||||
"channels": channels,
|
||||
"interim_results": "true",
|
||||
"punctuate": "true",
|
||||
"smart_format": "true",
|
||||
"vad_events": "true",
|
||||
"endpointing": self._deepgram_endpointing_ms,
|
||||
"utterance_end_ms": self._deepgram_utterance_end_ms,
|
||||
"mip_opt_out": "true" if self._deepgram_mip_opt_out else "false",
|
||||
}
|
||||
)
|
||||
url = f"{self._deepgram_stt_url}?{query}"
|
||||
try:
|
||||
socket = await self._deepgram_connect(
|
||||
url,
|
||||
additional_headers={
|
||||
"Authorization": f"Token {self._deepgram_api_key}",
|
||||
},
|
||||
open_timeout=10,
|
||||
close_timeout=5,
|
||||
ping_interval=20,
|
||||
ping_timeout=20,
|
||||
max_size=2 * 1024 * 1024,
|
||||
max_queue=16,
|
||||
write_limit=64 * 1024,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError("Deepgram streaming STT connection failed") from exc
|
||||
return DeepgramStreamingSession(
|
||||
socket,
|
||||
model=self._deepgram_stt_model,
|
||||
language=self._deepgram_stt_language,
|
||||
on_event=on_event,
|
||||
keepalive_seconds=self._deepgram_keepalive_seconds,
|
||||
finalize_timeout_seconds=self._deepgram_finalize_timeout_seconds,
|
||||
)
|
||||
|
||||
async def _open_local_whisper_transcription(
|
||||
self,
|
||||
*,
|
||||
sample_rate: int,
|
||||
channels: int,
|
||||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||||
) -> LocalWhisperStreamingSession:
|
||||
query = urlencode(
|
||||
{
|
||||
"model": self._local_whisper_stt_model,
|
||||
"language": self._local_whisper_stt_language,
|
||||
"sample_rate": sample_rate,
|
||||
"channels": channels,
|
||||
"endpointing": self._local_whisper_endpointing_ms,
|
||||
"utterance_end_ms": self._local_whisper_utterance_end_ms,
|
||||
}
|
||||
)
|
||||
url = f"{self._local_whisper_stt_url}?{query}"
|
||||
try:
|
||||
socket = await self._local_whisper_connect(
|
||||
url,
|
||||
open_timeout=10,
|
||||
close_timeout=5,
|
||||
ping_interval=20,
|
||||
ping_timeout=20,
|
||||
max_size=2 * 1024 * 1024,
|
||||
max_queue=16,
|
||||
write_limit=64 * 1024,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"Local whisper streaming STT connection failed"
|
||||
) from exc
|
||||
return LocalWhisperStreamingSession(
|
||||
socket,
|
||||
model=self._local_whisper_stt_model,
|
||||
language=self._local_whisper_stt_language,
|
||||
on_event=on_event,
|
||||
finalize_timeout_seconds=self._local_whisper_finalize_timeout_seconds,
|
||||
)
|
||||
|
||||
def tts_available(self, voice: VoicePreset | None = None) -> bool:
|
||||
if self._higgs_enabled() and (voice is None or self._should_use_higgs_tts(voice)):
|
||||
return True
|
||||
|
|
@ -702,6 +1503,13 @@ def _nonnegative_int(value: object) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def _optional_float(value: object) -> float | None:
|
||||
try:
|
||||
return float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _clean_optional_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
|
@ -718,6 +1526,10 @@ __all__ = [
|
|||
"VoiceUnavailable",
|
||||
"VoicePreset",
|
||||
"TranscriptResult",
|
||||
"TranscriptWord",
|
||||
"StreamingTranscriptEvent",
|
||||
"DeepgramStreamingSession",
|
||||
"LocalWhisperStreamingSession",
|
||||
"EndOfTurnDecision",
|
||||
"TTSChunk",
|
||||
"VoiceService",
|
||||
|
|
@ -732,6 +1544,7 @@ __all__ = [
|
|||
"PERSONA_CODE_TO_PRESET",
|
||||
"DEFAULT_OPENAI_VOICE",
|
||||
"STT_MODEL",
|
||||
"DEEPGRAM_STT_MODEL",
|
||||
"TTS_MODEL",
|
||||
"HIGGS_TTS_MODEL",
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue