1635 lines
63 KiB
Python
1635 lines
63 KiB
Python
"""음성 캐스케이드 — Deepgram/OpenAI STT + OpenAI/Higgs TTS 어댑터.
|
||
|
||
MASTERPLAN '음성 필수'(한신대 요구) / DESIGN_CONCEPT §5.2(음성 오브 4상태) / §4.3(립싱크 RMS):
|
||
STT : Deepgram 실시간 WebSocket 또는 OpenAI 배치 전사. 학습자 음성 → 텍스트.
|
||
TTS : OpenAI /v1/audio/speech 또는 로컬 Higgs v3. 내담자 텍스트 → 음성.
|
||
|
||
설계 원칙(이 모듈의 경계):
|
||
- 순수 어댑터: httpx 로 OpenAI 음성 엔드포인트만 호출한다. 상담 로직(orchestrator)·상태머신은
|
||
호출부(routes/voice.py)가 조립한다. 여기는 "오디오↔텍스트" 변환 + voice preset 매핑만.
|
||
- API 키 없으면 명확히 degraded: is_available()=False, 호출 시 VoiceUnavailable.
|
||
절대 크래시·무한대기 금지(라우트가 503/close 로 변환).
|
||
- 립싱크 힌트: TTS 오디오 청크를 흘리며 RMS(진폭) 힌트를 같이 산출(설계 §4.3 — viseme 정밀
|
||
매칭 안 함, RMS 1채널). PCM 디코딩 의존성 없이 바이트 에너지 근사로 임시 RMS 추정.
|
||
|
||
페르소나 voice preset(persona/*.json voice.preset, 설계 §4.6) → OpenAI voice 매핑은
|
||
PRESET_TO_OPENAI_VOICE 테이블이 흡수. 새 preset 추가는 이 테이블만 손대면 된다.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
import time
|
||
from collections.abc import Awaitable, Callable
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any, AsyncIterator, Mapping, Optional
|
||
from urllib.parse import urlencode
|
||
|
||
import httpx
|
||
from websockets.asyncio.client import connect as websocket_connect
|
||
|
||
from ..config import settings
|
||
from .voice_runtime import voice_runtime_metrics
|
||
from ..paths import repo_root, repo_path
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# OpenAI 음성 엔드포인트/모델 상수
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
OPENAI_BASE_URL = "https://api.openai.com/v1"
|
||
STT_ENDPOINT = "/audio/transcriptions"
|
||
TTS_ENDPOINT = "/audio/speech"
|
||
HIGGS_TTS_ENDPOINT = "/tts"
|
||
MELOTTS_TTS_ENDPOINT = "/tts"
|
||
MELOTTS_TTS_MODEL = "melotts-korean"
|
||
DEEPGRAM_STT_URL = "wss://api.deepgram.com/v1/listen"
|
||
DEEPGRAM_STT_MODEL = "nova-3"
|
||
|
||
# STT 모델: gpt-4o-transcribe(고품질) — 미가용 폴백은 whisper-1.
|
||
STT_MODEL = "gpt-4o-transcribe"
|
||
STT_MODEL_FALLBACK = "whisper-1"
|
||
# TTS 모델: gpt-4o-mini-tts(저지연·표현력) — 폴백 tts-1.
|
||
TTS_MODEL = "gpt-4o-mini-tts"
|
||
TTS_MODEL_FALLBACK = "tts-1"
|
||
HIGGS_TTS_MODEL = "higgs-audio-v3-tts-4b"
|
||
|
||
# 전사 언어 힌트(상담은 한국어). OpenAI 는 ISO-639-1.
|
||
STT_LANGUAGE = "ko"
|
||
|
||
# TTS 출력 포맷: 브라우저 MediaSource/<audio> 친화. 스트리밍은 mp3/opus 청크.
|
||
TTS_RESPONSE_FORMAT = "mp3"
|
||
|
||
# End-of-turn readiness default for cascaded STT providers.
|
||
EOT_SILENCE_THRESHOLD_MS = 1200
|
||
|
||
POC_SAMPLE_TTS_PRESET = "soft-young-fem"
|
||
POC_SAMPLE_TTS_DEFAULT_DIR = (
|
||
repo_path("docs", "voice-art", "p1-seoyeon-higgs-v3-20260627")
|
||
)
|
||
POC_SAMPLE_TTS_CHUNK_SIZE = 4096
|
||
_POC_SAMPLE_TTS_DEFAULT_SAMPLE = "p1_seoyeon_01_depressed_slow"
|
||
_POC_SAMPLE_TTS_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||
(
|
||
"p1_seoyeon_03_anxious_guarded",
|
||
("엄마", "비밀", "말하지", "불안", "무서", "걱정", "들키", "싫", "갈래"),
|
||
),
|
||
(
|
||
"p1_seoyeon_02_tired_flat",
|
||
("잠", "피곤", "무거", "아무것도", "지쳐", "힘들", "에너지"),
|
||
),
|
||
(
|
||
"p1_seoyeon_05_recovered_lively",
|
||
("오늘은", "친구", "웃", "괜찮았", "좋았", "해냈"),
|
||
),
|
||
(
|
||
"p1_seoyeon_04_rapport_relief",
|
||
("괜찮", "들어", "고마", "선생님", "편해", "조금", "말해"),
|
||
),
|
||
)
|
||
|
||
_HIGGS_DELIVERY_TAGS: tuple[tuple[tuple[str, ...], str], ...] = (
|
||
(
|
||
("엄마", "비밀", "말하지", "불안", "무서", "걱정", "들키", "갈래"),
|
||
"<|emotion:fear|><|prosody:speed_fast|><|prosody:pitch_high|>",
|
||
),
|
||
(
|
||
("잠", "피곤", "무거", "아무것도", "지쳐", "힘들", "에너지"),
|
||
"<|emotion:sadness|><|prosody:speed_slow|><|prosody:expressive_low|>",
|
||
),
|
||
(
|
||
("오늘은", "친구", "웃", "괜찮았", "좋았", "해냈"),
|
||
"<|emotion:contentment|><|prosody:speed_fast|>",
|
||
),
|
||
(
|
||
("괜찮", "들어", "고마", "선생님", "편해", "조금", "말해"),
|
||
"<|emotion:relief|><|prosody:speed_slow|>",
|
||
),
|
||
)
|
||
|
||
# OpenAI 공식 voice 풀(2026 기준): alloy, ash, ballad, coral, echo, fable,
|
||
# nova, onyx, sage, shimmer, verse. 페르소나 톤별로 골라 매핑한다.
|
||
_OPENAI_VOICES = {
|
||
"alloy", "ash", "ballad", "coral", "echo", "fable",
|
||
"nova", "onyx", "sage", "shimmer", "verse",
|
||
}
|
||
DEFAULT_OPENAI_VOICE = "sage"
|
||
|
||
# ── 페르소나 voice preset(설계 §4.6) → OpenAI voice ──────────────────────────
|
||
# preset 명명: <톤><연령><성별> 조합(soft-young-fem 등). 새 페르소나는 여기에만 추가.
|
||
PRESET_TO_OPENAI_VOICE: dict[str, str] = {
|
||
# 청소년 여성(P1 서연) — 부드럽고 톤 높은
|
||
"soft-young-fem": "coral",
|
||
# 성인 남성(P2 민재) — 차분·안정·약간 긴장
|
||
"calm-adult-male": "ash",
|
||
# 성인 여성(P3 지우) — 따뜻하지만 지친
|
||
"warm-adult-fem": "shimmer",
|
||
# 범용 폴백 프리셋
|
||
"neutral": "sage",
|
||
}
|
||
|
||
# ── 페르소나 code(P1/P2/P3) → 기본 preset (persona 카드에 voice 필드 없을 때) ──
|
||
# persona.py 시드는 voice 필드를 갖지 않으므로 code 로 기본 preset 을 정한다.
|
||
# DB/JSON 페르소나가 voice.preset 을 직접 주면 그걸 우선한다(resolve_voice 참조).
|
||
PERSONA_CODE_TO_PRESET: dict[str, str] = {
|
||
"P1": "soft-young-fem",
|
||
"P2": "calm-adult-male",
|
||
"P3": "warm-adult-fem",
|
||
}
|
||
|
||
# TTS rate(말 속도) 페르소나 기본값(설계 §4.6 voice.rate). 1.0=표준.
|
||
PRESET_RATE: dict[str, float] = {
|
||
"soft-young-fem": 0.96,
|
||
"calm-adult-male": 1.0,
|
||
"warm-adult-fem": 0.98,
|
||
"neutral": 1.0,
|
||
}
|
||
|
||
|
||
class VoiceUnavailable(RuntimeError):
|
||
"""음성 미설정/장애. 라우트가 503/WS close(degraded) 로 변환."""
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class VoicePreset:
|
||
"""해석된 음성 프리셋(페르소나 → OpenAI 파라미터)."""
|
||
|
||
preset: str # 논리 preset 명(soft-young-fem 등)
|
||
openai_voice: str # OpenAI voice 파라미터
|
||
rate: float = 1.0 # 말 속도(speed)
|
||
instructions: Optional[str] = None # gpt-4o-mini-tts 표현 지시(선택)
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class TranscriptResult:
|
||
"""STT 결과."""
|
||
|
||
text: str
|
||
language: Optional[str] = None
|
||
model: str = STT_MODEL
|
||
duration: Optional[float] = None
|
||
provider_events: list[dict[str, object]] = field(default_factory=list)
|
||
words: list["TranscriptWord"] = field(default_factory=list)
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class TranscriptWord:
|
||
"""Provider word timing kept in process until it is privacy-safe hashed."""
|
||
|
||
word: str
|
||
start: float
|
||
end: float
|
||
confidence: float | None = None
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class StreamingTranscriptEvent:
|
||
"""Provider-neutral live transcript update emitted by streaming STT."""
|
||
|
||
text: str
|
||
final: bool
|
||
speech_final: bool
|
||
confidence: float | None = None
|
||
|
||
|
||
class DeepgramStreamingSession:
|
||
"""One Deepgram Listen WebSocket, scoped to exactly one learner utterance."""
|
||
|
||
def __init__(
|
||
self,
|
||
socket: Any,
|
||
*,
|
||
model: str,
|
||
language: str,
|
||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||
keepalive_seconds: float,
|
||
finalize_timeout_seconds: float,
|
||
) -> None:
|
||
self._socket = socket
|
||
self._model = model
|
||
self._language = language
|
||
self._on_event = on_event
|
||
self._keepalive_seconds = keepalive_seconds
|
||
self._finalize_timeout_seconds = finalize_timeout_seconds
|
||
self._send_lock = asyncio.Lock()
|
||
self._last_audio_sent_at = time.monotonic()
|
||
self._final_segments: list[str] = []
|
||
self._words: list[TranscriptWord] = []
|
||
self._provider_events: list[dict[str, object]] = []
|
||
self._duration: float | None = None
|
||
self._error: RuntimeError | None = None
|
||
self._finishing = False
|
||
self._closed = False
|
||
self._runtime_closed = False
|
||
voice_runtime_metrics.streaming_provider_opened()
|
||
self._receiver_task = asyncio.create_task(self._receive())
|
||
self._keepalive_task = asyncio.create_task(self._keepalive())
|
||
|
||
async def send_audio(self, audio: bytes) -> None:
|
||
if not audio:
|
||
return
|
||
if self._error is not None:
|
||
raise self._error
|
||
if self._closed or self._receiver_task.done():
|
||
if self._error is not None:
|
||
raise self._error
|
||
raise RuntimeError("Deepgram streaming STT connection closed")
|
||
try:
|
||
async with self._send_lock:
|
||
await self._socket.send(audio)
|
||
self._last_audio_sent_at = time.monotonic()
|
||
except Exception as exc:
|
||
raise RuntimeError("Deepgram streaming STT transport failed") from exc
|
||
|
||
async def finish(self) -> TranscriptResult:
|
||
"""Flush remaining audio with CloseStream and await final Results/Metadata."""
|
||
|
||
if self._closed:
|
||
if self._error is not None:
|
||
raise self._error
|
||
return self._result()
|
||
self._finishing = True
|
||
self._keepalive_task.cancel()
|
||
try:
|
||
async with self._send_lock:
|
||
await self._socket.send(json.dumps({"type": "CloseStream"}))
|
||
await asyncio.wait_for(
|
||
asyncio.shield(self._receiver_task),
|
||
timeout=self._finalize_timeout_seconds,
|
||
)
|
||
except TimeoutError as exc:
|
||
await self.abort()
|
||
raise RuntimeError("Deepgram streaming STT finalization timed out") from exc
|
||
except Exception as exc:
|
||
await self.abort()
|
||
if isinstance(exc, RuntimeError):
|
||
raise
|
||
raise RuntimeError("Deepgram streaming STT finalization failed") from exc
|
||
finally:
|
||
await self._cancel_keepalive()
|
||
self._closed = True
|
||
self._close_runtime_metrics(outcome="finalized")
|
||
if self._error is not None:
|
||
raise self._error
|
||
return self._result()
|
||
|
||
async def abort(self) -> None:
|
||
"""Close without asking the provider to process buffered audio."""
|
||
|
||
if self._closed:
|
||
return
|
||
self._closed = True
|
||
self._finishing = True
|
||
self._keepalive_task.cancel()
|
||
if not self._receiver_task.done():
|
||
self._receiver_task.cancel()
|
||
try:
|
||
await self._socket.close(code=1000, reason="utterance aborted")
|
||
except TypeError:
|
||
try:
|
||
await self._socket.close()
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
await self._cancel_keepalive()
|
||
if not self._receiver_task.done():
|
||
try:
|
||
await self._receiver_task
|
||
except (asyncio.CancelledError, Exception):
|
||
pass
|
||
self._close_runtime_metrics(outcome="aborted")
|
||
|
||
def _close_runtime_metrics(self, *, outcome: str) -> None:
|
||
if self._runtime_closed:
|
||
return
|
||
self._runtime_closed = True
|
||
voice_runtime_metrics.streaming_provider_closed(outcome=outcome)
|
||
|
||
async def _receive(self) -> None:
|
||
try:
|
||
async for raw in self._socket:
|
||
if not isinstance(raw, str):
|
||
continue
|
||
try:
|
||
payload = json.loads(raw)
|
||
except (json.JSONDecodeError, TypeError):
|
||
continue
|
||
if not isinstance(payload, dict):
|
||
continue
|
||
message_type = str(payload.get("type") or "")
|
||
if message_type == "Results":
|
||
await self._consume_results(payload)
|
||
elif message_type == "Metadata":
|
||
duration = _optional_float(payload.get("duration"))
|
||
if duration is not None and duration >= 0:
|
||
self._duration = max(self._duration or 0.0, duration)
|
||
elif message_type in {"Error", "Warning"}:
|
||
self._error = RuntimeError("Deepgram streaming STT provider failed")
|
||
return
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as exc:
|
||
self._error = RuntimeError("Deepgram streaming STT receive failed")
|
||
self._error.__cause__ = exc
|
||
|
||
async def _consume_results(self, payload: dict[str, object]) -> None:
|
||
channel = payload.get("channel")
|
||
if not isinstance(channel, dict):
|
||
return
|
||
alternatives = channel.get("alternatives")
|
||
if not isinstance(alternatives, list) or not alternatives:
|
||
return
|
||
alternative = alternatives[0]
|
||
if not isinstance(alternative, dict):
|
||
return
|
||
transcript = str(alternative.get("transcript") or "").strip()
|
||
is_final = bool(payload.get("is_final"))
|
||
speech_final = bool(payload.get("speech_final"))
|
||
confidence = _optional_float(alternative.get("confidence"))
|
||
start = max(0.0, _optional_float(payload.get("start")) or 0.0)
|
||
duration = max(0.0, _optional_float(payload.get("duration")) or 0.0)
|
||
if duration:
|
||
self._duration = max(self._duration or 0.0, start + duration)
|
||
|
||
if is_final and transcript:
|
||
self._final_segments.append(transcript)
|
||
self._consume_final_words(alternative.get("words"))
|
||
|
||
display_parts = list(self._final_segments)
|
||
if transcript and not is_final:
|
||
display_parts.append(transcript)
|
||
display_text = " ".join(part for part in display_parts if part).strip()
|
||
if not display_text and not speech_final:
|
||
return
|
||
|
||
event_type = "speech_final" if speech_final else (
|
||
"speech_end" if is_final else "voice_activity"
|
||
)
|
||
provider_event: dict[str, object] = {
|
||
"type": event_type,
|
||
"provider": "deepgram",
|
||
"source": "streaming_stt",
|
||
"start_ms": round(start * 1000),
|
||
"duration_ms": round(duration * 1000),
|
||
"is_final": is_final,
|
||
}
|
||
if confidence is not None:
|
||
provider_event["confidence"] = confidence
|
||
if is_final or speech_final:
|
||
self._provider_events.append(provider_event)
|
||
await self._on_event(
|
||
StreamingTranscriptEvent(
|
||
text=display_text,
|
||
final=is_final,
|
||
speech_final=speech_final,
|
||
confidence=confidence,
|
||
)
|
||
)
|
||
|
||
def _consume_final_words(self, value: object) -> None:
|
||
if not isinstance(value, list):
|
||
return
|
||
for item in value:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
word = str(item.get("punctuated_word") or item.get("word") or "").strip()
|
||
start = _optional_float(item.get("start"))
|
||
end = _optional_float(item.get("end"))
|
||
if not word or start is None or end is None or end <= start:
|
||
continue
|
||
confidence = _optional_float(item.get("confidence"))
|
||
self._words.append(
|
||
TranscriptWord(
|
||
word=word,
|
||
start=max(0.0, start),
|
||
end=max(0.0, end),
|
||
confidence=confidence,
|
||
)
|
||
)
|
||
timing_event: dict[str, object] = {
|
||
"type": "stt_word",
|
||
"provider": "deepgram",
|
||
"source": "stt_word_timestamps",
|
||
"start_ms": round(start * 1000),
|
||
"end_ms": round(end * 1000),
|
||
"is_final": True,
|
||
}
|
||
if confidence is not None:
|
||
timing_event["confidence"] = confidence
|
||
self._provider_events.append(timing_event)
|
||
self._duration = max(self._duration or 0.0, end)
|
||
|
||
async def _keepalive(self) -> None:
|
||
try:
|
||
while not self._finishing and not self._closed:
|
||
await asyncio.sleep(self._keepalive_seconds)
|
||
idle_for = time.monotonic() - self._last_audio_sent_at
|
||
if idle_for < self._keepalive_seconds:
|
||
continue
|
||
async with self._send_lock:
|
||
await self._socket.send(json.dumps({"type": "KeepAlive"}))
|
||
except asyncio.CancelledError:
|
||
return
|
||
except Exception as exc:
|
||
if not self._finishing:
|
||
self._error = RuntimeError("Deepgram streaming STT keepalive failed")
|
||
self._error.__cause__ = exc
|
||
|
||
async def _cancel_keepalive(self) -> None:
|
||
if self._keepalive_task.done():
|
||
return
|
||
self._keepalive_task.cancel()
|
||
try:
|
||
await self._keepalive_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
def _result(self) -> TranscriptResult:
|
||
return TranscriptResult(
|
||
text=" ".join(self._final_segments).strip(),
|
||
language=self._language,
|
||
model=self._model,
|
||
duration=self._duration,
|
||
provider_events=list(self._provider_events),
|
||
words=list(self._words),
|
||
)
|
||
|
||
|
||
class LocalWhisperStreamingSession:
|
||
"""One loopback faster-whisper stream, scoped to exactly one learner utterance.
|
||
|
||
Same public surface as the Deepgram session so the WebSocket route does not
|
||
branch on provider. Audio never leaves the host: the sidecar keeps only an
|
||
in-memory utterance buffer and drops it when the utterance ends.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
socket: Any,
|
||
*,
|
||
model: str,
|
||
language: str,
|
||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||
finalize_timeout_seconds: float,
|
||
) -> None:
|
||
self._socket = socket
|
||
self._model = model
|
||
self._language = language
|
||
self._on_event = on_event
|
||
self._finalize_timeout_seconds = finalize_timeout_seconds
|
||
self._send_lock = asyncio.Lock()
|
||
self._final_segments: list[str] = []
|
||
self._words: list[TranscriptWord] = []
|
||
self._provider_events: list[dict[str, object]] = []
|
||
self._duration: float | None = None
|
||
# 확정된 발화들의 누적 길이. interim 은 같은 발화라 오프셋을 밀지 않는다.
|
||
self._utterance_offset = 0.0
|
||
self._error: RuntimeError | None = None
|
||
self._finishing = False
|
||
self._closed = False
|
||
self._runtime_closed = False
|
||
voice_runtime_metrics.streaming_provider_opened()
|
||
self._receiver_task = asyncio.create_task(self._receive())
|
||
|
||
async def send_audio(self, audio: bytes) -> None:
|
||
if not audio:
|
||
return
|
||
if self._error is not None:
|
||
raise self._error
|
||
if self._closed or self._receiver_task.done():
|
||
if self._error is not None:
|
||
raise self._error
|
||
raise RuntimeError("Local whisper streaming STT connection closed")
|
||
try:
|
||
async with self._send_lock:
|
||
await self._socket.send(audio)
|
||
except Exception as exc:
|
||
self._error = RuntimeError("Local whisper streaming STT transport failed")
|
||
self._error.__cause__ = exc
|
||
raise self._error from exc
|
||
|
||
async def finish(self) -> TranscriptResult:
|
||
if self._closed:
|
||
if self._error is not None:
|
||
raise self._error
|
||
return self._result()
|
||
self._finishing = True
|
||
try:
|
||
async with self._send_lock:
|
||
await self._socket.send(json.dumps({"type": "CloseStream"}))
|
||
await asyncio.wait_for(
|
||
asyncio.shield(self._receiver_task),
|
||
timeout=self._finalize_timeout_seconds,
|
||
)
|
||
except TimeoutError as exc:
|
||
await self.abort()
|
||
raise RuntimeError(
|
||
"Local whisper streaming STT finalization timed out"
|
||
) from exc
|
||
except Exception as exc:
|
||
await self.abort()
|
||
if isinstance(exc, RuntimeError):
|
||
raise
|
||
raise RuntimeError(
|
||
"Local whisper streaming STT finalization failed"
|
||
) from exc
|
||
self._closed = True
|
||
self._close_runtime_metrics(outcome="finalized")
|
||
if self._error is not None:
|
||
raise self._error
|
||
return self._result()
|
||
|
||
async def abort(self) -> None:
|
||
if self._closed:
|
||
return
|
||
self._closed = True
|
||
self._finishing = True
|
||
if not self._receiver_task.done():
|
||
self._receiver_task.cancel()
|
||
try:
|
||
await self._socket.close(code=1000, reason="utterance aborted")
|
||
except TypeError:
|
||
try:
|
||
await self._socket.close()
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
if not self._receiver_task.done():
|
||
try:
|
||
await self._receiver_task
|
||
except (asyncio.CancelledError, Exception):
|
||
pass
|
||
self._close_runtime_metrics(outcome="aborted")
|
||
|
||
def _close_runtime_metrics(self, *, outcome: str) -> None:
|
||
if self._runtime_closed:
|
||
return
|
||
self._runtime_closed = True
|
||
voice_runtime_metrics.streaming_provider_closed(outcome=outcome)
|
||
|
||
async def _receive(self) -> None:
|
||
try:
|
||
async for message in self._socket:
|
||
if isinstance(message, (bytes, bytearray)):
|
||
continue
|
||
try:
|
||
payload = json.loads(message)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if not isinstance(payload, dict):
|
||
continue
|
||
kind = str(payload.get("type") or "")
|
||
if kind == "error":
|
||
self._error = RuntimeError(
|
||
"Local whisper streaming STT provider failed"
|
||
)
|
||
return
|
||
if kind == "transcript":
|
||
await self._consume_transcript(payload)
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as exc:
|
||
if not self._finishing:
|
||
self._error = RuntimeError("Local whisper streaming STT receive failed")
|
||
self._error.__cause__ = exc
|
||
|
||
async def _consume_transcript(self, payload: dict[str, object]) -> None:
|
||
transcript = str(payload.get("text") or "").strip()
|
||
is_final = bool(payload.get("is_final"))
|
||
speech_final = bool(payload.get("speech_final"))
|
||
confidence = _optional_float(payload.get("confidence"))
|
||
duration = max(0.0, _optional_float(payload.get("duration")) or 0.0)
|
||
start = self._utterance_offset
|
||
if duration:
|
||
self._duration = max(self._duration or 0.0, start + duration)
|
||
|
||
if is_final and transcript:
|
||
self._final_segments.append(transcript)
|
||
self._consume_final_words(payload.get("words"), offset=start)
|
||
if is_final:
|
||
# 다음 발화는 이 발화가 끝난 지점부터 시작한다.
|
||
self._utterance_offset = start + duration
|
||
|
||
display_parts = list(self._final_segments)
|
||
if transcript and not is_final:
|
||
display_parts.append(transcript)
|
||
display_text = " ".join(part for part in display_parts if part).strip()
|
||
if not display_text and not speech_final:
|
||
return
|
||
|
||
event_type = "speech_final" if speech_final else (
|
||
"speech_end" if is_final else "voice_activity"
|
||
)
|
||
provider_event: dict[str, object] = {
|
||
"type": event_type,
|
||
"provider": "local_whisper",
|
||
"source": "streaming_stt",
|
||
"start_ms": round(start * 1000),
|
||
"duration_ms": round(duration * 1000),
|
||
"is_final": is_final,
|
||
}
|
||
if confidence is not None:
|
||
provider_event["confidence"] = confidence
|
||
if is_final or speech_final:
|
||
self._provider_events.append(provider_event)
|
||
await self._on_event(
|
||
StreamingTranscriptEvent(
|
||
text=display_text,
|
||
final=is_final,
|
||
speech_final=speech_final,
|
||
confidence=confidence,
|
||
)
|
||
)
|
||
|
||
def _consume_final_words(self, value: object, *, offset: float) -> None:
|
||
if not isinstance(value, list):
|
||
return
|
||
for item in value:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
word = str(item.get("word") or "").strip()
|
||
start = _optional_float(item.get("start"))
|
||
end = _optional_float(item.get("end"))
|
||
if not word or start is None or end is None or end <= start:
|
||
continue
|
||
confidence = _optional_float(item.get("confidence"))
|
||
absolute_start = max(0.0, offset + start)
|
||
absolute_end = max(absolute_start, offset + end)
|
||
self._words.append(
|
||
TranscriptWord(
|
||
word=word,
|
||
start=absolute_start,
|
||
end=absolute_end,
|
||
confidence=confidence,
|
||
)
|
||
)
|
||
timing_event: dict[str, object] = {
|
||
"type": "stt_word",
|
||
"provider": "local_whisper",
|
||
"source": "stt_word_timestamps",
|
||
"start_ms": round(absolute_start * 1000),
|
||
"end_ms": round(absolute_end * 1000),
|
||
"is_final": True,
|
||
}
|
||
if confidence is not None:
|
||
timing_event["confidence"] = confidence
|
||
self._provider_events.append(timing_event)
|
||
self._duration = max(self._duration or 0.0, absolute_end)
|
||
|
||
def _result(self) -> TranscriptResult:
|
||
return TranscriptResult(
|
||
text=" ".join(self._final_segments).strip(),
|
||
language=self._language,
|
||
model=self._model,
|
||
duration=self._duration,
|
||
provider_events=list(self._provider_events),
|
||
words=list(self._words),
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class EndOfTurnDecision:
|
||
"""Provider-neutral readiness signal for a completed learner utterance."""
|
||
|
||
ready: bool
|
||
transcript_ready: bool
|
||
silence_ready: bool
|
||
silence_ms: int
|
||
threshold_ms: int
|
||
reason: str
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class TTSChunk:
|
||
"""TTS 스트림 1청크(오디오 바이트). 립싱크는 프론트 Web Audio AnalyserNode가 자체 산출."""
|
||
|
||
audio: bytes
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# voice preset 해석 (페르소나 → OpenAI 파라미터)
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
def resolve_voice(
|
||
*,
|
||
persona_code: Optional[str] = None,
|
||
preset: Optional[str] = None,
|
||
instructions: Optional[str] = None,
|
||
) -> VoicePreset:
|
||
"""페르소나 code 또는 명시 preset → OpenAI voice 파라미터로 해석.
|
||
|
||
우선순위: 명시 preset > persona_code 기본 preset > 'neutral'.
|
||
알 수 없는 preset 은 DEFAULT_OPENAI_VOICE 로 안전 폴백(크래시 없음).
|
||
"""
|
||
chosen = preset
|
||
if not chosen and persona_code:
|
||
chosen = PERSONA_CODE_TO_PRESET.get(persona_code.upper())
|
||
if not chosen:
|
||
chosen = "neutral"
|
||
|
||
openai_voice = PRESET_TO_OPENAI_VOICE.get(chosen, DEFAULT_OPENAI_VOICE)
|
||
if openai_voice not in _OPENAI_VOICES:
|
||
openai_voice = DEFAULT_OPENAI_VOICE
|
||
rate = PRESET_RATE.get(chosen, 1.0)
|
||
return VoicePreset(
|
||
preset=chosen,
|
||
openai_voice=openai_voice,
|
||
rate=rate,
|
||
instructions=instructions,
|
||
)
|
||
|
||
|
||
def resolve_voice_from_map(
|
||
*,
|
||
provider: str,
|
||
voice_id: str,
|
||
base_params: Mapping[str, Any] | None,
|
||
persona_code: Optional[str] = None,
|
||
) -> VoicePreset | None:
|
||
"""DB persona_voice_map row -> live OpenAI VoicePreset.
|
||
|
||
provider-agnostic rows are allowed in the catalog, but this service only
|
||
knows how to send OpenAI TTS. Unsupported providers return None so callers
|
||
can fall back to the existing preset resolver.
|
||
"""
|
||
if provider.strip().lower() != "openai":
|
||
return None
|
||
|
||
fallback = resolve_voice(persona_code=persona_code)
|
||
params = dict(base_params or {})
|
||
preset = _clean_optional_text(params.get("preset")) or fallback.preset
|
||
mapped_voice = _clean_optional_text(params.get("openai_voice"))
|
||
voice_id_value = _clean_optional_text(voice_id)
|
||
if not mapped_voice and voice_id_value in _OPENAI_VOICES:
|
||
mapped_voice = voice_id_value
|
||
if not mapped_voice:
|
||
mapped_voice = PRESET_TO_OPENAI_VOICE.get(preset, fallback.openai_voice)
|
||
if mapped_voice not in _OPENAI_VOICES:
|
||
mapped_voice = fallback.openai_voice
|
||
if mapped_voice not in _OPENAI_VOICES:
|
||
mapped_voice = DEFAULT_OPENAI_VOICE
|
||
|
||
rate = PRESET_RATE.get(preset, fallback.rate)
|
||
if "rate" in params:
|
||
try:
|
||
rate = float(params["rate"])
|
||
except (TypeError, ValueError):
|
||
rate = fallback.rate
|
||
|
||
return VoicePreset(
|
||
preset=preset,
|
||
openai_voice=mapped_voice,
|
||
rate=rate,
|
||
instructions=_clean_optional_text(params.get("instructions")),
|
||
)
|
||
|
||
|
||
def build_higgs_prompt(text: str, voice: VoicePreset) -> str:
|
||
"""합성 seed의 화자 정체성을 지키면서 감정·속도 태그를 첫 단어 뒤에 넣는다."""
|
||
normalized = text.casefold()
|
||
tags = ""
|
||
if voice.preset == POC_SAMPLE_TTS_PRESET:
|
||
for keywords, candidate in _HIGGS_DELIVERY_TAGS:
|
||
if any(keyword.casefold() in normalized for keyword in keywords):
|
||
tags = candidate
|
||
break
|
||
if not tags:
|
||
tags = (
|
||
"<|emotion:helplessness|><|prosody:speed_slow|>"
|
||
"<|prosody:expressive_low|>"
|
||
)
|
||
elif voice.rate <= 0.85:
|
||
tags = "<|prosody:speed_slow|>"
|
||
elif voice.rate >= 1.15:
|
||
tags = "<|prosody:speed_fast|>"
|
||
if not tags:
|
||
return text
|
||
|
||
# Higgs 강한 감정 태그를 맨 앞에 두면 reference 화자가 흔들릴 수 있다. 첫 단어로
|
||
# 화자를 먼저 고정한 뒤 태그 다음 단어를 공백 없이 이어 붙인다.
|
||
match = re.match(r"^(\S+\s+)(.+)$", text, flags=re.DOTALL)
|
||
if match:
|
||
return f"{match.group(1)}{tags}{match.group(2).lstrip()}"
|
||
return tags + text
|
||
|
||
|
||
# 비언어 지문 패턴: (…)·（…）·[…]·【…】. 내담자 발화의 무대지시(고개 끄덕/한숨/침묵 등).
|
||
_STAGE_DIRECTION_RE = re.compile(r"[\(（\[【][^\)）\]】]*[\)）\]】]")
|
||
|
||
|
||
def speakable_text(text: str) -> str:
|
||
"""TTS로 읽을 텍스트만 남긴다 — 비언어 지문((고개 살짝 끄덕)·（한숨）·[침묵])을 제거.
|
||
|
||
지문은 자막/회기리뷰에 남고 아바타 애니메이션이 표현하며, 음성으로는 읽지 않는다.
|
||
지문만으로 이뤄진 발화(예: "(침묵)")는 빈 문자열을 반환 → 합성 생략.
|
||
"""
|
||
if not text:
|
||
return ""
|
||
stripped = _STAGE_DIRECTION_RE.sub(" ", text)
|
||
# 말줄임표/중복 공백 정리 + 고아 구두점 앞 공백 제거
|
||
stripped = re.sub(r"\s+", " ", stripped)
|
||
stripped = re.sub(r"\s+([,.!?…」』】)])", r"\1", stripped)
|
||
return stripped.strip()
|
||
|
||
|
||
def build_tts_payload(
|
||
text: str,
|
||
voice: VoicePreset,
|
||
*,
|
||
model: str = TTS_MODEL,
|
||
response_format: str = TTS_RESPONSE_FORMAT,
|
||
) -> dict[str, object]:
|
||
"""Build the deterministic OpenAI TTS payload for a resolved voice preset."""
|
||
payload: dict[str, object] = {
|
||
"model": model,
|
||
"voice": voice.openai_voice,
|
||
"input": text,
|
||
"response_format": response_format,
|
||
"speed": _clamp_speed(voice.rate),
|
||
}
|
||
if voice.instructions and model.startswith("gpt-4o"):
|
||
payload["instructions"] = voice.instructions
|
||
return payload
|
||
|
||
|
||
def assess_end_of_turn(
|
||
*,
|
||
transcript_text: Optional[str],
|
||
transcript_final: bool,
|
||
silence_ms: Optional[int],
|
||
silence_threshold_ms: int = EOT_SILENCE_THRESHOLD_MS,
|
||
) -> EndOfTurnDecision:
|
||
"""Return whether final STT text plus observed silence is enough to run a turn."""
|
||
observed_silence = _nonnegative_int(silence_ms)
|
||
threshold = max(0, _nonnegative_int(silence_threshold_ms))
|
||
has_text = bool((transcript_text or "").strip())
|
||
transcript_ready = bool(transcript_final and has_text)
|
||
silence_ready = observed_silence >= threshold
|
||
ready = transcript_ready and silence_ready
|
||
if ready:
|
||
reason = "ready"
|
||
elif not has_text:
|
||
reason = "empty_transcript"
|
||
elif not transcript_final:
|
||
reason = "final_transcript_pending"
|
||
else:
|
||
reason = "silence_threshold_pending"
|
||
return EndOfTurnDecision(
|
||
ready=ready,
|
||
transcript_ready=transcript_ready,
|
||
silence_ready=silence_ready,
|
||
silence_ms=observed_silence,
|
||
threshold_ms=threshold,
|
||
reason=reason,
|
||
)
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# OpenAI 음성 서비스
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
class VoiceService:
|
||
"""Deepgram/OpenAI STT와 선택형 OpenAI/Higgs TTS 어댑터."""
|
||
|
||
def __init__(
|
||
self,
|
||
api_key: Optional[str] = None,
|
||
base_url: Optional[str] = None,
|
||
*,
|
||
poc_sample_tts_enabled: Optional[bool] = None,
|
||
environment: Optional[str] = None,
|
||
poc_sample_tts_dir: Optional[str | Path] = None,
|
||
tts_provider: Optional[str] = None,
|
||
higgs_base_url: Optional[str] = None,
|
||
higgs_timeout_seconds: Optional[float] = None,
|
||
stt_provider: Optional[str] = None,
|
||
deepgram_api_key: Optional[str] = None,
|
||
deepgram_stt_url: Optional[str] = None,
|
||
deepgram_stt_model: Optional[str] = None,
|
||
deepgram_stt_language: Optional[str] = None,
|
||
deepgram_endpointing_ms: Optional[int] = None,
|
||
deepgram_utterance_end_ms: Optional[int] = None,
|
||
deepgram_keepalive_seconds: Optional[float] = None,
|
||
deepgram_finalize_timeout_seconds: Optional[float] = None,
|
||
deepgram_mip_opt_out: Optional[bool] = None,
|
||
deepgram_connect: Optional[Callable[..., Awaitable[Any]]] = None,
|
||
local_whisper_stt_url: Optional[str] = None,
|
||
local_whisper_stt_model: Optional[str] = None,
|
||
local_whisper_stt_language: Optional[str] = None,
|
||
local_whisper_endpointing_ms: Optional[int] = None,
|
||
local_whisper_utterance_end_ms: Optional[int] = None,
|
||
local_whisper_finalize_timeout_seconds: Optional[float] = None,
|
||
local_whisper_connect: Optional[Callable[..., Awaitable[Any]]] = None,
|
||
melotts_base_url: Optional[str] = None,
|
||
melotts_timeout_seconds: Optional[float] = None,
|
||
) -> None:
|
||
self._api_key = (api_key if api_key is not None else settings.openai_api_key) or ""
|
||
self._base_url = (base_url or settings.openai_base_url or OPENAI_BASE_URL).rstrip("/")
|
||
self._environment = environment if environment is not None else settings.environment
|
||
self._stt_provider = (
|
||
stt_provider if stt_provider is not None else settings.voice_stt_provider
|
||
).strip().lower()
|
||
configured_deepgram_key = settings.deepgram_api_key.get_secret_value()
|
||
self._deepgram_api_key = (
|
||
deepgram_api_key
|
||
if deepgram_api_key is not None
|
||
else configured_deepgram_key
|
||
).strip()
|
||
self._deepgram_stt_url = (
|
||
deepgram_stt_url
|
||
if deepgram_stt_url is not None
|
||
else settings.deepgram_stt_url
|
||
).rstrip("?")
|
||
self._deepgram_stt_model = (
|
||
deepgram_stt_model
|
||
if deepgram_stt_model is not None
|
||
else settings.deepgram_stt_model
|
||
).strip() or DEEPGRAM_STT_MODEL
|
||
self._deepgram_stt_language = (
|
||
deepgram_stt_language
|
||
if deepgram_stt_language is not None
|
||
else settings.deepgram_stt_language
|
||
).strip() or STT_LANGUAGE
|
||
self._deepgram_endpointing_ms = max(
|
||
10,
|
||
int(
|
||
deepgram_endpointing_ms
|
||
if deepgram_endpointing_ms is not None
|
||
else settings.deepgram_endpointing_ms
|
||
),
|
||
)
|
||
self._deepgram_utterance_end_ms = max(
|
||
1000,
|
||
int(
|
||
deepgram_utterance_end_ms
|
||
if deepgram_utterance_end_ms is not None
|
||
else settings.deepgram_utterance_end_ms
|
||
),
|
||
)
|
||
self._deepgram_keepalive_seconds = max(
|
||
1.0,
|
||
float(
|
||
deepgram_keepalive_seconds
|
||
if deepgram_keepalive_seconds is not None
|
||
else settings.deepgram_keepalive_seconds
|
||
),
|
||
)
|
||
self._deepgram_finalize_timeout_seconds = max(
|
||
1.0,
|
||
float(
|
||
deepgram_finalize_timeout_seconds
|
||
if deepgram_finalize_timeout_seconds is not None
|
||
else settings.deepgram_finalize_timeout_seconds
|
||
),
|
||
)
|
||
self._deepgram_mip_opt_out = (
|
||
bool(settings.deepgram_mip_opt_out)
|
||
if deepgram_mip_opt_out is None
|
||
else bool(deepgram_mip_opt_out)
|
||
)
|
||
self._deepgram_connect = deepgram_connect or websocket_connect
|
||
self._local_whisper_stt_url = (
|
||
local_whisper_stt_url
|
||
if local_whisper_stt_url is not None
|
||
else settings.local_whisper_stt_url
|
||
).strip().rstrip("?")
|
||
self._local_whisper_stt_model = (
|
||
local_whisper_stt_model
|
||
if local_whisper_stt_model is not None
|
||
else settings.local_whisper_stt_model
|
||
).strip() or "small"
|
||
self._local_whisper_stt_language = (
|
||
local_whisper_stt_language
|
||
if local_whisper_stt_language is not None
|
||
else settings.local_whisper_stt_language
|
||
).strip() or STT_LANGUAGE
|
||
self._local_whisper_endpointing_ms = max(
|
||
10,
|
||
int(
|
||
local_whisper_endpointing_ms
|
||
if local_whisper_endpointing_ms is not None
|
||
else settings.local_whisper_endpointing_ms
|
||
),
|
||
)
|
||
self._local_whisper_utterance_end_ms = max(
|
||
1000,
|
||
int(
|
||
local_whisper_utterance_end_ms
|
||
if local_whisper_utterance_end_ms is not None
|
||
else settings.local_whisper_utterance_end_ms
|
||
),
|
||
)
|
||
self._local_whisper_finalize_timeout_seconds = max(
|
||
1.0,
|
||
float(
|
||
local_whisper_finalize_timeout_seconds
|
||
if local_whisper_finalize_timeout_seconds is not None
|
||
else settings.local_whisper_finalize_timeout_seconds
|
||
),
|
||
)
|
||
self._local_whisper_connect = local_whisper_connect or websocket_connect
|
||
self._melotts_base_url = (
|
||
melotts_base_url
|
||
if melotts_base_url is not None
|
||
else settings.melotts_tts_url
|
||
).rstrip("/")
|
||
self._melotts_timeout_seconds = max(
|
||
1.0,
|
||
float(
|
||
melotts_timeout_seconds
|
||
if melotts_timeout_seconds is not None
|
||
else settings.melotts_tts_timeout_seconds
|
||
),
|
||
)
|
||
self._melotts_client: httpx.AsyncClient | None = None
|
||
self._tts_provider = (
|
||
tts_provider if tts_provider is not None else settings.voice_tts_provider
|
||
).strip().lower()
|
||
self._higgs_base_url = (
|
||
higgs_base_url if higgs_base_url is not None else settings.higgs_tts_url
|
||
).rstrip("/")
|
||
self._higgs_timeout_seconds = max(
|
||
1.0,
|
||
float(
|
||
higgs_timeout_seconds
|
||
if higgs_timeout_seconds is not None
|
||
else settings.higgs_tts_timeout_seconds
|
||
),
|
||
)
|
||
self._poc_sample_tts_enabled = (
|
||
bool(settings.voice_poc_sample_tts_enabled)
|
||
if poc_sample_tts_enabled is None
|
||
else bool(poc_sample_tts_enabled)
|
||
)
|
||
sample_dir_value: str | Path = (
|
||
poc_sample_tts_dir
|
||
if poc_sample_tts_dir is not None
|
||
else (settings.voice_poc_sample_tts_dir or POC_SAMPLE_TTS_DEFAULT_DIR)
|
||
)
|
||
sample_dir = Path(sample_dir_value)
|
||
if not sample_dir.is_absolute():
|
||
sample_dir = repo_root() / sample_dir
|
||
self._poc_sample_tts_dir = sample_dir
|
||
self._client: Optional[httpx.AsyncClient] = None
|
||
self._higgs_client: Optional[httpx.AsyncClient] = None
|
||
|
||
# ── 수명주기 ──────────────────────────────────────────
|
||
async def startup(self) -> None:
|
||
if self._api_key:
|
||
self._client = httpx.AsyncClient(
|
||
base_url=self._base_url,
|
||
headers={"Authorization": f"Bearer {self._api_key}"},
|
||
timeout=httpx.Timeout(60.0, connect=10.0),
|
||
)
|
||
if self._higgs_enabled():
|
||
self._higgs_client = httpx.AsyncClient(
|
||
base_url=self._higgs_base_url,
|
||
timeout=httpx.Timeout(self._higgs_timeout_seconds, connect=3.0),
|
||
)
|
||
if self._melotts_enabled():
|
||
self._melotts_client = httpx.AsyncClient(
|
||
base_url=self._melotts_base_url,
|
||
timeout=httpx.Timeout(self._melotts_timeout_seconds, connect=3.0),
|
||
)
|
||
|
||
async def shutdown(self) -> None:
|
||
if self._client is not None:
|
||
await self._client.aclose()
|
||
self._client = None
|
||
if self._higgs_client is not None:
|
||
await self._higgs_client.aclose()
|
||
self._higgs_client = None
|
||
if self._melotts_client is not None:
|
||
await self._melotts_client.aclose()
|
||
self._melotts_client = None
|
||
|
||
def is_available(self) -> bool:
|
||
"""마이크 캐스케이드(STT+TTS) 전체 가용 여부."""
|
||
return self.stt_available() and self.tts_available()
|
||
|
||
def stt_available(self) -> bool:
|
||
return self.streaming_stt_enabled() or self.batch_stt_available()
|
||
|
||
def batch_stt_available(self) -> bool:
|
||
return bool(self._api_key)
|
||
|
||
def streaming_stt_enabled(self) -> bool:
|
||
if self._stt_provider == "deepgram":
|
||
return bool(self._deepgram_api_key)
|
||
if self._stt_provider == "local_whisper":
|
||
# 로컬 사이드카는 키가 없다. URL 설정만으로 활성화된다.
|
||
return bool(self._local_whisper_stt_url)
|
||
return False
|
||
|
||
def stt_provider(self) -> str:
|
||
if self.streaming_stt_enabled():
|
||
return self._stt_provider
|
||
if (
|
||
self._stt_provider in {"deepgram", "local_whisper"}
|
||
and self.batch_stt_available()
|
||
):
|
||
return "openai-batch-fallback"
|
||
if self.batch_stt_available():
|
||
return "openai"
|
||
return "unavailable"
|
||
|
||
def stt_model(self) -> str:
|
||
if self.streaming_stt_enabled():
|
||
if self._stt_provider == "local_whisper":
|
||
return self._local_whisper_stt_model
|
||
return self._deepgram_stt_model
|
||
return STT_MODEL
|
||
|
||
def can_stream_audio(
|
||
self,
|
||
*,
|
||
fmt: str | None,
|
||
sample_rate: int | None,
|
||
channels: int | None,
|
||
sample_width: int | None,
|
||
) -> bool:
|
||
normalized = (fmt or "").strip().lower()
|
||
return (
|
||
self.streaming_stt_enabled()
|
||
and normalized in {"pcm", "s16le", "linear16", "audio/pcm"}
|
||
and sample_width == 2
|
||
and sample_rate is not None
|
||
and 8000 <= sample_rate <= 192000
|
||
and channels in {1, 2}
|
||
)
|
||
|
||
async def open_streaming_transcription(
|
||
self,
|
||
*,
|
||
fmt: str | None,
|
||
sample_rate: int | None,
|
||
channels: int | None,
|
||
sample_width: int | None,
|
||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||
) -> DeepgramStreamingSession | LocalWhisperStreamingSession:
|
||
if not self.can_stream_audio(
|
||
fmt=fmt,
|
||
sample_rate=sample_rate,
|
||
channels=channels,
|
||
sample_width=sample_width,
|
||
):
|
||
raise VoiceUnavailable(
|
||
"streaming STT requires linear16 PCM metadata"
|
||
)
|
||
assert sample_rate is not None and channels is not None
|
||
if self._stt_provider == "local_whisper":
|
||
return await self._open_local_whisper_transcription(
|
||
sample_rate=sample_rate, channels=channels, on_event=on_event
|
||
)
|
||
query = urlencode(
|
||
{
|
||
"model": self._deepgram_stt_model,
|
||
"language": self._deepgram_stt_language,
|
||
"encoding": "linear16",
|
||
"sample_rate": sample_rate,
|
||
"channels": channels,
|
||
"interim_results": "true",
|
||
"punctuate": "true",
|
||
"smart_format": "true",
|
||
"vad_events": "true",
|
||
"endpointing": self._deepgram_endpointing_ms,
|
||
"utterance_end_ms": self._deepgram_utterance_end_ms,
|
||
"mip_opt_out": "true" if self._deepgram_mip_opt_out else "false",
|
||
}
|
||
)
|
||
url = f"{self._deepgram_stt_url}?{query}"
|
||
try:
|
||
socket = await self._deepgram_connect(
|
||
url,
|
||
additional_headers={
|
||
"Authorization": f"Token {self._deepgram_api_key}",
|
||
},
|
||
open_timeout=10,
|
||
close_timeout=5,
|
||
ping_interval=20,
|
||
ping_timeout=20,
|
||
max_size=2 * 1024 * 1024,
|
||
max_queue=16,
|
||
write_limit=64 * 1024,
|
||
)
|
||
except Exception as exc:
|
||
raise RuntimeError("Deepgram streaming STT connection failed") from exc
|
||
return DeepgramStreamingSession(
|
||
socket,
|
||
model=self._deepgram_stt_model,
|
||
language=self._deepgram_stt_language,
|
||
on_event=on_event,
|
||
keepalive_seconds=self._deepgram_keepalive_seconds,
|
||
finalize_timeout_seconds=self._deepgram_finalize_timeout_seconds,
|
||
)
|
||
|
||
async def _open_local_whisper_transcription(
|
||
self,
|
||
*,
|
||
sample_rate: int,
|
||
channels: int,
|
||
on_event: Callable[[StreamingTranscriptEvent], Awaitable[None]],
|
||
) -> LocalWhisperStreamingSession:
|
||
query = urlencode(
|
||
{
|
||
"model": self._local_whisper_stt_model,
|
||
"language": self._local_whisper_stt_language,
|
||
"sample_rate": sample_rate,
|
||
"channels": channels,
|
||
"endpointing": self._local_whisper_endpointing_ms,
|
||
"utterance_end_ms": self._local_whisper_utterance_end_ms,
|
||
}
|
||
)
|
||
url = f"{self._local_whisper_stt_url}?{query}"
|
||
try:
|
||
socket = await self._local_whisper_connect(
|
||
url,
|
||
open_timeout=10,
|
||
close_timeout=5,
|
||
ping_interval=20,
|
||
ping_timeout=20,
|
||
max_size=2 * 1024 * 1024,
|
||
max_queue=16,
|
||
write_limit=64 * 1024,
|
||
)
|
||
except Exception as exc:
|
||
raise RuntimeError(
|
||
"Local whisper streaming STT connection failed"
|
||
) from exc
|
||
return LocalWhisperStreamingSession(
|
||
socket,
|
||
model=self._local_whisper_stt_model,
|
||
language=self._local_whisper_stt_language,
|
||
on_event=on_event,
|
||
finalize_timeout_seconds=self._local_whisper_finalize_timeout_seconds,
|
||
)
|
||
|
||
def tts_available(self, voice: VoicePreset | None = None) -> bool:
|
||
if self._melotts_enabled():
|
||
return True
|
||
if self._higgs_enabled() and (voice is None or self._should_use_higgs_tts(voice)):
|
||
return True
|
||
if self._poc_sample_tts_available():
|
||
return True
|
||
return bool(self._api_key)
|
||
|
||
def tts_provider(self) -> str:
|
||
if self._melotts_enabled():
|
||
return "melotts"
|
||
if self._higgs_enabled():
|
||
return "higgs"
|
||
if self._poc_sample_tts_available():
|
||
return "p1-sample-poc"
|
||
if self._tts_provider == "higgs" and self._environment != "dev":
|
||
return "disabled-non-dev"
|
||
if self._api_key:
|
||
return "openai"
|
||
if self._poc_sample_tts_enabled and self._environment != "dev":
|
||
return "disabled-non-dev"
|
||
return "unavailable"
|
||
|
||
def tts_provider_for_voice(self, voice: VoicePreset) -> str:
|
||
if self._should_use_melotts_tts(voice):
|
||
return "melotts"
|
||
if self._should_use_higgs_tts(voice):
|
||
return "higgs"
|
||
if self._should_use_poc_sample_tts(voice):
|
||
return "p1-sample-poc"
|
||
return "openai" if self._api_key else "unavailable"
|
||
|
||
def tts_model_for_voice(self, voice: VoicePreset) -> str:
|
||
if self._should_use_melotts_tts(voice):
|
||
return MELOTTS_TTS_MODEL
|
||
return HIGGS_TTS_MODEL if self._should_use_higgs_tts(voice) else TTS_MODEL
|
||
|
||
def tts_media_type_for_voice(self, voice: VoicePreset) -> str:
|
||
if self._should_use_melotts_tts(voice) or self._should_use_higgs_tts(voice):
|
||
return "audio/wav"
|
||
return "audio/mpeg"
|
||
|
||
def _higgs_enabled(self) -> bool:
|
||
return self._tts_provider == "higgs" and self._environment == "dev"
|
||
|
||
def _melotts_enabled(self) -> bool:
|
||
# MIT 라이선스라 환경 제한이 없다. Higgs 와 달리 운영에서도 쓸 수 있다.
|
||
return self._tts_provider == "melotts" and bool(self._melotts_base_url)
|
||
|
||
def _should_use_melotts_tts(self, voice: VoicePreset) -> bool:
|
||
# 사전학습 다화자 모델이라 프리셋별 reference 제약이 없다.
|
||
del voice
|
||
return self._melotts_enabled()
|
||
|
||
def _should_use_higgs_tts(self, voice: VoicePreset) -> bool:
|
||
# 현재 권리 안전한 synthetic reference는 P1 서연 프리셋만 보유한다.
|
||
return self._higgs_enabled() and voice.preset == POC_SAMPLE_TTS_PRESET
|
||
|
||
def poc_sample_tts_available(self) -> bool:
|
||
return self._poc_sample_tts_available()
|
||
|
||
def _poc_sample_tts_available(self) -> bool:
|
||
return (
|
||
self._poc_sample_tts_enabled
|
||
and self._environment == "dev"
|
||
and self._poc_sample_path(_POC_SAMPLE_TTS_DEFAULT_SAMPLE).is_file()
|
||
)
|
||
|
||
def _should_use_poc_sample_tts(self, voice: VoicePreset) -> bool:
|
||
return (
|
||
self._poc_sample_tts_enabled
|
||
and self._environment == "dev"
|
||
and voice.preset == POC_SAMPLE_TTS_PRESET
|
||
)
|
||
|
||
def _poc_sample_path(self, sample_id: str) -> Path:
|
||
return self._poc_sample_tts_dir / f"{sample_id}.mp3"
|
||
|
||
@property
|
||
def _http(self) -> httpx.AsyncClient:
|
||
if not self._api_key:
|
||
raise VoiceUnavailable("OPENAI_API_KEY 미설정 — 음성 기능 degraded")
|
||
if self._client is None:
|
||
# lazy 보강(테스트/지연 startup 대비)
|
||
self._client = httpx.AsyncClient(
|
||
base_url=self._base_url,
|
||
headers={"Authorization": f"Bearer {self._api_key}"},
|
||
timeout=httpx.Timeout(60.0, connect=10.0),
|
||
)
|
||
return self._client
|
||
|
||
@property
|
||
def _higgs_http(self) -> httpx.AsyncClient:
|
||
if not self._higgs_enabled():
|
||
raise VoiceUnavailable("Higgs TTS는 로컬 dev 환경에서만 사용할 수 있습니다.")
|
||
if self._higgs_client is None:
|
||
self._higgs_client = httpx.AsyncClient(
|
||
base_url=self._higgs_base_url,
|
||
timeout=httpx.Timeout(self._higgs_timeout_seconds, connect=3.0),
|
||
)
|
||
return self._higgs_client
|
||
|
||
# ── STT (transcriptions) ─────────────────────────────
|
||
async def transcribe(
|
||
self,
|
||
audio: bytes,
|
||
*,
|
||
filename: str = "audio.webm",
|
||
content_type: str = "audio/webm",
|
||
language: str = STT_LANGUAGE,
|
||
model: str = STT_MODEL,
|
||
) -> TranscriptResult:
|
||
"""오디오 바이트 → 텍스트 전사(OpenAI /audio/transcriptions).
|
||
|
||
클라가 보낸 webm/opus(또는 wav/mp3) 청크를 multipart 로 OpenAI 에 올린다.
|
||
키 없으면 VoiceUnavailable, OpenAI 오류는 그대로 RuntimeError 로 전파(라우트가 처리).
|
||
"""
|
||
if not audio:
|
||
return TranscriptResult(text="", model=model)
|
||
files = {"file": (filename, audio, content_type)}
|
||
data = {
|
||
"model": model,
|
||
"language": language,
|
||
"response_format": "json",
|
||
}
|
||
try:
|
||
r = await self._http.post(STT_ENDPOINT, files=files, data=data)
|
||
if r.status_code == 404 and model != STT_MODEL_FALLBACK:
|
||
# 모델 미가용(계정 권한) → whisper-1 폴백 1회
|
||
data["model"] = STT_MODEL_FALLBACK
|
||
r = await self._http.post(STT_ENDPOINT, files=files, data=data)
|
||
r.raise_for_status()
|
||
except VoiceUnavailable:
|
||
raise
|
||
except httpx.HTTPStatusError as e:
|
||
raise RuntimeError(f"STT {e.response.status_code}: {e.response.text[:200]}") from e
|
||
except httpx.HTTPError as e:
|
||
raise RuntimeError(f"STT transport error: {e}") from e
|
||
|
||
body = r.json()
|
||
return TranscriptResult(
|
||
text=(body.get("text") or "").strip(),
|
||
language=body.get("language"),
|
||
model=str(data["model"]),
|
||
duration=body.get("duration"),
|
||
)
|
||
|
||
# ── TTS (speech) — 스트리밍 ──────────────────────────
|
||
async def synthesize_stream(
|
||
self,
|
||
text: str,
|
||
voice: VoicePreset,
|
||
*,
|
||
model: str = TTS_MODEL,
|
||
response_format: str = TTS_RESPONSE_FORMAT,
|
||
) -> AsyncIterator[TTSChunk]:
|
||
"""텍스트 → 음성 스트리밍(OpenAI 또는 로컬 Higgs). 오디오 청크를 yield한다.
|
||
|
||
설계 §5.2 'speaking' 상태: 오디오 청크를 흘리며 진폭 힌트(립싱크)를 같이 보낸다.
|
||
선택 provider가 준비되지 않으면 VoiceUnavailable, 전송 오류는 RuntimeError로 전파한다.
|
||
"""
|
||
# 비언어 지문((고개 끄덕)·（한숨）·[침묵])은 음성으로 읽지 않는다. 자막엔 남고
|
||
# 아바타 애니메이션이 표현한다. 지문만 있는 발화는 합성 생략(빈 오디오).
|
||
text = speakable_text(text)
|
||
if not text:
|
||
return
|
||
if self._should_use_melotts_tts(voice):
|
||
async for chunk in self._synthesize_melotts_tts(text, voice):
|
||
yield chunk
|
||
return
|
||
if self._should_use_higgs_tts(voice):
|
||
async for chunk in self._synthesize_higgs_tts(text, voice):
|
||
yield chunk
|
||
return
|
||
if self._should_use_poc_sample_tts(voice):
|
||
async for chunk in self._synthesize_poc_sample_tts(text):
|
||
yield chunk
|
||
return
|
||
payload = build_tts_payload(
|
||
text,
|
||
voice,
|
||
model=model,
|
||
response_format=response_format,
|
||
)
|
||
|
||
try:
|
||
async with self._http.stream("POST", TTS_ENDPOINT, json=payload) as r:
|
||
if r.status_code == 404 and model != TTS_MODEL_FALLBACK:
|
||
# 모델 미가용 → tts-1 폴백(비스트림 재시도). instructions 제거.
|
||
payload["model"] = TTS_MODEL_FALLBACK
|
||
payload.pop("instructions", None)
|
||
await r.aclose()
|
||
async for c in self._synthesize_fallback(payload):
|
||
yield c
|
||
return
|
||
r.raise_for_status()
|
||
async for chunk in r.aiter_bytes(chunk_size=4096):
|
||
if not chunk:
|
||
continue
|
||
yield TTSChunk(audio=chunk)
|
||
except VoiceUnavailable:
|
||
raise
|
||
except httpx.HTTPStatusError as e:
|
||
text_body = ""
|
||
try:
|
||
text_body = (await e.response.aread()).decode("utf-8", "ignore")[:200]
|
||
except Exception:
|
||
pass
|
||
raise RuntimeError(f"TTS {e.response.status_code}: {text_body}") from e
|
||
except httpx.HTTPError as e:
|
||
raise RuntimeError(f"TTS transport error: {e}") from e
|
||
|
||
async def _synthesize_poc_sample_tts(self, text: str) -> AsyncIterator[TTSChunk]:
|
||
sample_id = self._select_poc_sample_id(text)
|
||
sample_path = self._poc_sample_path(sample_id)
|
||
try:
|
||
data = sample_path.read_bytes()
|
||
except OSError as e:
|
||
raise VoiceUnavailable(f"P1 sample TTS asset is missing: {sample_path}") from e
|
||
for i in range(0, len(data), POC_SAMPLE_TTS_CHUNK_SIZE):
|
||
chunk = data[i : i + POC_SAMPLE_TTS_CHUNK_SIZE]
|
||
if chunk:
|
||
yield TTSChunk(audio=chunk)
|
||
|
||
@property
|
||
def _melotts_http(self) -> httpx.AsyncClient:
|
||
if not self._melotts_enabled():
|
||
raise VoiceUnavailable("MeloTTS provider is not configured")
|
||
if self._melotts_client is None:
|
||
self._melotts_client = httpx.AsyncClient(
|
||
base_url=self._melotts_base_url,
|
||
timeout=httpx.Timeout(self._melotts_timeout_seconds, connect=3.0),
|
||
)
|
||
return self._melotts_client
|
||
|
||
async def _synthesize_melotts_tts(
|
||
self, text: str, voice: VoicePreset
|
||
) -> AsyncIterator[TTSChunk]:
|
||
payload = {"text": speakable_text(text), "speed": _clamp_speed(voice.rate)}
|
||
try:
|
||
async with self._melotts_http.stream(
|
||
"POST", MELOTTS_TTS_ENDPOINT, json=payload
|
||
) as response:
|
||
response.raise_for_status()
|
||
async for chunk in response.aiter_bytes(
|
||
chunk_size=POC_SAMPLE_TTS_CHUNK_SIZE
|
||
):
|
||
if chunk:
|
||
yield TTSChunk(audio=chunk)
|
||
except httpx.HTTPStatusError as exc:
|
||
body = ""
|
||
try:
|
||
body = (await exc.response.aread()).decode("utf-8", "ignore")[:200]
|
||
except Exception:
|
||
pass
|
||
raise RuntimeError(f"MeloTTS {exc.response.status_code}: {body}") from exc
|
||
except httpx.HTTPError as exc:
|
||
raise RuntimeError(f"MeloTTS transport error: {exc}") from exc
|
||
|
||
async def _synthesize_higgs_tts(
|
||
self, text: str, voice: VoicePreset
|
||
) -> AsyncIterator[TTSChunk]:
|
||
payload = {
|
||
"text": build_higgs_prompt(text, voice),
|
||
"preset": voice.preset,
|
||
}
|
||
try:
|
||
async with self._higgs_http.stream(
|
||
"POST", HIGGS_TTS_ENDPOINT, json=payload
|
||
) as response:
|
||
response.raise_for_status()
|
||
async for chunk in response.aiter_bytes(chunk_size=POC_SAMPLE_TTS_CHUNK_SIZE):
|
||
if chunk:
|
||
yield TTSChunk(audio=chunk)
|
||
except httpx.HTTPStatusError as exc:
|
||
body = ""
|
||
try:
|
||
body = (await exc.response.aread()).decode("utf-8", "ignore")[:200]
|
||
except Exception:
|
||
pass
|
||
raise RuntimeError(f"Higgs TTS {exc.response.status_code}: {body}") from exc
|
||
except httpx.HTTPError as exc:
|
||
raise RuntimeError(f"Higgs TTS transport error: {exc}") from exc
|
||
|
||
def _select_poc_sample_id(self, text: str) -> str:
|
||
normalized = text.casefold()
|
||
for sample_id, keywords in _POC_SAMPLE_TTS_KEYWORDS:
|
||
if any(keyword.casefold() in normalized for keyword in keywords):
|
||
return sample_id
|
||
return _POC_SAMPLE_TTS_DEFAULT_SAMPLE
|
||
|
||
async def _synthesize_fallback(self, payload: dict[str, object]) -> AsyncIterator[TTSChunk]:
|
||
"""tts-1 폴백(비스트림 POST → 전체 바이트를 청크로 분할)."""
|
||
try:
|
||
r = await self._http.post(TTS_ENDPOINT, json=payload)
|
||
r.raise_for_status()
|
||
except httpx.HTTPStatusError as e:
|
||
raise RuntimeError(f"TTS(fallback) {e.response.status_code}: {e.response.text[:200]}") from e
|
||
except httpx.HTTPError as e:
|
||
raise RuntimeError(f"TTS(fallback) transport error: {e}") from e
|
||
data = r.content
|
||
for i in range(0, len(data), 4096):
|
||
chunk = data[i : i + 4096]
|
||
yield TTSChunk(audio=chunk)
|
||
|
||
|
||
def _clamp_speed(rate: float) -> float:
|
||
"""OpenAI speed 허용범위 [0.25, 4.0] 클램프."""
|
||
try:
|
||
return max(0.25, min(4.0, float(rate)))
|
||
except (TypeError, ValueError):
|
||
return 1.0
|
||
|
||
|
||
def _nonnegative_int(value: object) -> int:
|
||
try:
|
||
return max(0, int(value)) # type: ignore[arg-type]
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def _optional_float(value: object) -> float | None:
|
||
try:
|
||
return float(value) # type: ignore[arg-type]
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _clean_optional_text(value: object) -> str | None:
|
||
if value is None:
|
||
return None
|
||
text = str(value).strip()
|
||
return text or None
|
||
|
||
|
||
# 앱 전역 싱글톤 (main lifespan 이 startup/shutdown — Foundation 이 관리하거나
|
||
# 라우트가 lazy 사용). engine_client 패턴과 동일.
|
||
voice_service = VoiceService()
|
||
|
||
|
||
__all__ = [
|
||
"VoiceUnavailable",
|
||
"VoicePreset",
|
||
"TranscriptResult",
|
||
"TranscriptWord",
|
||
"StreamingTranscriptEvent",
|
||
"DeepgramStreamingSession",
|
||
"MELOTTS_TTS_MODEL",
|
||
"LocalWhisperStreamingSession",
|
||
"EndOfTurnDecision",
|
||
"TTSChunk",
|
||
"VoiceService",
|
||
"voice_service",
|
||
"resolve_voice",
|
||
"resolve_voice_from_map",
|
||
"build_tts_payload",
|
||
"build_higgs_prompt",
|
||
"assess_end_of_turn",
|
||
"EOT_SILENCE_THRESHOLD_MS",
|
||
"PRESET_TO_OPENAI_VOICE",
|
||
"PERSONA_CODE_TO_PRESET",
|
||
"DEFAULT_OPENAI_VOICE",
|
||
"STT_MODEL",
|
||
"DEEPGRAM_STT_MODEL",
|
||
"TTS_MODEL",
|
||
"HIGGS_TTS_MODEL",
|
||
]
|