yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

텍스트 응답 음성 재생 연결

Browse source
This commit is contained in:
Yun Chan 2026-07-13 16:09:34 +09:00
parent 64e06a1185
commit d80e33da5e
9 changed files with 524 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

110
apps/api/app/routes/voice.py
View file

@ -18,14 +18,15 @@ import hashlib
import time
from typing import Optional
from fastapi import APIRouter, WebSocket, WebSocketDisconnect, HTTPException
from fastapi.responses import JSONResponse
from fastapi import APIRouter, WebSocket, WebSocketDisconnect, HTTPException, status
from fastapi.responses import JSONResponse, Response
from pydantic import BaseModel, Field
from starlette.websockets import WebSocketState
from .. import session_persistence, turn_runtime
from ..auth_sessions import get_session, user_has_consent, user_onboarding_complete
from ..config import settings
from ..deps import Principal, Role
from ..deps import CurrentPrincipal, Principal, Role
from ..engine_client import EngineError, engine_client
from ..persona_repository import (
PersonaVoiceMap,
@ -41,6 +42,13 @@ from ..store import InProcSession, TurnRecord, store
router = APIRouter(prefix="/voice", tags=["voice"])
class VoiceSpeechRequest(BaseModel):
"""Request OpenAI TTS for an already-persisted client reply."""
session_id: str = Field(min_length=1, max_length=80)
turn_seq: int = Field(ge=1)
# WebSocket close codes.
WS_CLOSE_DEGRADED = 1011
WS_CLOSE_BAD_REQUEST = 1008
@ -123,6 +131,89 @@ async def voice_health() -> JSONResponse:
return JSONResponse(body, status_code=200 if available else 503)
@router.post("/speech")
async def voice_speech(body: VoiceSpeechRequest, principal: CurrentPrincipal) -> Response:
"""Synthesize the persisted client reply for a completed text turn.
The browser sends only session/turn identifiers. The server reloads the
owned session and speaks the stored client-visible reply, so this endpoint
cannot be used as an arbitrary paid text-to-speech proxy.
"""
learner = principal
if learner.role != Role.LEARNER and learner.can_access_role(Role.LEARNER):
learner = learner.with_role(Role.LEARNER)
if learner.role != Role.LEARNER:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="only learners can use voice",
)
access_error = await _practice_access_error(learner)
if access_error is not None:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=access_error)
if not voice_service.is_available():
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="OPENAI_API_KEY is not configured",
)
sess, err = await turn_runtime.load_owned_session(
body.session_id,
learner,
allow_ended=False,
)
if sess is None:
status_code = {
turn_runtime.SessionAccessError.FORBIDDEN: status.HTTP_403_FORBIDDEN,
turn_runtime.SessionAccessError.ENDED: status.HTTP_409_CONFLICT,
}.get(err, status.HTTP_404_NOT_FOUND)
raise HTTPException(status_code=status_code, detail=f"voice session {err or 'not_found'}")
text = _client_turn_text_for_speech(sess, body.turn_seq)
if text is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="client reply not found for turn",
)
voice_preset = await _resolve_session_voice(
session_id=body.session_id,
persona_code=sess.persona.code,
explicit_preset=None,
)
try:
chunks = [
chunk.audio
async for chunk in voice_service.synthesize_stream(text, voice_preset)
]
except VoiceUnavailable as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=str(exc),
) from exc
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"TTS failed: {exc}",
) from exc
audio = b"".join(chunks)
if not audio:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="client reply has no speakable text",
)
return Response(
content=audio,
media_type="audio/mpeg",
headers={
"Cache-Control": "no-store",
"X-Vignette-TTS-Model": voice_svc.TTS_MODEL,
"X-Vignette-TTS-Provider": voice_service.tts_provider(),
},
)
@router.websocket("/ws")
async def voice_ws(websocket: WebSocket) -> None:
"""Run one authenticated learner voice cascade."""
@ -596,6 +687,19 @@ async def _load_voice_session(
return sess, None
def _client_turn_text_for_speech(sess: InProcSession, turn_seq: int) -> str | None:
"""Return the persisted client-visible reply for one completed turn."""
for turn in reversed(sess.turns):
if (
turn.turn_seq == turn_seq
and turn.speaker == "client"
and turn.is_visible_to("client")
):
text = (turn.text_masked or turn.text).strip()
return text or None
return None
async def _principal_from_websocket(websocket: WebSocket) -> Principal | None:
"""Restore the same server-side browser session used by REST routes."""
raw_cookie = websocket.cookies.get(settings.cookie_name)

95
apps/api/app/test_voice_ws.py
View file

@ -12,7 +12,8 @@ from fastapi import HTTPException
from .deps import Principal, Role
from .persona_repository import PersonaVoiceMap
from .routes import voice as voice_routes
from .services.voice import VoicePreset
from .services.voice import TTSChunk, VoicePreset
from .store import TurnRecord
SESSION_ID = "voice-ws-contract-session"
@ -130,6 +131,98 @@ class VoiceWebSocketContractTest(unittest.IsolatedAsyncioTestCase):
],
)
def test_text_tts_uses_only_the_persisted_client_visible_reply(self) -> None:
session = SimpleNamespace(
turns=[
TurnRecord(
turn_seq=2,
speaker="counselor",
stage="초기",
text="raw learner text",
text_masked="masked learner text",
),
TurnRecord(
turn_seq=2,
speaker="client",
stage="초기",
text="raw client reply",
text_masked="마스킹된 내담자 응답",
),
TurnRecord(
turn_seq=3,
speaker="client",
stage="초기",
text="hidden evaluator reply",
text_masked="hidden evaluator reply",
visible_to=("evaluator",),
),
]
)
self.assertEqual(
voice_routes._client_turn_text_for_speech(session, 2),
"마스킹된 내담자 응답",
)
self.assertIsNone(voice_routes._client_turn_text_for_speech(session, 3))
self.assertIsNone(voice_routes._client_turn_text_for_speech(session, 99))
async def test_text_turn_speech_returns_openai_audio_for_owned_persisted_turn(self) -> None:
session = SimpleNamespace(
persona=SimpleNamespace(code="P1"),
turns=[
TurnRecord(
turn_seq=4,
speaker="client",
stage="초기",
text="내담자 응답",
text_masked="내담자 응답",
)
],
)
synthesized: list[tuple[str, VoicePreset]] = []
async def synthesize(text: str, voice: VoicePreset):
synthesized.append((text, voice))
yield TTSChunk(audio=b"mp3-a")
yield TTSChunk(audio=b"mp3-b")
with patch.object(
voice_routes,
"_practice_access_error",
AsyncMock(return_value=None),
), patch.object(
voice_routes.turn_runtime,
"load_owned_session",
AsyncMock(return_value=(session, None)),
), patch.object(
voice_routes,
"_resolve_session_voice",
AsyncMock(return_value=VOICE_PRESET),
), patch.object(
voice_routes.voice_service,
"is_available",
return_value=True,
), patch.object(
voice_routes.voice_service,
"tts_provider",
return_value="openai",
), patch.object(
voice_routes.voice_service,
"synthesize_stream",
new=synthesize,
):
response = await voice_routes.voice_speech(
voice_routes.VoiceSpeechRequest(session_id=SESSION_ID, turn_seq=4),
_principal(),
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.media_type, "audio/mpeg")
self.assertEqual(response.body, b"mp3-amp3-b")
self.assertEqual(response.headers["cache-control"], "no-store")
self.assertEqual(response.headers["x-vignette-tts-provider"], "openai")
self.assertEqual(synthesized, [("내담자 응답", VOICE_PRESET)])
async def test_audio_start_binary_chunks_audio_end_ping_close_contract(self) -> None:
websocket = FakeWebSocket(
[

59
apps/web/e2e/voice-success.spec.ts
View file

@ -139,6 +139,25 @@ async function startFakeEngine(): Promise<TestServer> {
);
return;
}
if (req.method === "POST" && req.url === "/v1/stream") {
res.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
});
res.write(`event: token\ndata: ${JSON.stringify({ text: "괜찮아요. " })}\n\n`);
res.write(`event: token\ndata: ${JSON.stringify({ text: "천천히 말해볼게요." })}\n\n`);
res.end(
`event: done\ndata: ${JSON.stringify({
provider: "e2e",
model: "fake-client",
tokens_in: 1,
tokens_out: 1,
cost_usd: 0,
turns: 1,
})}\n\n`,
);
return;
}
res.writeHead(404, { "content-type": "application/json" });
res.end(JSON.stringify({ error: "not found" }));
});
@ -836,7 +855,7 @@ test.describe("voice cascade success path", () => {
test("drives one voice turn through the Session mic UI with synthetic browser audio @single-run", async ({
page,
}, testInfo) => {
test.setTimeout(90_000);
test.setTimeout(150_000);
const diagnostics: string[] = [];
page.on("pageerror", (error) => diagnostics.push(`pageerror: ${error.message}`));
@ -848,7 +867,10 @@ test.describe("voice cascade success path", () => {
});
page.on("response", (response) => {
const url = response.url();
if (response.status() >= 400 && (url.includes("/sessions") || url.includes("/voice/ws"))) {
if (
response.status() >= 400 &&
(url.includes("/sessions") || url.includes("/voice/ws") || url.includes("/voice/speech"))
) {
diagnostics.push(`response: ${response.status()} ${url}`);
}
});
@ -964,6 +986,39 @@ test.describe("voice cascade success path", () => {
].join("\n\n"),
).toBeVisible({ timeout: 20_000 });
const textInput = page.getByLabel("학습자 발화 입력");
const sendButton = page.getByRole("button", { name: "보내기", exact: true });
await textInput.fill("텍스트로 말해도 내담자 음성을 들려주세요.");
await sendButton.click();
await expect
.poll(
() => openai.requests().filter((request) => request === "POST /v1/audio/speech").length,
{ timeout: 30_000 },
)
.toBeGreaterThan(0);
await expect
.poll(async () => (await readVoiceUiProbe(page)).audioBufferStarts, { timeout: 30_000 })
.toBeGreaterThan(0);
await expect(page.locator(".sx-utt").filter({ hasText: "괜찮아요. 천천히 말해볼게요." })).toBeVisible();
const freshVoiceSession = await page.evaluate(async ({ apiBase, personaCode }) => {
const response = await fetch(`${apiBase}/sessions`, {
method: "POST",
credentials: "include",
headers: { "content-type": "application/json" },
body: JSON.stringify({ persona_code: personaCode, theory_mode: "humanistic" }),
});
return {
ok: response.ok,
status: response.status,
body: await response.json() as { session_id?: string },
};
}, { apiBase: api.baseURL, personaCode: SEEDED_VOICE_PERSONA_CODE });
expect(freshVoiceSession, api.logs()).toMatchObject({ ok: true });
expect(typeof freshVoiceSession.body.session_id).toBe("string");
await page.goto(`${web.baseURL}/learn/session/${freshVoiceSession.body.session_id}`);
await expect(page.locator(".sx-page.sx-page--active")).toBeVisible({ timeout: 20_000 });
const mic = page.locator(".sx-mic");
await expect(mic).toBeEnabled();
await mic.click();

190
apps/web/src/lib/api.gen.ts
View file

@ -407,6 +407,26 @@ export interface paths {
patch?: never;
trace?: never;
};
"/client-diagnostics": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Record Client Diagnostic
* @description Log a minimal browser-side boot/render failure report.
*/
post: operations["record_client_diagnostic_client_diagnostics_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/eval/health": {
parameters: {
query?: never;
@ -1341,6 +1361,30 @@ export interface paths {
patch?: never;
trace?: never;
};
"/voice/speech": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Voice Speech
* @description Synthesize the persisted client reply for a completed text turn.
*
* The browser sends only session/turn identifiers. The server reloads the
* owned session and speaks the stored client-visible reply, so this endpoint
* cannot be used as an arbitrary paid text-to-speech proxy.
*/
post: operations["voice_speech_voice_speech_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
}
export type webhooks = Record<string, never>;
export interface components {
@ -1951,6 +1995,71 @@ export interface components {
/** Source Id */
source_id?: string | null;
};
/** ClientDiagnosticError */
ClientDiagnosticError: {
/** At */
at?: string | null;
/**
* Detail
* @default
*/
detail: string;
/**
* Kind
* @default
*/
kind: string;
};
/** ClientDiagnosticRequest */
ClientDiagnosticRequest: {
/**
* Asset
* @default
*/
asset: string;
/** Body Text Length */
body_text_length?: number | null;
/**
* Document Ready State
* @default
*/
document_ready_state: string;
/** Elapsed Ms */
elapsed_ms?: number | null;
/** Errors */
errors?: components["schemas"]["ClientDiagnosticError"][];
/**
* Href
* @default
*/
href: string;
/**
* Path
* @default
*/
path: string;
/**
* Reason
* @default unknown
*/
reason: string;
/** Root Child Count */
root_child_count?: number | null;
/** Root Text Length */
root_text_length?: number | null;
/**
* User Agent
* @default
*/
user_agent: string;
/**
* Viewport
* @default
*/
viewport: string;
/** Visible Nodes */
visible_nodes?: number | null;
};
/**
* ClientStateRead
* @description 내담자 상태 '읽기' — 학습자 발화 직후 내담자 응답에서 관측된 상태(읽기 채점 근거).
@ -4368,6 +4477,16 @@ export interface components {
/** Voice Id */
voice_id: string;
};
/**
* VoiceSpeechRequest
* @description Request OpenAI TTS for an already-persisted client reply.
*/
VoiceSpeechRequest: {
/** Session Id */
session_id: string;
/** Turn Seq */
turn_seq: number;
};
};
responses: never;
parameters: never;
@ -5140,6 +5259,41 @@ export interface operations {
};
};
};
record_client_diagnostic_client_diagnostics_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["ClientDiagnosticRequest"];
};
};
responses: {
/** @description Successful Response */
202: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": {
[key: string]: boolean;
};
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
eval_health_eval_health_get: {
parameters: {
query?: never;
@ -6953,4 +7107,40 @@ export interface operations {
};
};
};
voice_speech_voice_speech_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: {
"__Host-vignette_sid"?: string | null;
vignette_sid?: string | null;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["VoiceSpeechRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
}

15
apps/web/src/lib/api.ts
View file

@ -447,6 +447,21 @@ export const sessionApi = {
return { available: false, reason: "voice health check failed" };
}
},
speakClientTurn: async (sessionId: string, turnSeq: number): Promise<Blob> => {
const r = await fetch(apiUrl("/voice/speech"), {
method: "POST",
credentials: "include",
headers: {
Accept: "audio/mpeg",
"Content-Type": "application/json",
},
body: JSON.stringify({ session_id: sessionId, turn_seq: turnSeq }),
});
if (!r.ok) throw await parseError(r);
const audio = await r.blob();
if (!audio.size) throw new ApiError(502, "TTS 응답 오디오가 비어 있습니다.");
return audio;
},
get: (sessionId: string) =>
api.get<SessionDetailResponse>(`/sessions/${encodeURIComponent(sessionId)}`),
start: (persona_code: string, theory_mode: "humanistic" | "cbt" | "integrative" = "humanistic") =>

44
apps/web/src/pages/Session.tsx
View file

@ -886,6 +886,8 @@ export default function Session() {
const audioContextRef = useRef<AudioContext | null>(null);
const ttsChunksRef = useRef<BlobPart[]>([]);
const ttsPlaybackCleanupRef = useRef<(() => void) | null>(null);
const ttsPlaybackRequestRef = useRef(0);
const playTtsAudioRef = useRef<(() => Promise<void>) | null>(null);
const pendingVoiceLearnerIdRef = useRef<number | null>(null);
const pendingVoiceLearnerTextRef = useRef<string>("");
const coachEvidenceCloseRef = useRef<HTMLButtonElement>(null);
@ -930,6 +932,7 @@ export default function Session() {
}, [ensureVoiceAudioContext]);
const stopTtsPlayback = useCallback(() => {
ttsPlaybackRequestRef.current += 1;
const cleanup = ttsPlaybackCleanupRef.current;
ttsPlaybackCleanupRef.current = null;
if (cleanup) cleanup();
@ -1405,6 +1408,30 @@ export default function Session() {
}
}, [acceptConsent, consentChecked, pushSignal]);
const speakTextClientTurn = useCallback(
async (sessionId: string, turnSeq: number) => {
const requestId = ttsPlaybackRequestRef.current + 1;
ttsPlaybackRequestRef.current = requestId;
setVoiceStatus("speaking");
setVoiceDetail("OpenAI 음성을 생성하고 있습니다.");
try {
const audio = await sessionApi.speakClientTurn(sessionId, turnSeq);
if (ttsPlaybackRequestRef.current !== requestId) return;
ttsChunksRef.current = [audio];
const play = playTtsAudioRef.current;
if (!play) throw new Error("voice playback is not ready");
await play();
} catch {
if (ttsPlaybackRequestRef.current !== requestId) return;
setAvatarState("listening");
setVoiceStatus("degraded");
setVoiceDetail("OpenAI 음성을 재생하지 못했습니다. 자막 응답은 화면에 남겼습니다.");
pushSignal("warn", "AI 음성 재생 실패");
}
},
[pushSignal],
);
const appendServerClientReply = useCallback(
(replyText: string | null, at = elapsed, options?: { suppressEmptyWarning?: boolean }) => {
setClientReplyPending(false);
@ -1450,6 +1477,8 @@ export default function Session() {
setClientReplyPending(true);
setAvatarState("thinking");
setTurnError(null);
stopTtsPlayback();
void primeVoicePlayback();
let clientId: number | null = null;
let clientReply = "";
@ -1539,6 +1568,9 @@ export default function Session() {
}
setAvatarState("listening");
if (!conversationStopped && !qualityRetryable) {
if (clientReply && typeof done.turn_seq === "number") {
void speakTextClientTurn(liveSessionId, done.turn_seq);
}
void requestLiveCoach({
learnerText: text,
clientReply,
@ -1580,7 +1612,10 @@ export default function Session() {
liveSessionId,
elapsed,
pushSignal,
primeVoicePlayback,
requestLiveCoach,
speakTextClientTurn,
stopTtsPlayback,
]);
const onComposeKey = (e: ReactKeyboardEvent<HTMLTextAreaElement>) => {
@ -1752,6 +1787,15 @@ export default function Session() {
}
}, [clientName, closeVoiceSocket, ensureVoiceAudioContext, stopTtsPlayback]);
useEffect(() => {
playTtsAudioRef.current = playTtsAudio;
return () => {
if (playTtsAudioRef.current === playTtsAudio) {
playTtsAudioRef.current = null;
}
};
}, [playTtsAudio]);
const startVoiceCapture = useCallback(async () => {
if (!liveSessionId || paused || sending || sessionEnded) return;
if (!navigator.mediaDevices?.getUserMedia) {