텍스트 응답 음성 재생 연결
This commit is contained in:
Yun Chan
parent
64e06a1185
commit
d80e33da5e
9 changed files with 524 additions and 16 deletions
|
|
@ -139,6 +139,25 @@ async function startFakeEngine(): Promise<TestServer> {
|
|||
);
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && req.url === "/v1/stream") {
|
||||
res.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-cache",
|
||||
});
|
||||
res.write(`event: token\ndata: ${JSON.stringify({ text: "괜찮아요. " })}\n\n`);
|
||||
res.write(`event: token\ndata: ${JSON.stringify({ text: "천천히 말해볼게요." })}\n\n`);
|
||||
res.end(
|
||||
`event: done\ndata: ${JSON.stringify({
|
||||
provider: "e2e",
|
||||
model: "fake-client",
|
||||
tokens_in: 1,
|
||||
tokens_out: 1,
|
||||
cost_usd: 0,
|
||||
turns: 1,
|
||||
})}\n\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "not found" }));
|
||||
});
|
||||
|
|
@ -836,7 +855,7 @@ test.describe("voice cascade success path", () => {
|
|||
test("drives one voice turn through the Session mic UI with synthetic browser audio @single-run", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(90_000);
|
||||
test.setTimeout(150_000);
|
||||
|
||||
const diagnostics: string[] = [];
|
||||
page.on("pageerror", (error) => diagnostics.push(`pageerror: ${error.message}`));
|
||||
|
|
@ -848,7 +867,10 @@ test.describe("voice cascade success path", () => {
|
|||
});
|
||||
page.on("response", (response) => {
|
||||
const url = response.url();
|
||||
if (response.status() >= 400 && (url.includes("/sessions") || url.includes("/voice/ws"))) {
|
||||
if (
|
||||
response.status() >= 400 &&
|
||||
(url.includes("/sessions") || url.includes("/voice/ws") || url.includes("/voice/speech"))
|
||||
) {
|
||||
diagnostics.push(`response: ${response.status()} ${url}`);
|
||||
}
|
||||
});
|
||||
|
|
@ -964,6 +986,39 @@ test.describe("voice cascade success path", () => {
|
|||
].join("\n\n"),
|
||||
).toBeVisible({ timeout: 20_000 });
|
||||
|
||||
const textInput = page.getByLabel("학습자 발화 입력");
|
||||
const sendButton = page.getByRole("button", { name: "보내기", exact: true });
|
||||
await textInput.fill("텍스트로 말해도 내담자 음성을 들려주세요.");
|
||||
await sendButton.click();
|
||||
await expect
|
||||
.poll(
|
||||
() => openai.requests().filter((request) => request === "POST /v1/audio/speech").length,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
.toBeGreaterThan(0);
|
||||
await expect
|
||||
.poll(async () => (await readVoiceUiProbe(page)).audioBufferStarts, { timeout: 30_000 })
|
||||
.toBeGreaterThan(0);
|
||||
await expect(page.locator(".sx-utt").filter({ hasText: "괜찮아요. 천천히 말해볼게요." })).toBeVisible();
|
||||
|
||||
const freshVoiceSession = await page.evaluate(async ({ apiBase, personaCode }) => {
|
||||
const response = await fetch(`${apiBase}/sessions`, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ persona_code: personaCode, theory_mode: "humanistic" }),
|
||||
});
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
body: await response.json() as { session_id?: string },
|
||||
};
|
||||
}, { apiBase: api.baseURL, personaCode: SEEDED_VOICE_PERSONA_CODE });
|
||||
expect(freshVoiceSession, api.logs()).toMatchObject({ ok: true });
|
||||
expect(typeof freshVoiceSession.body.session_id).toBe("string");
|
||||
await page.goto(`${web.baseURL}/learn/session/${freshVoiceSession.body.session_id}`);
|
||||
await expect(page.locator(".sx-page.sx-page--active")).toBeVisible({ timeout: 20_000 });
|
||||
|
||||
const mic = page.locator(".sx-mic");
|
||||
await expect(mic).toBeEnabled();
|
||||
await mic.click();
|
||||
|
|
|
|||
|
|
@ -407,6 +407,26 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/client-diagnostics": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Record Client Diagnostic
|
||||
* @description Log a minimal browser-side boot/render failure report.
|
||||
*/
|
||||
post: operations["record_client_diagnostic_client_diagnostics_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/eval/health": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -1341,6 +1361,30 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/voice/speech": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Voice Speech
|
||||
* @description Synthesize the persisted client reply for a completed text turn.
|
||||
*
|
||||
* The browser sends only session/turn identifiers. The server reloads the
|
||||
* owned session and speaks the stored client-visible reply, so this endpoint
|
||||
* cannot be used as an arbitrary paid text-to-speech proxy.
|
||||
*/
|
||||
post: operations["voice_speech_voice_speech_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
}
|
||||
export type webhooks = Record<string, never>;
|
||||
export interface components {
|
||||
|
|
@ -1951,6 +1995,71 @@ export interface components {
|
|||
/** Source Id */
|
||||
source_id?: string | null;
|
||||
};
|
||||
/** ClientDiagnosticError */
|
||||
ClientDiagnosticError: {
|
||||
/** At */
|
||||
at?: string | null;
|
||||
/**
|
||||
* Detail
|
||||
* @default
|
||||
*/
|
||||
detail: string;
|
||||
/**
|
||||
* Kind
|
||||
* @default
|
||||
*/
|
||||
kind: string;
|
||||
};
|
||||
/** ClientDiagnosticRequest */
|
||||
ClientDiagnosticRequest: {
|
||||
/**
|
||||
* Asset
|
||||
* @default
|
||||
*/
|
||||
asset: string;
|
||||
/** Body Text Length */
|
||||
body_text_length?: number | null;
|
||||
/**
|
||||
* Document Ready State
|
||||
* @default
|
||||
*/
|
||||
document_ready_state: string;
|
||||
/** Elapsed Ms */
|
||||
elapsed_ms?: number | null;
|
||||
/** Errors */
|
||||
errors?: components["schemas"]["ClientDiagnosticError"][];
|
||||
/**
|
||||
* Href
|
||||
* @default
|
||||
*/
|
||||
href: string;
|
||||
/**
|
||||
* Path
|
||||
* @default
|
||||
*/
|
||||
path: string;
|
||||
/**
|
||||
* Reason
|
||||
* @default unknown
|
||||
*/
|
||||
reason: string;
|
||||
/** Root Child Count */
|
||||
root_child_count?: number | null;
|
||||
/** Root Text Length */
|
||||
root_text_length?: number | null;
|
||||
/**
|
||||
* User Agent
|
||||
* @default
|
||||
*/
|
||||
user_agent: string;
|
||||
/**
|
||||
* Viewport
|
||||
* @default
|
||||
*/
|
||||
viewport: string;
|
||||
/** Visible Nodes */
|
||||
visible_nodes?: number | null;
|
||||
};
|
||||
/**
|
||||
* ClientStateRead
|
||||
* @description 내담자 상태 '읽기' — 학습자 발화 직후 내담자 응답에서 관측된 상태(읽기 채점 근거).
|
||||
|
|
@ -4368,6 +4477,16 @@ export interface components {
|
|||
/** Voice Id */
|
||||
voice_id: string;
|
||||
};
|
||||
/**
|
||||
* VoiceSpeechRequest
|
||||
* @description Request OpenAI TTS for an already-persisted client reply.
|
||||
*/
|
||||
VoiceSpeechRequest: {
|
||||
/** Session Id */
|
||||
session_id: string;
|
||||
/** Turn Seq */
|
||||
turn_seq: number;
|
||||
};
|
||||
};
|
||||
responses: never;
|
||||
parameters: never;
|
||||
|
|
@ -5140,6 +5259,41 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
record_client_diagnostic_client_diagnostics_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ClientDiagnosticRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
[key: string]: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
eval_health_eval_health_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -6953,4 +7107,40 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
voice_speech_voice_speech_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: {
|
||||
"__Host-vignette_sid"?: string | null;
|
||||
vignette_sid?: string | null;
|
||||
};
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["VoiceSpeechRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -447,6 +447,21 @@ export const sessionApi = {
|
|||
return { available: false, reason: "voice health check failed" };
|
||||
}
|
||||
},
|
||||
speakClientTurn: async (sessionId: string, turnSeq: number): Promise<Blob> => {
|
||||
const r = await fetch(apiUrl("/voice/speech"), {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "audio/mpeg",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ session_id: sessionId, turn_seq: turnSeq }),
|
||||
});
|
||||
if (!r.ok) throw await parseError(r);
|
||||
const audio = await r.blob();
|
||||
if (!audio.size) throw new ApiError(502, "TTS 응답 오디오가 비어 있습니다.");
|
||||
return audio;
|
||||
},
|
||||
get: (sessionId: string) =>
|
||||
api.get<SessionDetailResponse>(`/sessions/${encodeURIComponent(sessionId)}`),
|
||||
start: (persona_code: string, theory_mode: "humanistic" | "cbt" | "integrative" = "humanistic") =>
|
||||
|
|
|
|||
|
|
@ -886,6 +886,8 @@ export default function Session() {
|
|||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const ttsChunksRef = useRef<BlobPart[]>([]);
|
||||
const ttsPlaybackCleanupRef = useRef<(() => void) | null>(null);
|
||||
const ttsPlaybackRequestRef = useRef(0);
|
||||
const playTtsAudioRef = useRef<(() => Promise<void>) | null>(null);
|
||||
const pendingVoiceLearnerIdRef = useRef<number | null>(null);
|
||||
const pendingVoiceLearnerTextRef = useRef<string>("");
|
||||
const coachEvidenceCloseRef = useRef<HTMLButtonElement>(null);
|
||||
|
|
@ -930,6 +932,7 @@ export default function Session() {
|
|||
}, [ensureVoiceAudioContext]);
|
||||
|
||||
const stopTtsPlayback = useCallback(() => {
|
||||
ttsPlaybackRequestRef.current += 1;
|
||||
const cleanup = ttsPlaybackCleanupRef.current;
|
||||
ttsPlaybackCleanupRef.current = null;
|
||||
if (cleanup) cleanup();
|
||||
|
|
@ -1405,6 +1408,30 @@ export default function Session() {
|
|||
}
|
||||
}, [acceptConsent, consentChecked, pushSignal]);
|
||||
|
||||
const speakTextClientTurn = useCallback(
|
||||
async (sessionId: string, turnSeq: number) => {
|
||||
const requestId = ttsPlaybackRequestRef.current + 1;
|
||||
ttsPlaybackRequestRef.current = requestId;
|
||||
setVoiceStatus("speaking");
|
||||
setVoiceDetail("OpenAI 음성을 생성하고 있습니다.");
|
||||
try {
|
||||
const audio = await sessionApi.speakClientTurn(sessionId, turnSeq);
|
||||
if (ttsPlaybackRequestRef.current !== requestId) return;
|
||||
ttsChunksRef.current = [audio];
|
||||
const play = playTtsAudioRef.current;
|
||||
if (!play) throw new Error("voice playback is not ready");
|
||||
await play();
|
||||
} catch {
|
||||
if (ttsPlaybackRequestRef.current !== requestId) return;
|
||||
setAvatarState("listening");
|
||||
setVoiceStatus("degraded");
|
||||
setVoiceDetail("OpenAI 음성을 재생하지 못했습니다. 자막 응답은 화면에 남겼습니다.");
|
||||
pushSignal("warn", "AI 음성 재생 실패");
|
||||
}
|
||||
},
|
||||
[pushSignal],
|
||||
);
|
||||
|
||||
const appendServerClientReply = useCallback(
|
||||
(replyText: string | null, at = elapsed, options?: { suppressEmptyWarning?: boolean }) => {
|
||||
setClientReplyPending(false);
|
||||
|
|
@ -1450,6 +1477,8 @@ export default function Session() {
|
|||
setClientReplyPending(true);
|
||||
setAvatarState("thinking");
|
||||
setTurnError(null);
|
||||
stopTtsPlayback();
|
||||
void primeVoicePlayback();
|
||||
|
||||
let clientId: number | null = null;
|
||||
let clientReply = "";
|
||||
|
|
@ -1539,6 +1568,9 @@ export default function Session() {
|
|||
}
|
||||
setAvatarState("listening");
|
||||
if (!conversationStopped && !qualityRetryable) {
|
||||
if (clientReply && typeof done.turn_seq === "number") {
|
||||
void speakTextClientTurn(liveSessionId, done.turn_seq);
|
||||
}
|
||||
void requestLiveCoach({
|
||||
learnerText: text,
|
||||
clientReply,
|
||||
|
|
@ -1580,7 +1612,10 @@ export default function Session() {
|
|||
liveSessionId,
|
||||
elapsed,
|
||||
pushSignal,
|
||||
primeVoicePlayback,
|
||||
requestLiveCoach,
|
||||
speakTextClientTurn,
|
||||
stopTtsPlayback,
|
||||
]);
|
||||
|
||||
const onComposeKey = (e: ReactKeyboardEvent<HTMLTextAreaElement>) => {
|
||||
|
|
@ -1752,6 +1787,15 @@ export default function Session() {
|
|||
}
|
||||
}, [clientName, closeVoiceSocket, ensureVoiceAudioContext, stopTtsPlayback]);
|
||||
|
||||
useEffect(() => {
|
||||
playTtsAudioRef.current = playTtsAudio;
|
||||
return () => {
|
||||
if (playTtsAudioRef.current === playTtsAudio) {
|
||||
playTtsAudioRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [playTtsAudio]);
|
||||
|
||||
const startVoiceCapture = useCallback(async () => {
|
||||
if (!liveSessionId || paused || sending || sessionEnded) return;
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue