텍스트 응답 음성 재생 연결
This commit is contained in:
Yun Chan
parent
64e06a1185
commit
d80e33da5e
9 changed files with 524 additions and 16 deletions
|
|
@ -407,6 +407,26 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/client-diagnostics": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Record Client Diagnostic
|
||||
* @description Log a minimal browser-side boot/render failure report.
|
||||
*/
|
||||
post: operations["record_client_diagnostic_client_diagnostics_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/eval/health": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -1341,6 +1361,30 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/voice/speech": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Voice Speech
|
||||
* @description Synthesize the persisted client reply for a completed text turn.
|
||||
*
|
||||
* The browser sends only session/turn identifiers. The server reloads the
|
||||
* owned session and speaks the stored client-visible reply, so this endpoint
|
||||
* cannot be used as an arbitrary paid text-to-speech proxy.
|
||||
*/
|
||||
post: operations["voice_speech_voice_speech_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
}
|
||||
export type webhooks = Record<string, never>;
|
||||
export interface components {
|
||||
|
|
@ -1951,6 +1995,71 @@ export interface components {
|
|||
/** Source Id */
|
||||
source_id?: string | null;
|
||||
};
|
||||
/** ClientDiagnosticError */
|
||||
ClientDiagnosticError: {
|
||||
/** At */
|
||||
at?: string | null;
|
||||
/**
|
||||
* Detail
|
||||
* @default
|
||||
*/
|
||||
detail: string;
|
||||
/**
|
||||
* Kind
|
||||
* @default
|
||||
*/
|
||||
kind: string;
|
||||
};
|
||||
/** ClientDiagnosticRequest */
|
||||
ClientDiagnosticRequest: {
|
||||
/**
|
||||
* Asset
|
||||
* @default
|
||||
*/
|
||||
asset: string;
|
||||
/** Body Text Length */
|
||||
body_text_length?: number | null;
|
||||
/**
|
||||
* Document Ready State
|
||||
* @default
|
||||
*/
|
||||
document_ready_state: string;
|
||||
/** Elapsed Ms */
|
||||
elapsed_ms?: number | null;
|
||||
/** Errors */
|
||||
errors?: components["schemas"]["ClientDiagnosticError"][];
|
||||
/**
|
||||
* Href
|
||||
* @default
|
||||
*/
|
||||
href: string;
|
||||
/**
|
||||
* Path
|
||||
* @default
|
||||
*/
|
||||
path: string;
|
||||
/**
|
||||
* Reason
|
||||
* @default unknown
|
||||
*/
|
||||
reason: string;
|
||||
/** Root Child Count */
|
||||
root_child_count?: number | null;
|
||||
/** Root Text Length */
|
||||
root_text_length?: number | null;
|
||||
/**
|
||||
* User Agent
|
||||
* @default
|
||||
*/
|
||||
user_agent: string;
|
||||
/**
|
||||
* Viewport
|
||||
* @default
|
||||
*/
|
||||
viewport: string;
|
||||
/** Visible Nodes */
|
||||
visible_nodes?: number | null;
|
||||
};
|
||||
/**
|
||||
* ClientStateRead
|
||||
* @description 내담자 상태 '읽기' — 학습자 발화 직후 내담자 응답에서 관측된 상태(읽기 채점 근거).
|
||||
|
|
@ -4368,6 +4477,16 @@ export interface components {
|
|||
/** Voice Id */
|
||||
voice_id: string;
|
||||
};
|
||||
/**
|
||||
* VoiceSpeechRequest
|
||||
* @description Request OpenAI TTS for an already-persisted client reply.
|
||||
*/
|
||||
VoiceSpeechRequest: {
|
||||
/** Session Id */
|
||||
session_id: string;
|
||||
/** Turn Seq */
|
||||
turn_seq: number;
|
||||
};
|
||||
};
|
||||
responses: never;
|
||||
parameters: never;
|
||||
|
|
@ -5140,6 +5259,41 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
record_client_diagnostic_client_diagnostics_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ClientDiagnosticRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
[key: string]: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
eval_health_eval_health_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -6953,4 +7107,40 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
voice_speech_voice_speech_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: {
|
||||
"__Host-vignette_sid"?: string | null;
|
||||
vignette_sid?: string | null;
|
||||
};
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["VoiceSpeechRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -447,6 +447,21 @@ export const sessionApi = {
|
|||
return { available: false, reason: "voice health check failed" };
|
||||
}
|
||||
},
|
||||
speakClientTurn: async (sessionId: string, turnSeq: number): Promise<Blob> => {
|
||||
const r = await fetch(apiUrl("/voice/speech"), {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "audio/mpeg",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ session_id: sessionId, turn_seq: turnSeq }),
|
||||
});
|
||||
if (!r.ok) throw await parseError(r);
|
||||
const audio = await r.blob();
|
||||
if (!audio.size) throw new ApiError(502, "TTS 응답 오디오가 비어 있습니다.");
|
||||
return audio;
|
||||
},
|
||||
get: (sessionId: string) =>
|
||||
api.get<SessionDetailResponse>(`/sessions/${encodeURIComponent(sessionId)}`),
|
||||
start: (persona_code: string, theory_mode: "humanistic" | "cbt" | "integrative" = "humanistic") =>
|
||||
|
|
|
|||
|
|
@ -886,6 +886,8 @@ export default function Session() {
|
|||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const ttsChunksRef = useRef<BlobPart[]>([]);
|
||||
const ttsPlaybackCleanupRef = useRef<(() => void) | null>(null);
|
||||
const ttsPlaybackRequestRef = useRef(0);
|
||||
const playTtsAudioRef = useRef<(() => Promise<void>) | null>(null);
|
||||
const pendingVoiceLearnerIdRef = useRef<number | null>(null);
|
||||
const pendingVoiceLearnerTextRef = useRef<string>("");
|
||||
const coachEvidenceCloseRef = useRef<HTMLButtonElement>(null);
|
||||
|
|
@ -930,6 +932,7 @@ export default function Session() {
|
|||
}, [ensureVoiceAudioContext]);
|
||||
|
||||
const stopTtsPlayback = useCallback(() => {
|
||||
ttsPlaybackRequestRef.current += 1;
|
||||
const cleanup = ttsPlaybackCleanupRef.current;
|
||||
ttsPlaybackCleanupRef.current = null;
|
||||
if (cleanup) cleanup();
|
||||
|
|
@ -1405,6 +1408,30 @@ export default function Session() {
|
|||
}
|
||||
}, [acceptConsent, consentChecked, pushSignal]);
|
||||
|
||||
const speakTextClientTurn = useCallback(
|
||||
async (sessionId: string, turnSeq: number) => {
|
||||
const requestId = ttsPlaybackRequestRef.current + 1;
|
||||
ttsPlaybackRequestRef.current = requestId;
|
||||
setVoiceStatus("speaking");
|
||||
setVoiceDetail("OpenAI 음성을 생성하고 있습니다.");
|
||||
try {
|
||||
const audio = await sessionApi.speakClientTurn(sessionId, turnSeq);
|
||||
if (ttsPlaybackRequestRef.current !== requestId) return;
|
||||
ttsChunksRef.current = [audio];
|
||||
const play = playTtsAudioRef.current;
|
||||
if (!play) throw new Error("voice playback is not ready");
|
||||
await play();
|
||||
} catch {
|
||||
if (ttsPlaybackRequestRef.current !== requestId) return;
|
||||
setAvatarState("listening");
|
||||
setVoiceStatus("degraded");
|
||||
setVoiceDetail("OpenAI 음성을 재생하지 못했습니다. 자막 응답은 화면에 남겼습니다.");
|
||||
pushSignal("warn", "AI 음성 재생 실패");
|
||||
}
|
||||
},
|
||||
[pushSignal],
|
||||
);
|
||||
|
||||
const appendServerClientReply = useCallback(
|
||||
(replyText: string | null, at = elapsed, options?: { suppressEmptyWarning?: boolean }) => {
|
||||
setClientReplyPending(false);
|
||||
|
|
@ -1450,6 +1477,8 @@ export default function Session() {
|
|||
setClientReplyPending(true);
|
||||
setAvatarState("thinking");
|
||||
setTurnError(null);
|
||||
stopTtsPlayback();
|
||||
void primeVoicePlayback();
|
||||
|
||||
let clientId: number | null = null;
|
||||
let clientReply = "";
|
||||
|
|
@ -1539,6 +1568,9 @@ export default function Session() {
|
|||
}
|
||||
setAvatarState("listening");
|
||||
if (!conversationStopped && !qualityRetryable) {
|
||||
if (clientReply && typeof done.turn_seq === "number") {
|
||||
void speakTextClientTurn(liveSessionId, done.turn_seq);
|
||||
}
|
||||
void requestLiveCoach({
|
||||
learnerText: text,
|
||||
clientReply,
|
||||
|
|
@ -1580,7 +1612,10 @@ export default function Session() {
|
|||
liveSessionId,
|
||||
elapsed,
|
||||
pushSignal,
|
||||
primeVoicePlayback,
|
||||
requestLiveCoach,
|
||||
speakTextClientTurn,
|
||||
stopTtsPlayback,
|
||||
]);
|
||||
|
||||
const onComposeKey = (e: ReactKeyboardEvent<HTMLTextAreaElement>) => {
|
||||
|
|
@ -1752,6 +1787,15 @@ export default function Session() {
|
|||
}
|
||||
}, [clientName, closeVoiceSocket, ensureVoiceAudioContext, stopTtsPlayback]);
|
||||
|
||||
useEffect(() => {
|
||||
playTtsAudioRef.current = playTtsAudio;
|
||||
return () => {
|
||||
if (playTtsAudioRef.current === playTtsAudio) {
|
||||
playTtsAudioRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [playTtsAudio]);
|
||||
|
||||
const startVoiceCapture = useCallback(async () => {
|
||||
if (!liveSessionId || paused || sending || sessionEnded) return;
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue