런타임 계약과 학습자 흐름 보강

This commit is contained in:
Yun Chan 2026-06-29 08:12:14 +09:00
parent f456b8997a
commit 206018b088
56 changed files with 4306 additions and 1008 deletions

View file

@ -84,6 +84,29 @@ interface VoiceEvent {
}
type AudioContextWindow = Window & { webkitAudioContext?: typeof AudioContext };
type VoiceCaptureMode = "audio-worklet" | "media-recorder";
interface VoiceCaptureControl {
type: "audio_start" | "audio_end";
format: string;
sample_rate?: number;
channels?: number;
sample_width?: number;
}
interface VoiceCaptureController {
mode: VoiceCaptureMode;
startControl: VoiceCaptureControl;
start: () => Promise<void> | void;
finish: () => void;
abort: () => void;
isRecording: () => boolean;
}
interface VoiceWorkletMessage {
type?: string;
pcm?: ArrayBuffer;
}
interface Utterance {
id: number;
@ -121,6 +144,8 @@ const THEORY_MODE_OPTIONS: { value: TheoryMode; label: string; detail: string }[
{ value: "cbt", label: "CBT", detail: "생각·행동" },
{ value: "integrative", label: "통합", detail: "혼합 접근" },
];
const VOICE_WORKLET_MODULE_URL = "/worklets/voice-capture-worklet.js";
const VOICE_WORKLET_PROCESSOR = "voice-capture-processor";
function preferredTheoryMode(summary: PersonaSummary | null): TheoryMode {
const firstSupported = summary?.theory_target.find(
@ -130,6 +155,131 @@ function preferredTheoryMode(summary: PersonaSummary | null): TheoryMode {
return firstSupported ?? "humanistic";
}
interface SessionVoiceStatusInput {
voiceStatus: VoiceStatus;
sessionEnded: boolean;
paused: boolean;
sending: boolean;
clientReplyPending: boolean;
clientName: string;
utteranceCount: number;
voiceAvailable: boolean | null;
micOn: boolean;
}
interface SessionVoiceStatusView {
micDisabled: boolean;
micLabel: string;
micButtonAriaLabel: string;
sessionStatusLabel: string;
transcriptLiveLabel: string;
transcriptIsLive: boolean;
textTurnBlocked: boolean;
voiceInputStatus: string;
responseStatus: string;
}
function isVoiceStatusBusy(voiceStatus: VoiceStatus): boolean {
return (
voiceStatus === "requesting" ||
voiceStatus === "connecting" ||
voiceStatus === "thinking" ||
voiceStatus === "speaking"
);
}
function sessionVoiceStatusView(input: SessionVoiceStatusInput): SessionVoiceStatusView {
const {
voiceStatus,
sessionEnded,
paused,
sending,
clientReplyPending,
clientName,
utteranceCount,
voiceAvailable,
micOn,
} = input;
const micBusy = isVoiceStatusBusy(voiceStatus);
const micDisabled = sessionEnded || paused || micBusy || voiceAvailable === false;
const micButtonAriaLabel = micOn ? "발화 보내기" : "마이크 켜기";
const micLabel = sessionEnded
? "종료됨"
: paused
? "일시정지"
: voiceStatus === "requesting"
? "권한 요청 중"
: voiceStatus === "connecting"
? "연결 중"
: voiceStatus === "recording"
? "녹음 중"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "재생 중"
: voiceStatus === "degraded"
? "음성 미설정"
: voiceStatus === "error"
? "마이크 오류"
: "마이크 꺼짐";
const sessionStatusLabel = sessionEnded
? "회기 종료됨"
: paused
? "일시정지"
: voiceStatus === "recording"
? "학습자 발화 수신 중"
: voiceStatus === "thinking"
? "내담자 응답 준비 중"
: voiceStatus === "speaking"
? `${clientName} 응답 중`
: "회기 진행 중";
const transcriptLiveLabel =
voiceStatus === "recording"
? "받아쓰는 중"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "응답 표시 중"
: sending
? "응답 대기"
: utteranceCount > 0
? "기록 중"
: "준비됨";
const transcriptIsLive =
voiceStatus === "recording" ||
voiceStatus === "thinking" ||
voiceStatus === "speaking" ||
sending;
const textTurnBlocked = sending || voiceStatus === "thinking";
const voiceInputStatus =
voiceAvailable === false
? "음성 미설정"
: micOn
? "수신 중"
: voiceStatus === "requesting"
? "권한 요청"
: "대기";
const responseStatus =
sending || clientReplyPending
? "응답 대기"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "재생 중"
: "안정";
return {
micDisabled,
micLabel,
micButtonAriaLabel,
sessionStatusLabel,
transcriptLiveLabel,
transcriptIsLive,
textTurnBlocked,
voiceInputStatus,
responseStatus,
};
}
const PERSONA_AVATAR_LOOKS = {
P1: {
skinTone: "#F0DDC4",
@ -290,6 +440,174 @@ function voiceFormatFromMime(mime: string): string {
return "webm";
}
function sendVoiceControl(ws: WebSocket, payload: VoiceCaptureControl): void {
if (ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify(payload));
}
}
function supportsAudioWorkletCapture(ctx: AudioContext | null): ctx is AudioContext & {
audioWorklet: AudioWorklet;
} {
return Boolean(
ctx &&
ctx.state !== "closed" &&
ctx.audioWorklet &&
typeof AudioWorkletNode !== "undefined" &&
typeof ctx.createMediaStreamSource === "function",
);
}
function createMediaRecorderCapture(
stream: MediaStream,
ws: WebSocket,
onRelease: () => void,
): VoiceCaptureController | null {
if (typeof MediaRecorder === "undefined") return null;
const mimeType = recorderMimeType();
const format = voiceFormatFromMime(mimeType);
const startControl: VoiceCaptureControl = { type: "audio_start", format };
const endControl: VoiceCaptureControl = { type: "audio_end", format };
let recorder: MediaRecorder;
try {
recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
} catch {
return null;
}
let finishRequested = false;
let released = false;
const release = () => {
if (released) return;
released = true;
onRelease();
};
recorder.ondataavailable = (event) => {
if (!event.data.size || ws.readyState !== WebSocket.OPEN) return;
ws.send(event.data);
};
recorder.onstop = () => {
release();
if (finishRequested) sendVoiceControl(ws, endControl);
};
return {
mode: "media-recorder",
startControl,
start: () => {
recorder.start(500);
},
finish: () => {
finishRequested = true;
if (recorder.state !== "inactive") {
try {
recorder.stop();
return;
} catch {
/* recorder stop race 무시 */
}
}
release();
sendVoiceControl(ws, endControl);
},
abort: () => {
finishRequested = false;
if (recorder.state !== "inactive") {
try {
recorder.stop();
} catch {
/* recorder stop race 무시 */
}
}
release();
},
isRecording: () => recorder.state === "recording",
};
}
async function createAudioWorkletCapture(
stream: MediaStream,
ws: WebSocket,
ctx: AudioContext & { audioWorklet: AudioWorklet },
onRelease: () => void,
): Promise<VoiceCaptureController> {
await ctx.audioWorklet.addModule(VOICE_WORKLET_MODULE_URL);
const source = ctx.createMediaStreamSource(stream);
const node = new AudioWorkletNode(ctx, VOICE_WORKLET_PROCESSOR, {
numberOfInputs: 1,
numberOfOutputs: 0,
});
const startControl: VoiceCaptureControl = {
type: "audio_start",
format: "pcm",
sample_rate: Math.round(ctx.sampleRate || 48000),
channels: 1,
sample_width: 2,
};
const endControl: VoiceCaptureControl = { ...startControl, type: "audio_end" };
let recording = false;
let released = false;
let finishTimer: number | null = null;
const release = () => {
if (released) return;
released = true;
if (finishTimer !== null) {
window.clearTimeout(finishTimer);
finishTimer = null;
}
try {
source.disconnect();
node.disconnect();
} catch {
/* disconnect race 무시 */
}
node.port.onmessage = null;
try {
node.port.close();
} catch {
/* 일부 브라우저/테스트 double은 close가 없을 수 있다. */
}
onRelease();
};
node.port.onmessage = (event: MessageEvent<VoiceWorkletMessage>) => {
if (!recording || ws.readyState !== WebSocket.OPEN) return;
const payload = event.data;
if (payload?.type !== "chunk" || !payload.pcm?.byteLength) return;
ws.send(payload.pcm);
};
return {
mode: "audio-worklet",
startControl,
start: () => {
recording = true;
source.connect(node);
},
finish: () => {
if (!recording || finishTimer !== null) return;
try {
node.port.postMessage({ type: "flush" });
} catch {
/* flush 실패 시 이미 보낸 chunk만 사용한다. */
}
finishTimer = window.setTimeout(() => {
finishTimer = null;
recording = false;
release();
sendVoiceControl(ws, endControl);
}, 20);
},
abort: () => {
recording = false;
release();
},
isRecording: () => recording,
};
}
/* ── 작은 표현 유틸 ─────────────────────────────────────────────────── */
// 백엔드가 반환한 effective_openness(0~1)를 실시간 관찰 신호로만 변환한다.
@ -446,7 +764,7 @@ export default function Session() {
const fadeTimerRef = useRef<number | null>(null);
const voiceSocketRef = useRef<WebSocket | null>(null);
const recorderRef = useRef<MediaRecorder | null>(null);
const voiceCaptureRef = useRef<VoiceCaptureController | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const audioContextRef = useRef<AudioContext | null>(null);
const ttsChunksRef = useRef<BlobPart[]>([]);
@ -1004,15 +1322,9 @@ export default function Session() {
const shutdownVoice = useCallback(
(status: VoiceStatus = "idle", detail = "마이크 꺼짐") => {
stopTtsPlayback();
const recorder = recorderRef.current;
recorderRef.current = null;
if (recorder && recorder.state !== "inactive") {
try {
recorder.stop();
} catch {
/* recorder stop race 무시 */
}
}
const capture = voiceCaptureRef.current;
voiceCaptureRef.current = null;
capture?.abort();
stopMicStream();
closeVoiceSocket();
setMicOn(false);
@ -1024,13 +1336,10 @@ export default function Session() {
);
const finishVoiceUtterance = useCallback(() => {
const recorder = recorderRef.current;
if (recorder && recorder.state !== "inactive") {
try {
recorder.stop();
} catch {
stopMicStream();
}
const capture = voiceCaptureRef.current;
voiceCaptureRef.current = null;
if (capture) {
capture.finish();
} else {
stopMicStream();
}
@ -1155,7 +1464,7 @@ export default function Session() {
const startVoiceCapture = useCallback(async () => {
if (!liveSessionId || paused || sending || sessionEnded) return;
if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === "undefined") {
if (!navigator.mediaDevices?.getUserMedia) {
setVoiceStatus("error");
setVoiceDetail("이 브라우저는 마이크 녹음을 지원하지 않습니다.");
pushSignal("warn", "마이크 미지원");
@ -1181,8 +1490,6 @@ export default function Session() {
setVoiceStatus("connecting");
setVoiceDetail("음성 연결을 준비하는 중입니다.");
const mimeType = recorderMimeType();
const format = voiceFormatFromMime(mimeType);
const ws = new WebSocket(
apiWsUrl(`/voice/ws?session_id=${encodeURIComponent(liveSessionId)}`),
);
@ -1190,33 +1497,48 @@ export default function Session() {
voiceSocketRef.current = ws;
ttsChunksRef.current = [];
let recorder: MediaRecorder;
try {
recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
} catch {
stopMicStream();
closeVoiceSocket();
setVoiceStatus("error");
setVoiceDetail("마이크 녹음기를 만들지 못했습니다. 브라우저 오디오 설정을 확인해 주세요.");
return;
}
recorderRef.current = recorder;
setClientReplyPending(false);
recorder.ondataavailable = (event) => {
if (!event.data.size || ws.readyState !== WebSocket.OPEN) return;
ws.send(event.data);
};
recorder.onstop = () => {
stopMicStream();
if (ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({ type: "audio_end", format }));
let pendingCapture: VoiceCaptureController | null = null;
const capturePromise = (async () => {
const ctx = ensureVoiceAudioContext();
if (supportsAudioWorkletCapture(ctx)) {
try {
pendingCapture = await createAudioWorkletCapture(stream, ws, ctx, stopMicStream);
return pendingCapture;
} catch {
pendingCapture = null;
}
}
};
pendingCapture = createMediaRecorderCapture(stream, ws, stopMicStream);
return pendingCapture;
})();
ws.onopen = () => {
ws.send(JSON.stringify({ type: "audio_start", format }));
recorder.start(500);
ws.onopen = async () => {
const capture = await capturePromise;
if (ws.readyState !== WebSocket.OPEN) {
capture?.abort();
return;
}
if (!capture) {
stopMicStream();
closeVoiceSocket();
setVoiceStatus("error");
setVoiceDetail("마이크 녹음기를 만들지 못했습니다. 브라우저 오디오 설정을 확인해 주세요.");
return;
}
voiceCaptureRef.current = capture;
sendVoiceControl(ws, capture.startControl);
try {
await capture.start();
} catch {
voiceCaptureRef.current = null;
capture.abort();
closeVoiceSocket();
setVoiceStatus("error");
setVoiceDetail("마이크 녹음기를 시작하지 못했습니다. 브라우저 오디오 설정을 확인해 주세요.");
return;
}
setMicOn(true);
setAvatarState("listening");
setVoiceStatus("recording");
@ -1252,7 +1574,7 @@ export default function Session() {
setAvatarState("speaking");
setVoiceStatus("speaking");
setVoiceDetail(`${clientName} 음성을 받는 중입니다.`);
} else if (payload.state === "idle" && recorderRef.current?.state !== "recording") {
} else if (payload.state === "idle" && !voiceCaptureRef.current?.isRecording()) {
setAvatarState("listening");
setVoiceStatus("idle");
setVoiceDetail("마이크를 다시 켜 발화하세요.");
@ -1341,7 +1663,13 @@ export default function Session() {
};
ws.onclose = () => {
recorderRef.current = null;
const capture = voiceCaptureRef.current;
voiceCaptureRef.current = null;
capture?.abort();
if (pendingCapture && pendingCapture !== capture) {
pendingCapture.abort();
pendingCapture = null;
}
stopMicStream();
setMicOn(false);
setVoiceStatus((current) => (current === "degraded" || current === "error" ? current : "idle"));
@ -1361,6 +1689,7 @@ export default function Session() {
appendServerClientReply,
closeVoiceSocket,
elapsed,
ensureVoiceAudioContext,
liveSessionId,
paused,
sessionEnded,
@ -1390,12 +1719,7 @@ export default function Session() {
pushSignal("warn", "음성 미설정 — 텍스트로 진행");
return;
}
if (
voiceStatus === "requesting" ||
voiceStatus === "connecting" ||
voiceStatus === "thinking" ||
voiceStatus === "speaking"
) {
if (isVoiceStatusBusy(voiceStatus)) {
return;
}
if (micOn || voiceStatus === "recording") {
@ -1557,59 +1881,27 @@ export default function Session() {
? "thinking"
: "idle";
const micBusy =
voiceStatus === "requesting" ||
voiceStatus === "connecting" ||
voiceStatus === "thinking" ||
voiceStatus === "speaking";
const micLabel = sessionEnded
? "종료됨"
: paused
? "일시정지"
: voiceStatus === "requesting"
? "권한 요청 중"
: voiceStatus === "connecting"
? "연결 중"
: voiceStatus === "recording"
? "녹음 중"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "재생 중"
: voiceStatus === "degraded"
? "음성 미설정"
: voiceStatus === "error"
? "마이크 오류"
: "마이크 꺼짐";
const sessionStatusLabel = sessionEnded
? "회기 종료됨"
: paused
? "일시정지"
: voiceStatus === "recording"
? "학습자 발화 수신 중"
: voiceStatus === "thinking"
? "내담자 응답 준비 중"
: voiceStatus === "speaking"
? `${clientName} 응답 중`
: "회기 진행 중";
const transcriptLiveLabel =
voiceStatus === "recording"
? "받아쓰는 중"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "응답 표시 중"
: sending
? "응답 대기"
: utterances.length > 0
? "기록 중"
: "준비됨";
const transcriptIsLive =
voiceStatus === "recording" ||
voiceStatus === "thinking" ||
voiceStatus === "speaking" ||
sending;
const textTurnBlocked = sending || voiceStatus === "thinking";
const {
micDisabled,
micLabel,
micButtonAriaLabel,
sessionStatusLabel,
transcriptLiveLabel,
transcriptIsLive,
textTurnBlocked,
voiceInputStatus,
responseStatus,
} = sessionVoiceStatusView({
voiceStatus,
sessionEnded,
paused,
sending,
clientReplyPending,
clientName,
utteranceCount: utterances.length,
voiceAvailable,
micOn,
});
const elapsedLabel = formatElapsed(elapsed);
const recommendedSessionSeconds = 30 * 60;
const remainingLabel = formatElapsed(Math.max(0, recommendedSessionSeconds - elapsed));
@ -1621,22 +1913,6 @@ export default function Session() {
latestClientUtterance?.text ??
primaryContext?.v ??
`${clientName}님이 당신의 첫 질문을 기다리고 있습니다.`;
const voiceInputStatus =
voiceAvailable === false
? "음성 미설정"
: micOn
? "수신 중"
: voiceStatus === "requesting"
? "권한 요청"
: "대기";
const responseStatus =
sending || clientReplyPending
? "응답 대기"
: voiceStatus === "thinking"
? "전사 중"
: voiceStatus === "speaking"
? "재생 중"
: "안정";
const safetyStatusText = safety ? "확인 필요" : "안전";
const personaIsUsable = personaSummary ? isUsablePersona(personaSummary) : false;
const consentRequired = user?.role === "learner" && user.consentAt == null;
@ -2494,9 +2770,9 @@ export default function Session() {
type="button"
className={"sx-mic " + (micOn ? "is-on" : "is-off")}
onClick={toggleMic}
disabled={sessionEnded || paused || micBusy || voiceAvailable === false}
disabled={micDisabled}
aria-pressed={micOn}
aria-label={micOn ? "발화 보내기" : "마이크 켜기"}
aria-label={micButtonAriaLabel}
>
<Icon name={micOn ? "mic" : "mic-off"} size={22} />
</button>