세션 평가와 교수자 분석 보강
This commit is contained in:
Yun Chan
parent
5c4ac04e06
commit
fe2796f05a
51 changed files with 4928 additions and 240 deletions
|
|
@ -86,12 +86,22 @@ interface VoiceEvent {
|
|||
type AudioContextWindow = Window & { webkitAudioContext?: typeof AudioContext };
|
||||
type VoiceCaptureMode = "audio-worklet" | "media-recorder";
|
||||
|
||||
interface VoiceProviderEvent {
|
||||
type: string;
|
||||
source: string;
|
||||
duration_ms?: number;
|
||||
confidence?: number;
|
||||
}
|
||||
|
||||
interface VoiceCaptureControl {
|
||||
type: "audio_start" | "audio_end";
|
||||
format: string;
|
||||
sample_rate?: number;
|
||||
channels?: number;
|
||||
sample_width?: number;
|
||||
silence_ms?: number;
|
||||
barge_in?: boolean;
|
||||
provider_events?: VoiceProviderEvent[];
|
||||
}
|
||||
|
||||
interface VoiceCaptureController {
|
||||
|
|
@ -106,6 +116,16 @@ interface VoiceCaptureController {
|
|||
interface VoiceWorkletMessage {
|
||||
type?: string;
|
||||
pcm?: ArrayBuffer;
|
||||
metrics?: VoiceCaptureMetrics;
|
||||
}
|
||||
|
||||
interface VoiceCaptureMetrics {
|
||||
durationMs?: number;
|
||||
voiceMs?: number;
|
||||
silenceMs?: number;
|
||||
trailingSilenceMs?: number;
|
||||
rms?: number;
|
||||
peak?: number;
|
||||
}
|
||||
|
||||
interface Utterance {
|
||||
|
|
@ -446,6 +466,59 @@ function sendVoiceControl(ws: WebSocket, payload: VoiceCaptureControl): void {
|
|||
}
|
||||
}
|
||||
|
||||
function roundedNonnegativeMs(value: number | undefined): number | undefined {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return undefined;
|
||||
return Math.max(0, Math.round(value));
|
||||
}
|
||||
|
||||
function voiceActivityConfidence(metrics: VoiceCaptureMetrics): number | undefined {
|
||||
const peak = typeof metrics.peak === "number" && Number.isFinite(metrics.peak) ? metrics.peak : 0;
|
||||
if (peak <= 0) return undefined;
|
||||
return Math.max(0.01, Math.min(0.99, Number(peak.toFixed(2))));
|
||||
}
|
||||
|
||||
function voiceProviderEventsFromMetrics(
|
||||
metrics: VoiceCaptureMetrics | null,
|
||||
source: string,
|
||||
): VoiceProviderEvent[] {
|
||||
if (!metrics) return [];
|
||||
const durationMs = roundedNonnegativeMs(metrics.durationMs);
|
||||
const voiceMs = roundedNonnegativeMs(metrics.voiceMs);
|
||||
const trailingSilenceMs = roundedNonnegativeMs(metrics.trailingSilenceMs);
|
||||
const events: VoiceProviderEvent[] = [];
|
||||
if (voiceMs !== undefined) {
|
||||
const activity: VoiceProviderEvent = {
|
||||
type: "voice_activity",
|
||||
source,
|
||||
duration_ms: voiceMs,
|
||||
};
|
||||
const confidence = voiceActivityConfidence(metrics);
|
||||
if (confidence !== undefined) activity.confidence = confidence;
|
||||
events.push(activity);
|
||||
} else if (durationMs !== undefined) {
|
||||
events.push({ type: "voice_activity", source, duration_ms: durationMs });
|
||||
}
|
||||
if (trailingSilenceMs !== undefined && trailingSilenceMs >= 1000) {
|
||||
events.push({ type: "silence", source, duration_ms: trailingSilenceMs });
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function audioEndControlWithMetrics(
|
||||
base: VoiceCaptureControl,
|
||||
metrics: VoiceCaptureMetrics | null,
|
||||
source: string,
|
||||
): VoiceCaptureControl {
|
||||
const silenceMs = roundedNonnegativeMs(metrics?.trailingSilenceMs);
|
||||
const providerEvents = voiceProviderEventsFromMetrics(metrics, source);
|
||||
return {
|
||||
...base,
|
||||
...(silenceMs !== undefined ? { silence_ms: silenceMs } : {}),
|
||||
barge_in: false,
|
||||
...(providerEvents.length ? { provider_events: providerEvents } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function supportsAudioWorkletCapture(ctx: AudioContext | null): ctx is AudioContext & {
|
||||
audioWorklet: AudioWorklet;
|
||||
} {
|
||||
|
|
@ -477,6 +550,7 @@ function createMediaRecorderCapture(
|
|||
|
||||
let finishRequested = false;
|
||||
let released = false;
|
||||
let startedAt = 0;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
|
|
@ -489,13 +563,20 @@ function createMediaRecorderCapture(
|
|||
};
|
||||
recorder.onstop = () => {
|
||||
release();
|
||||
if (finishRequested) sendVoiceControl(ws, endControl);
|
||||
if (finishRequested) {
|
||||
const durationMs = startedAt > 0 ? performance.now() - startedAt : undefined;
|
||||
sendVoiceControl(
|
||||
ws,
|
||||
audioEndControlWithMetrics(endControl, { durationMs, voiceMs: durationMs }, "browser_media_recorder"),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
mode: "media-recorder",
|
||||
startControl,
|
||||
start: () => {
|
||||
startedAt = performance.now();
|
||||
recorder.start(500);
|
||||
},
|
||||
finish: () => {
|
||||
|
|
@ -509,7 +590,11 @@ function createMediaRecorderCapture(
|
|||
}
|
||||
}
|
||||
release();
|
||||
sendVoiceControl(ws, endControl);
|
||||
const durationMs = startedAt > 0 ? performance.now() - startedAt : undefined;
|
||||
sendVoiceControl(
|
||||
ws,
|
||||
audioEndControlWithMetrics(endControl, { durationMs, voiceMs: durationMs }, "browser_media_recorder"),
|
||||
);
|
||||
},
|
||||
abort: () => {
|
||||
finishRequested = false;
|
||||
|
|
@ -549,6 +634,7 @@ async function createAudioWorkletCapture(
|
|||
let recording = false;
|
||||
let released = false;
|
||||
let finishTimer: number | null = null;
|
||||
let latestMetrics: VoiceCaptureMetrics | null = null;
|
||||
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
|
|
@ -576,6 +662,7 @@ async function createAudioWorkletCapture(
|
|||
if (!recording || ws.readyState !== WebSocket.OPEN) return;
|
||||
const payload = event.data;
|
||||
if (payload?.type !== "chunk" || !payload.pcm?.byteLength) return;
|
||||
latestMetrics = payload.metrics ?? latestMetrics;
|
||||
ws.send(payload.pcm);
|
||||
};
|
||||
|
||||
|
|
@ -597,7 +684,7 @@ async function createAudioWorkletCapture(
|
|||
finishTimer = null;
|
||||
recording = false;
|
||||
release();
|
||||
sendVoiceControl(ws, endControl);
|
||||
sendVoiceControl(ws, audioEndControlWithMetrics(endControl, latestMetrics, "browser_audio_worklet"));
|
||||
}, 20);
|
||||
},
|
||||
abort: () => {
|
||||
|
|
@ -1135,11 +1222,13 @@ export default function Session() {
|
|||
}, [acceptConsent, consentChecked, pushSignal]);
|
||||
|
||||
const appendServerClientReply = useCallback(
|
||||
(replyText: string | null, at = elapsed) => {
|
||||
(replyText: string | null, at = elapsed, options?: { suppressEmptyWarning?: boolean }) => {
|
||||
setClientReplyPending(false);
|
||||
if (!replyText) {
|
||||
setAvatarState("listening");
|
||||
pushSignal("warn", "내담자 응답 없음");
|
||||
if (!options?.suppressEmptyWarning) {
|
||||
pushSignal("warn", "내담자 응답 없음");
|
||||
}
|
||||
return;
|
||||
}
|
||||
setUtterances((prev) => [
|
||||
|
|
@ -1180,6 +1269,7 @@ export default function Session() {
|
|||
|
||||
let clientId: number | null = null;
|
||||
let clientReply = "";
|
||||
let conversationStopped = false;
|
||||
try {
|
||||
const done = await sessionApi.stream(liveSessionId, text, {
|
||||
onToken: (chunk) => {
|
||||
|
|
@ -1210,6 +1300,7 @@ export default function Session() {
|
|||
setSafety("방금 발화에서 위기 신호가 감지됐어요. 안전을 우선해 머물러 주세요.");
|
||||
}
|
||||
if (data.conversation_stopped || data.crisis_resource) {
|
||||
conversationStopped = true;
|
||||
applyCrisisGate(data.crisis_resource);
|
||||
}
|
||||
},
|
||||
|
|
@ -1218,10 +1309,16 @@ export default function Session() {
|
|||
if (payload && typeof payload === "object") {
|
||||
const resource = (payload as { crisis_resource?: CrisisResource }).crisis_resource;
|
||||
const stopped = (payload as { conversation_stopped?: boolean }).conversation_stopped;
|
||||
if (resource || stopped) applyCrisisGate(resource);
|
||||
if (resource || stopped) {
|
||||
conversationStopped = true;
|
||||
applyCrisisGate(resource);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
if (done.conversation_stopped || done.crisis_resource) {
|
||||
conversationStopped = true;
|
||||
}
|
||||
|
||||
if (clientId != null) {
|
||||
const id = clientId;
|
||||
|
|
@ -1247,14 +1344,18 @@ export default function Session() {
|
|||
setUtterances((prev) =>
|
||||
prev.map((u) => (u.id === learnerId ? { ...u, turnSeq: done.turn_seq } : u)),
|
||||
);
|
||||
pushSignal("warn", "내담자 응답 없음");
|
||||
if (!conversationStopped) {
|
||||
pushSignal("warn", "내담자 응답 없음");
|
||||
}
|
||||
}
|
||||
setAvatarState("listening");
|
||||
void requestLiveCoach({
|
||||
learnerText: text,
|
||||
clientReply,
|
||||
turnSeq: done.turn_seq,
|
||||
});
|
||||
if (!conversationStopped) {
|
||||
void requestLiveCoach({
|
||||
learnerText: text,
|
||||
clientReply,
|
||||
turnSeq: done.turn_seq,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
setComposeText(text);
|
||||
setClientReplyPending(false);
|
||||
|
|
@ -1606,6 +1707,7 @@ export default function Session() {
|
|||
if (payload.conversation_stopped || payload.crisis_resource) {
|
||||
applyCrisisGate(payload.crisis_resource);
|
||||
}
|
||||
const conversationStopped = !!(payload.conversation_stopped || payload.crisis_resource);
|
||||
const pendingId = pendingVoiceLearnerIdRef.current;
|
||||
if (pendingId != null) {
|
||||
setUtterances((prev) =>
|
||||
|
|
@ -1615,10 +1717,12 @@ export default function Session() {
|
|||
);
|
||||
pendingVoiceLearnerIdRef.current = null;
|
||||
}
|
||||
appendServerClientReply(payload.text ?? null);
|
||||
appendServerClientReply(payload.text ?? null, elapsed, {
|
||||
suppressEmptyWarning: conversationStopped,
|
||||
});
|
||||
const learnerText = pendingVoiceLearnerTextRef.current;
|
||||
pendingVoiceLearnerTextRef.current = "";
|
||||
if (learnerText) {
|
||||
if (learnerText && !conversationStopped) {
|
||||
void requestLiveCoach({
|
||||
learnerText,
|
||||
clientReply: payload.text ?? null,
|
||||
|
|
@ -1830,27 +1934,37 @@ export default function Session() {
|
|||
endPreviousFocusRef.current = null;
|
||||
}, [endDialogOpen, ending]);
|
||||
|
||||
// 회기 종료 — 명시 확인 후 end 호출, 현재 화면을 정지 상태로 둔다.
|
||||
// 회기 종료 — 명시 확인 후 end 호출, 실패하면 리뷰 화면으로 넘어가지 않는다.
|
||||
const handleEnd = useCallback(async () => {
|
||||
if (ending || sessionEnded) return;
|
||||
setEnding(true);
|
||||
setTurnError(null);
|
||||
try {
|
||||
if (liveSessionId) await sessionApi.end(liveSessionId);
|
||||
} catch {
|
||||
/* 종료 실패는 UI 종료 처리를 막지 않는다. */
|
||||
if (!liveSessionId) {
|
||||
throw new Error("회기 ID가 없습니다.");
|
||||
}
|
||||
await sessionApi.end(liveSessionId);
|
||||
} catch (err) {
|
||||
const detail =
|
||||
err instanceof ApiError
|
||||
? err.detail
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: "알 수 없는 오류";
|
||||
setEndDialogOpen(false);
|
||||
setTurnError(`회기 종료에 실패했습니다. ${detail}`);
|
||||
pushSignal("warn", "회기 종료 실패");
|
||||
setEnding(false);
|
||||
return;
|
||||
}
|
||||
setEndDialogOpen(false);
|
||||
setSessionEnded(true);
|
||||
setReviewReady(true);
|
||||
setReviewReady(false);
|
||||
setMicOn(false);
|
||||
setPaused(true);
|
||||
setAvatarState("idle");
|
||||
pushSignal("neutral", "회기 종료");
|
||||
if (liveSessionId) {
|
||||
navigate(`/learn/session/${liveSessionId}/review`, { replace: true });
|
||||
} else {
|
||||
setEnding(false);
|
||||
}
|
||||
navigate(`/learn/session/${liveSessionId}/review`, { replace: true });
|
||||
}, [ending, liveSessionId, navigate, pushSignal, sessionEnded]);
|
||||
|
||||
const meters = metersFromOpenness(openness);
|
||||
|
|
@ -1914,6 +2028,7 @@ export default function Session() {
|
|||
primaryContext?.v ??
|
||||
`${clientName}님이 당신의 첫 질문을 기다리고 있습니다.`;
|
||||
const safetyStatusText = safety ? "확인 필요" : "안전";
|
||||
const turnErrorTitle = turnError?.startsWith("회기 종료") ? "회기 종료" : "엔진 연결";
|
||||
const personaIsUsable = personaSummary ? isUsablePersona(personaSummary) : false;
|
||||
const consentRequired = user?.role === "learner" && user.consentAt == null;
|
||||
const canStartSession = personaLoadState === "ready" && personaIsUsable && !consentRequired;
|
||||
|
|
@ -2526,7 +2641,7 @@ export default function Session() {
|
|||
<Icon name="alert" size={17} />
|
||||
</span>
|
||||
<span className="sx-safety__text">
|
||||
<b>엔진 연결</b> · {turnError}
|
||||
<b>{turnErrorTitle}</b> · {turnError}
|
||||
</span>
|
||||
</section>
|
||||
) : null}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue