/* ===================================================================== AvatarLab — 아바타 v3 표현 엔진 dev Lab (무인증). /dev/avatar-lab. 실 서비스 라우트 아님 — 채널·표정·반응 클립·지문 파싱·엔진 상태를 눈으로 확인하는 도구다(결정문 §9 1단계 완료 기준). ===================================================================== */ import { useEffect, useMemo, useRef, useState, type ChangeEvent } from "react"; import DebugFace from "../components/avatar/v3/DebugFace"; import LinocutAvatar from "../components/avatar/v3/LinocutAvatar"; import LinocutMotif from "../components/avatar/v3/LinocutMotif"; import { labLinocutRigs } from "../components/avatar/v3/rigs"; import type { LinocutRig } from "../components/avatar/v3/linocutRig"; import { CHANNEL_IDS } from "../components/avatar/engine/channels"; import { createAvatarEngine, type AvatarEngine, type DebugSnapshot } from "../components/avatar/engine/engine"; import { REACTION_CLIPS, REACTION_CLIP_IDS, type ReactionClipId } from "../components/avatar/engine/clipCatalog"; import { demeanorFor } from "../components/avatar/engine/demeanorDefaults"; import type { Performance, PerformanceCue } from "../components/avatar/engine/performance"; import type { PhraseKind, SpeechShape, VisemeId } from "../components/avatar/engine/lipsync"; import { startSpeech, type SpeechHandle } from "../components/avatar/engine/speechDriver"; import { AVATAR_EXPRESSION_LIBRARY, type AvatarExpression, type AvatarState } from "../components/avatar/persona"; import "./avatar-lab.css"; const PERSONA_OPTIONS: Array<{ code: string; label: string }> = [ { code: "P1", label: "P1 서연" }, { code: "P2", label: "P2 민재" }, { code: "P3", label: "P3 지우" }, { code: "P4", label: "P4 하늘" }, { code: "P5", label: "P5 도윤" }, { code: "P6", label: "P6 하린" }, { code: "P7", label: "P7 도현" }, ]; /** 리노컷 리그가 있는 페르소나만. 없는 페르소나는 debug 렌더러를 강제한다. 공통 파이프라인이 만든 pNRig.ts를 전부 모은 값이라, 새 페르소나가 빌드되는 즉시(재빌드 없이) 이 Lab에서 보인다 — v3/rigs/index.ts labLinocutRigs() 참고. */ const LINOCUT_RIGS: Partial> = labLinocutRigs(); type RendererMode = "linocut" | "debug"; type FeedbackMode = "immersive" | "ambient" | "coached"; const STATE_OPTIONS: Array<{ id: AvatarState; label: string }> = [ { id: "idle", label: "대기" }, { id: "listening", label: "경청" }, { id: "thinking", label: "생각" }, { id: "speaking", label: "발화" }, ]; interface Clock { now(): number; pause(): void; resume(): void; isPaused(): boolean; } function createClock(): Clock { let offset = 0; let paused = false; let frozen = 0; return { now(): number { return paused ? frozen : performance.now() - offset; }, pause(): void { if (paused) return; frozen = performance.now() - offset; paused = true; }, resume(): void { if (!paused) return; offset = performance.now() - frozen; paused = false; }, isPaused(): boolean { return paused; }, }; } function seedFromQuery(): number { const raw = new URLSearchParams(window.location.search).get("seed"); const parsed = raw === null ? NaN : Number(raw); return Number.isFinite(parsed) ? parsed : 1; } /** ?persona=P3 같은 쿼리로 처음 고를 페르소나를 정한다(캡처 도구용). 모르는 코드거나 없으면 기본 P1. */ function personaFromQuery(): string { const raw = new URLSearchParams(window.location.search).get("persona"); const upper = raw?.toUpperCase() ?? ""; return PERSONA_OPTIONS.some((p) => p.code === upper) ? upper : "P1"; } export default function AvatarLab() { const seed = useMemo(seedFromQuery, []); const clock = useRef(createClock()).current; const [personaCode, setPersonaCode] = useState(personaFromQuery); const [avatarState, setAvatarState] = useState("idle"); const [openness, setOpenness] = useState(0.4); const [reducedMotion, setReducedMotion] = useState(false); const [paused, setPaused] = useState(false); const [selectedExpression, setSelectedExpression] = useState("neutral"); const [intensity, setIntensity] = useState(0.7); const [mirrorClip, setMirrorClip] = useState(false); const rig = LINOCUT_RIGS[personaCode]; const [rendererMode, setRendererMode] = useState(rig ? "linocut" : "debug"); const [feedbackMode, setFeedbackMode] = useState("coached"); const [linocutLoadState, setLinocutLoadState] = useState<"loading" | "ready" | "error">("loading"); useEffect(() => { if (!rig) setRendererMode("debug"); }, [rig]); const backdropGroup = useMemo( () => AVATAR_EXPRESSION_LIBRARY.find((e) => e.id === selectedExpression)?.group ?? "cognitive", [selectedExpression], ); const [speechText, setSpeechText] = useState("(한숨) 그냥요. 별로 할 말 없어요."); const [parsedCues, setParsedCues] = useState([]); const [unmatched, setUnmatched] = useState([]); const [snapshot, setSnapshot] = useState(null); const speechHandleRef = useRef(null); const meterRefs = useRef>({}); /* 발화층(립싱크) — 원시 SpeechShape·비짐 표시용. 채널 미터와 달리 매 프레임 갱신하지 않고(React state 금지, 결정문 §3) 아래 100ms interval에서 ref로 직접 쓴다. */ const speechDisplayRef = useRef<{ viseme: VisemeId; shape: SpeechShape }>({ viseme: "X", shape: { open: 0, wide: 0, round: 0, press: 0 }, }); const speechMeterRefs = useRef<{ viseme: HTMLElement | null; open: HTMLElement | null; wide: HTMLElement | null; round: HTMLElement | null; press: HTMLElement | null; }>({ viseme: null, open: null, wide: null, round: null, press: null }); /* 발화 동반층(§5.5) 표시용 — 현재 구 종류·강세 박 여부. speechDisplayRef와 같은 이유로 React state가 아니라 ref로 갱신하고 100ms interval에서 DOM에 직접 쓴다. */ const coSpeechDisplayRef = useRef<{ phraseKind: PhraseKind | null; stressed: boolean }>({ phraseKind: null, stressed: false }); const coSpeechMeterRefs = useRef<{ phraseKind: HTMLElement | null; stressed: HTMLElement | null }>({ phraseKind: null, stressed: null, }); const audioCtxRef = useRef(null); const audioSourceRef = useRef(null); const audioFileInputRef = useRef(null); const [audioFileName, setAudioFileName] = useState(null); const engine: AvatarEngine = useMemo( () => createAvatarEngine({ demeanor: demeanorFor(personaCode), seed, reducedMotion }), [personaCode, seed, reducedMotion], ); useEffect(() => { engine.setState(avatarState, clock.now()); engine.setOpenness(openness); engine.setSurface(selectedExpression, intensity); }, [engine]); // eslint-disable-line react-hooks/exhaustive-deps useEffect(() => { engine.setState(avatarState, clock.now()); }, [engine, avatarState]); // eslint-disable-line react-hooks/exhaustive-deps useEffect(() => { engine.setOpenness(openness); }, [engine, openness]); useEffect(() => { engine.setSurface(selectedExpression, intensity); }, [engine, selectedExpression, intensity]); useEffect(() => { setSnapshot(engine.debugSnapshot()); const interval = window.setInterval(() => { const v = engine.evaluate(clock.now()); for (const id of CHANNEL_IDS) { const cell = meterRefs.current[id]; if (cell) cell.textContent = v[id].toFixed(2); } setSnapshot(engine.debugSnapshot()); const sd = speechDisplayRef.current; const sm = speechMeterRefs.current; if (sm.viseme) sm.viseme.textContent = sd.viseme; if (sm.open) sm.open.textContent = sd.shape.open.toFixed(2); if (sm.wide) sm.wide.textContent = sd.shape.wide.toFixed(2); if (sm.round) sm.round.textContent = sd.shape.round.toFixed(2); if (sm.press) sm.press.textContent = sd.shape.press.toFixed(2); const cd = coSpeechDisplayRef.current; const cm = coSpeechMeterRefs.current; if (cm.phraseKind) cm.phraseKind.textContent = cd.phraseKind ?? "-"; if (cm.stressed) cm.stressed.textContent = cd.stressed ? "예" : "아니오"; }, 100); return () => window.clearInterval(interval); }, [engine]); // eslint-disable-line react-hooks/exhaustive-deps useEffect(() => { return () => { speechHandleRef.current?.stop(); try { audioSourceRef.current?.stop(); } catch { /* 이미 정지된 소스면 무시한다. */ } }; }, []); function togglePause(): void { if (clock.isPaused()) { clock.resume(); setPaused(false); } else { clock.pause(); setPaused(true); } } function handleClipClick(id: ReactionClipId): void { engine.triggerClip(id, clock.now(), { mirror: mirrorClip }); } const ZERO_SPEECH_SHAPE: SpeechShape = { open: 0, wide: 0, round: 0, press: 0 }; function endSpeech(nowMs: number): void { engine.setSpeechShape(null); engine.setSpeechMotion(null); engine.setState("listening", nowMs); setAvatarState("listening"); speechDisplayRef.current = { viseme: "X", shape: ZERO_SPEECH_SHAPE }; coSpeechDisplayRef.current = { phraseKind: null, stressed: false }; } function playSpeech(): void { speechHandleRef.current?.stop(); const nowMs = clock.now(); const speech = demeanorFor(personaCode).speech; setAvatarState("speaking"); engine.setState("speaking", nowMs); const handle = startSpeech({ engine, text: speechText, expression: selectedExpression, intensity, openness, seed, speech, now: clock.now, onFrame: ({ viseme, shape, phraseKind, stressed }) => { speechDisplayRef.current = { viseme, shape }; coSpeechDisplayRef.current = { phraseKind, stressed }; }, onEnd: () => endSpeech(clock.now()), }); speechHandleRef.current = handle; setParsedCues(handle.cues); setUnmatched(handle.unmatched); } function handleAudioFileChange(e: ChangeEvent): void { setAudioFileName(e.target.files?.[0]?.name ?? null); } /** "오디오 파일로 말하기" — 로컬 오디오를 디코드해 speechDriver에 넘긴다(포락선 선분석· 오디오 시계 표본은 startSpeech 안에서 한다). */ async function playSpeechWithAudioFile(): Promise { const file = audioFileInputRef.current?.files?.[0]; if (!file) return; const AudioContextCtor = window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; if (!AudioContextCtor) return; const ctx = audioCtxRef.current ?? new AudioContextCtor(); audioCtxRef.current = ctx; if (ctx.state === "suspended") await ctx.resume(); const arrayBuffer = await file.arrayBuffer(); const audioBuffer = await ctx.decodeAudioData(arrayBuffer.slice(0)); try { audioSourceRef.current?.stop(); } catch { /* 이미 정지된 소스면 무시한다. */ } const source = ctx.createBufferSource(); source.buffer = audioBuffer; source.connect(ctx.destination); audioSourceRef.current = source; const startAtCtx = ctx.currentTime + 0.3; source.start(startAtCtx); speechHandleRef.current?.stop(); const nowMs = clock.now(); const speech = demeanorFor(personaCode).speech; setAvatarState("speaking"); engine.setState("speaking", nowMs); const handle = startSpeech({ engine, text: speechText, expression: selectedExpression, intensity, openness, seed, speech, now: clock.now, audio: { buffer: audioBuffer, context: ctx, startAt: startAtCtx }, onFrame: ({ viseme, shape, phraseKind, stressed }) => { speechDisplayRef.current = { viseme, shape }; coSpeechDisplayRef.current = { phraseKind, stressed }; }, onEnd: () => endSpeech(clock.now()), }); speechHandleRef.current = handle; setParsedCues(handle.cues); setUnmatched(handle.unmatched); } function playLeakTest(): void { const nowMs = clock.now(); const perf: Performance = { surface: { expression: "neutral", intensity: 0.3 }, leak: { expression: "sad", intensity: 0.8, atMs: 500, durationMs: 350 }, cues: [], }; setAvatarState("listening"); engine.setState("listening", nowMs); engine.playPerformance(perf, { speechStartMs: nowMs, speechDurationMs: 1200 }, nowMs); setParsedCues([]); setUnmatched([]); } return (

아바타 v3 표현 엔진 Lab

채널·표정·반응 클립·지문 파서를 렌더러와 분리해 확인하는 개발 도구다(실 서비스 아님).

렌더러
{rendererMode === "linocut" && rig ? ( <>
) : ( )}
{rendererMode === "linocut" && rig ? ( <>

로드 상태: {linocutLoadState}

) : null}

표정 (28개)

{AVATAR_EXPRESSION_LIBRARY.map((expr) => ( ))}

반응 클립 (26개)

{REACTION_CLIP_IDS.map((id) => ( ))}

내담자 발화 · 공연 재생