import { expect, test, type Page } from "@playwright/test"; import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { existsSync } from "node:fs"; import http, { type IncomingMessage, type ServerResponse } from "node:http"; import net from "node:net"; interface TestServer { url: string; requests: () => string[]; close: () => Promise; } interface SpawnedApi { baseURL: string; logs: () => string; stop: () => Promise; } interface SpawnedWeb { baseURL: string; logs: () => string; stop: () => Promise; } interface VoiceProbe { code: number; messages: string[]; binaryChunks: number; } interface VoiceUiProbeMessage { direction: "sent" | "received"; kind: "text" | "binary"; data?: string; byteLength?: number; } interface VoiceUiProbeState { getUserMediaCalls: number; recorderStarts: number; recorderStops: number; workletModuleLoads: number; workletNodes: number; workletChunks: number; trackStops: number; audioPlays: number; audioContextResumes: number; audioBufferStarts: number; mediaPlayRejections: number; messages: VoiceUiProbeMessage[]; closeEvents: number[]; } // This fixture intentionally starts a DB-offline API with ALLOW_SEED_PERSONA_FALLBACK=true // so the voice provider cascade can be exercised without a Postgres dependency. const SEEDED_VOICE_PERSONA_CODE = "P1"; function readBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); req.on("end", () => resolve(Buffer.concat(chunks))); req.on("error", reject); }); } async function freePort(): Promise { return new Promise((resolve, reject) => { const server = net.createServer(); server.unref(); server.on("error", reject); server.listen(0, "127.0.0.1", () => { const address = server.address(); const port = typeof address === "object" && address ? address.port : 0; server.close(() => resolve(port)); }); }); } async function startHttpServer( handler: (req: IncomingMessage, res: ServerResponse) => void | Promise, ): Promise { const port = await freePort(); const requests: string[] = []; const server = http.createServer((req, res) => { requests.push(`${req.method ?? "?"} ${req.url ?? "?"}`); void Promise.resolve(handler(req, res)).catch((err) => { res.writeHead(500, { "content-type": "text/plain" }); res.end(String(err)); }); }); await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)); return { url: `http://127.0.0.1:${port}`, requests: () => requests, close: () => new Promise((resolve) => server.close(() => resolve())), }; } async function startFakeOpenAI(): Promise { return startHttpServer(async (req, res) => { await readBody(req); if (req.method === "POST" && req.url === "/v1/audio/transcriptions") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ text: "요즘 잠을 잘 못 자요.", language: "ko", duration: 1.2 })); return; } if (req.method === "POST" && req.url === "/v1/audio/speech") { res.writeHead(200, { "content-type": "audio/mpeg" }); res.end(Buffer.alloc(8192, 128)); return; } res.writeHead(404, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "not found" })); }); } async function startFakeEngine(): Promise { return startHttpServer(async (req, res) => { await readBody(req); if (req.method === "GET" && req.url === "/health") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, engine: "fake" })); return; } if (req.method === "POST" && req.url === "/v1/generate") { res.writeHead(200, { "content-type": "application/json" }); res.end( JSON.stringify({ text: "괜찮아요. 천천히 말해볼게요.", model: "fake-client", provider: "e2e", tokens_in: 1, tokens_out: 1, cost_usd: 0, inference_geo: "us", structured: null, }), ); return; } if (req.method === "POST" && req.url === "/v1/stream") { res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", }); res.write(`event: token\ndata: ${JSON.stringify({ text: "괜찮아요. " })}\n\n`); res.write(`event: token\ndata: ${JSON.stringify({ text: "천천히 말해볼게요." })}\n\n`); res.end( `event: done\ndata: ${JSON.stringify({ provider: "e2e", model: "fake-client", tokens_in: 1, tokens_out: 1, cost_usd: 0, turns: 1, })}\n\n`, ); return; } res.writeHead(404, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "not found" })); }); } async function waitForApi(baseURL: string, proc: ChildProcessWithoutNullStreams): Promise { const started = Date.now(); let lastError = ""; while (Date.now() - started < 20_000) { if (proc.exitCode !== null) { throw new Error(`API exited early with code ${proc.exitCode}: ${lastError}`); } try { const response = await fetch(`${baseURL}/health`); if (response.ok) return; lastError = await response.text(); } catch (err) { lastError = err instanceof Error ? err.message : String(err); } await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error(`Timed out waiting for API ${baseURL}: ${lastError}`); } async function startApi({ engineURL, openAIBaseURL, frontendBaseURL = "http://localhost:5173", }: { engineURL: string; openAIBaseURL: string; frontendBaseURL?: string; }): Promise { const port = await freePort(); const baseURL = `http://127.0.0.1:${port}`; const localPython311 = process.env.USERPROFILE ? `${process.env.USERPROFILE}\\AppData\\Local\\Programs\\Python\\Python311\\python.exe` : ""; const python = process.env.E2E_PYTHON ?? process.env.PYTHON311 ?? (localPython311 && existsSync(localPython311) ? localPython311 : (process.env.PYTHON ?? "python")); const proc = spawn( python, [ "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", String(port), "--log-level", "debug", ], { cwd: "../api", env: { ...process.env, PYTHONUNBUFFERED: "1", ENVIRONMENT: "dev", AUTH_DEV_LOGIN_ENABLED: "true", AUTH_ALLOWED_EMAIL_DOMAINS: '["hs.ac.kr","twentyoz.kr"]', ALLOW_SEED_PERSONA_FALLBACK: "true", DATABASE_URL: "postgresql://user:pass@127.0.0.1:1/vignette", DB_POOL_MIN_SIZE: "0", DB_COMMAND_TIMEOUT: "1", ENGINE_URL: engineURL, ENGINE_MODE: "claude_api", ENGINE_TIMEOUT: "10", ENGINE_CONNECT_TIMEOUT: "2", OPENAI_API_KEY: "e2e-fake-key", OPENAI_BASE_URL: `${openAIBaseURL}/v1`, VIGNETTE_VOICE_POC_SAMPLE_TTS: "false", FRONTEND_BASE_URL: frontendBaseURL, CORS_ORIGINS: JSON.stringify([frontendBaseURL]), }, windowsHide: true, }, ); let logs = ""; proc.stdout.on("data", (chunk) => { logs += String(chunk).slice(-4000); }); proc.stderr.on("data", (chunk) => { logs += String(chunk).slice(-4000); }); await waitForApi(baseURL, proc).catch((err) => { proc.kill(); throw new Error(`${err instanceof Error ? err.message : String(err)}\n${logs}`); }); return { baseURL, logs: () => logs, stop: async () => { if (proc.exitCode === null) proc.kill(); await new Promise((resolve) => { if (proc.exitCode !== null) { resolve(); return; } proc.once("exit", () => resolve()); setTimeout(resolve, 3000); }); }, }; } async function waitForWeb(baseURL: string, proc: ChildProcessWithoutNullStreams): Promise { const started = Date.now(); let lastError = ""; while (Date.now() - started < 30_000) { if (proc.exitCode !== null) { throw new Error(`Web exited early with code ${proc.exitCode}: ${lastError}`); } try { const response = await fetch(baseURL); if (response.ok) return; lastError = await response.text(); } catch (err) { lastError = err instanceof Error ? err.message : String(err); } await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error(`Timed out waiting for web ${baseURL}: ${lastError}`); } async function startWeb({ apiBaseURL, port, }: { apiBaseURL: string; port: number; }): Promise { const baseURL = `http://127.0.0.1:${port}`; const proc = spawn( process.execPath, ["node_modules/vite/bin/vite.js", "--host", "127.0.0.1", "--port", String(port)], { cwd: ".", env: { ...process.env, VITE_API_BASE: apiBaseURL, }, windowsHide: true, }, ); let logs = ""; proc.stdout.on("data", (chunk) => { logs += String(chunk).slice(-4000); }); proc.stderr.on("data", (chunk) => { logs += String(chunk).slice(-4000); }); await waitForWeb(baseURL, proc).catch((err) => { proc.kill(); throw new Error(`${err instanceof Error ? err.message : String(err)}\n${logs}`); }); return { baseURL, logs: () => logs, stop: async () => { if (proc.exitCode === null) proc.kill(); await new Promise((resolve) => { if (proc.exitCode !== null) { resolve(); return; } proc.once("exit", () => resolve()); setTimeout(resolve, 3000); }); }, }; } async function probeVoiceCascade(page: Page, apiBaseURL: string, sessionId: string): Promise { return page.evaluate( ({ apiBase, sid }) => new Promise((resolve) => { const wsURL = new URL(`/voice/ws?session_id=${encodeURIComponent(sid)}`, apiBase); wsURL.protocol = "ws:"; const ws = new WebSocket(wsURL.href); ws.binaryType = "arraybuffer"; const messages: string[] = []; let binaryChunks = 0; let sawTtsEnd = false; const timeout = window.setTimeout(() => { ws.close(); resolve({ code: -1, messages, binaryChunks }); }, 20_000); ws.onopen = () => { ws.send(JSON.stringify({ type: "audio_start", format: "webm" })); ws.send(new Uint8Array([1, 2, 3, 4, 5, 6]).buffer); ws.send(JSON.stringify({ type: "audio_end", format: "webm" })); }; ws.onmessage = (event) => { if (typeof event.data === "string") { messages.push(event.data); try { const parsed = JSON.parse(event.data) as { type?: string; state?: string }; if (parsed.type === "tts_end") sawTtsEnd = true; if (sawTtsEnd && parsed.type === "state" && parsed.state === "idle") { ws.send(JSON.stringify({ type: "close" })); } } catch { messages.push(JSON.stringify({ type: "error", detail: "invalid json from ws" })); } } else { binaryChunks += 1; } }; ws.onerror = () => { messages.push(JSON.stringify({ type: "error", detail: "browser websocket error" })); }; ws.onclose = (event) => { window.clearTimeout(timeout); resolve({ code: event.code, messages, binaryChunks }); }; }), { apiBase: apiBaseURL, sid: sessionId }, ); } async function installSyntheticVoiceCapture( page: Page, options: { blockMediaElementPlayback?: boolean } = {}, ): Promise { await page.addInitScript((opts) => { type ProbeMessage = { direction: "sent" | "received"; kind: "text" | "binary"; data?: string; byteLength?: number; }; type ProbeState = { getUserMediaCalls: number; recorderStarts: number; recorderStops: number; workletModuleLoads: number; workletNodes: number; workletChunks: number; trackStops: number; audioPlays: number; audioContextResumes: number; audioBufferStarts: number; mediaPlayRejections: number; messages: ProbeMessage[]; closeEvents: number[]; }; const w = window as Window & { __voiceUiProbe?: ProbeState }; const probe: ProbeState = { getUserMediaCalls: 0, recorderStarts: 0, recorderStops: 0, workletModuleLoads: 0, workletNodes: 0, workletChunks: 0, trackStops: 0, audioPlays: 0, audioContextResumes: 0, audioBufferStarts: 0, mediaPlayRejections: 0, messages: [], closeEvents: [], }; w.__voiceUiProbe = probe; const fakeTrack = { kind: "audio", readyState: "live", stop() { probe.trackStops += 1; this.readyState = "ended"; }, }; const fakeStream = { id: "synthetic-voice-ui-stream", active: true, getTracks: () => [fakeTrack], getAudioTracks: () => [fakeTrack], }; Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: { getUserMedia: async () => { probe.getUserMediaCalls += 1; return fakeStream; }, }, }); class FakeMediaRecorder extends EventTarget { static isTypeSupported() { return true; } state = "inactive"; mimeType: string; private timer: number | null = null; ondataavailable: ((event: Event & { data: Blob }) => void) | null = null; onstop: ((event: Event) => void) | null = null; constructor(_stream: unknown, options?: { mimeType?: string }) { super(); this.mimeType = options?.mimeType ?? "audio/webm"; } start(timeslice?: number) { this.state = "recording"; probe.recorderStarts += 1; const emit = () => { if (this.state !== "recording") return; const data = new Blob([new Uint8Array([1, 2, 3, 4, 5, 6])], { type: this.mimeType || "audio/webm", }); const event = new Event("dataavailable") as Event & { data: Blob }; Object.defineProperty(event, "data", { value: data }); this.ondataavailable?.(event); this.dispatchEvent(event); }; window.setTimeout(emit, 25); if (timeslice && timeslice > 0) { this.timer = window.setInterval(emit, timeslice); } } stop() { if (this.state === "inactive") return; this.state = "inactive"; if (this.timer !== null) { window.clearInterval(this.timer); this.timer = null; } probe.recorderStops += 1; const event = new Event("stop"); this.onstop?.(event); this.dispatchEvent(event); } } Object.defineProperty(window, "MediaRecorder", { configurable: true, value: FakeMediaRecorder, }); const NativeWebSocket = window.WebSocket; const sizeOf = (data: unknown) => { if (typeof data === "string") return data.length; if (data instanceof Blob) return data.size; if (data instanceof ArrayBuffer) return data.byteLength; if (ArrayBuffer.isView(data)) return data.byteLength; return 0; }; class ProbeWebSocket extends NativeWebSocket { constructor(url: string | URL, protocols?: string | string[]) { if (protocols === undefined) super(url); else super(url, protocols); this.addEventListener("message", (event) => { if (typeof event.data === "string") { probe.messages.push({ direction: "received", kind: "text", data: event.data }); } else { probe.messages.push({ direction: "received", kind: "binary", byteLength: sizeOf(event.data), }); } }); this.addEventListener("close", (event) => { probe.closeEvents.push(event.code); }); } send(data: string | ArrayBufferLike | Blob | ArrayBufferView) { if (typeof data === "string") { probe.messages.push({ direction: "sent", kind: "text", data }); } else { probe.messages.push({ direction: "sent", kind: "binary", byteLength: sizeOf(data) }); } return super.send(data); } } for (const key of ["CONNECTING", "OPEN", "CLOSING", "CLOSED"] as const) { Object.defineProperty(ProbeWebSocket, key, { value: NativeWebSocket[key] }); } Object.defineProperty(window, "WebSocket", { configurable: true, value: ProbeWebSocket, }); class FakeAnalyser { fftSize = 1024; connect() { return this; } disconnect() { return undefined; } getFloatTimeDomainData(buf: Float32Array) { for (let i = 0; i < buf.length; i += 1) buf[i] = 0; } } class FakeBufferSource { buffer: unknown = null; onended: ((event: Event) => void) | null = null; connect() { return this; } disconnect() { return undefined; } start() { probe.audioBufferStarts += 1; window.setTimeout(() => { this.onended?.(new Event("ended")); }, 120); } stop() { this.onended = null; } } class FakeWorkletPort { onmessage: ((event: MessageEvent) => void) | null = null; private closed = false; postMessage(message: unknown) { if ((message as { type?: string })?.type === "flush") { this.emitChunk(); } } close() { this.closed = true; } emitChunk() { if (this.closed) return; probe.workletChunks += 1; const pcm = new Int16Array([0, 1024, -1024, 0]); this.onmessage?.({ data: { type: "chunk", pcm: pcm.buffer, metrics: { durationMs: 1600, voiceMs: 350, silenceMs: 1250, trailingSilenceMs: 1250, rms: 0.08, peak: 0.64, }, }, } as MessageEvent); } } class FakeAudioWorkletNode { port = new FakeWorkletPort(); constructor(_ctx: unknown, _name: string, _options?: unknown) { probe.workletNodes += 1; } connect() { return this; } disconnect() { return undefined; } __start() { window.setTimeout(() => this.port.emitChunk(), 25); } } class FakeMediaStreamSource { connect(node: { __start?: () => void }) { node.__start?.(); return node; } disconnect() { return undefined; } } class FakeAudioContext { state = "running"; destination = {}; sampleRate = 16000; audioWorklet = { addModule: async (_url: string) => { probe.workletModuleLoads += 1; }, }; async resume() { probe.audioContextResumes += 1; this.state = "running"; } async close() { this.state = "closed"; } async decodeAudioData(_data: ArrayBuffer) { return { duration: 0.12 }; } createAnalyser() { return new FakeAnalyser(); } createBufferSource() { return new FakeBufferSource(); } createMediaStreamSource(_stream: unknown) { return new FakeMediaStreamSource(); } } Object.defineProperty(window, "AudioWorkletNode", { configurable: true, value: FakeAudioWorkletNode, }); Object.defineProperty(window, "AudioContext", { configurable: true, value: FakeAudioContext, }); Object.defineProperty(window, "webkitAudioContext", { configurable: true, value: FakeAudioContext, }); HTMLMediaElement.prototype.play = function patchedPlay() { if (opts.blockMediaElementPlayback) { probe.mediaPlayRejections += 1; return Promise.reject(new DOMException("Synthetic autoplay block", "NotAllowedError")); } probe.audioPlays += 1; window.setTimeout(() => { this.dispatchEvent(new Event("ended")); }, 120); return Promise.resolve(); }; }, options); } async function readVoiceUiProbe(page: Page): Promise { return page.evaluate(() => { const probe = (window as Window & { __voiceUiProbe?: VoiceUiProbeState }).__voiceUiProbe; if (!probe) throw new Error("voice UI probe was not installed"); return probe; }); } async function parsedVoiceUiEvents(page: Page): Promise> { const probe = await readVoiceUiProbe(page); return probe.messages .filter((message) => message.direction === "received" && message.kind === "text" && message.data) .map((message) => JSON.parse(message.data ?? "{}") as { type?: string; [key: string]: unknown }); } test.describe("voice cascade success path", () => { test("runs STT, client turn, TTS, and audio chunks against controlled providers @single-run", async ({ page, }, testInfo) => { test.setTimeout(60_000); const openai = await startFakeOpenAI(); const engine = await startFakeEngine(); const api = await startApi({ engineURL: engine.url, openAIBaseURL: openai.url }); try { const health = await page.request.get(`${api.baseURL}/voice/health`); expect(health.ok(), await health.text()).toBeTruthy(); await expect(await health.json()).toMatchObject({ status: "ok", available: true }); await page.goto(`${api.baseURL}/health`); const browserSetup = await page.evaluate(async ({ apiBase, seededPersonaCode, workerIndex }) => { const login = await fetch(`${apiBase}/auth/dev-login`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ email: `voice-success.${workerIndex}@hs.ac.kr`, role: "learner", display_name: "Voice Success", }), }); const loginBody = await login.text(); if (!login.ok) { return { ok: false, step: "login", status: login.status, body: loginBody }; } const onboarding = await fetch(`${apiBase}/users/me/onboarding`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ legal_name: "Voice Success", affiliation: "한신대학교", department: "상담심리학과", grade_level: "3학년", phone: "010-5555-5555", contact_address: "경기도 오산시 한신대학교", nickname: "Voice Success", self_introduction: "음성 캐스케이드 성공 경로 검증용 사용자입니다.", avatar_url: "", terms_accepted: true, privacy_accepted: true, }), }); const onboardingBody = await onboarding.text(); if (!onboarding.ok) { return { ok: false, step: "onboarding", status: onboarding.status, body: onboardingBody, }; } const me = await fetch(`${apiBase}/auth/me`, { credentials: "include" }); const meBody = await me.text(); if (!me.ok) { return { ok: false, step: "me", status: me.status, body: meBody }; } const start = await fetch(`${apiBase}/sessions`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ persona_code: seededPersonaCode, theory_mode: "humanistic" }), }); const startBody = await start.text(); if (!start.ok) { return { ok: false, step: "sessions", status: start.status, body: startBody }; } return { ok: true, me: JSON.parse(meBody) as unknown, started: JSON.parse(startBody) as { session_id: string }, }; }, { apiBase: api.baseURL, seededPersonaCode: SEEDED_VOICE_PERSONA_CODE, workerIndex: testInfo.workerIndex, }); expect(browserSetup, api.logs()).toMatchObject({ ok: true }); if (!browserSetup.ok) throw new Error(JSON.stringify(browserSetup)); const started = browserSetup.started; const result = await probeVoiceCascade(page, api.baseURL, started.session_id); const events = result.messages.map((message) => JSON.parse(message) as { type: string; [key: string]: unknown }); expect( result.code, [ JSON.stringify(result, null, 2), `fakeOpenAI=${JSON.stringify(openai.requests())}`, `fakeEngine=${JSON.stringify(engine.requests())}`, api.logs(), ].join("\n\n"), ).toBe(1000); expect(events.some((event) => event.type === "degraded")).toBe(false); expect(events.some((event) => event.type === "error")).toBe(false); expect(events).toEqual( expect.arrayContaining([ expect.objectContaining({ type: "ready", session_id: started.session_id }), expect.objectContaining({ type: "state", state: "listening" }), expect.objectContaining({ type: "state", state: "thinking" }), expect.objectContaining({ type: "transcript", text: "요즘 잠을 잘 못 자요." }), expect.objectContaining({ type: "reply", text: "괜찮아요. 천천히 말해볼게요." }), expect.objectContaining({ type: "state", state: "speaking" }), expect.objectContaining({ type: "tts_end" }), expect.objectContaining({ type: "state", state: "idle" }), ]), ); expect(result.binaryChunks).toBeGreaterThan(0); } finally { await api.stop(); await engine.close(); await openai.close(); } }); test("drives one voice turn through the Session mic UI with synthetic browser audio @single-run", async ({ page, }, testInfo) => { test.setTimeout(150_000); const diagnostics: string[] = []; page.on("pageerror", (error) => diagnostics.push(`pageerror: ${error.message}`)); page.on("console", (message) => { if (message.type() === "error") diagnostics.push(`console: ${message.text()}`); }); page.on("requestfailed", (request) => { diagnostics.push(`requestfailed: ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ""}`); }); page.on("response", (response) => { const url = response.url(); if ( response.status() >= 400 && (url.includes("/sessions") || url.includes("/voice/ws") || url.includes("/voice/speech")) ) { diagnostics.push(`response: ${response.status()} ${url}`); } }); await installSyntheticVoiceCapture(page, { blockMediaElementPlayback: true }); const openai = await startFakeOpenAI(); const engine = await startFakeEngine(); const webPort = await freePort(); const webBaseURL = `http://127.0.0.1:${webPort}`; const api = await startApi({ engineURL: engine.url, openAIBaseURL: openai.url, frontendBaseURL: webBaseURL, }); const web = await startWeb({ apiBaseURL: api.baseURL, port: webPort }); try { await page.route("**/personas", async (route) => { if (route.request().method() !== "GET") { await route.fallback(); return; } await route.fulfill({ status: 200, contentType: "application/json", headers: { "access-control-allow-origin": web.baseURL, "access-control-allow-credentials": "true", }, body: JSON.stringify([ { code: SEEDED_VOICE_PERSONA_CODE, display_name: "Voice UI fixture", difficulty: "hard", theory_target: ["humanistic"], demographics: { age_band: "teen" }, presenting_summary: "Synthetic browser audio UI proof", voice_preset: "soft-young-fem", source: "database", degraded: false, }, ]), }); }); await page.goto(`${web.baseURL}/login`, { waitUntil: "domcontentloaded" }); const browserLogin = await page.evaluate(async ({ apiBase, workerIndex }) => { const login = await fetch(`${apiBase}/auth/dev-login`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ email: `voice-ui.${workerIndex}@hs.ac.kr`, role: "learner", display_name: "Voice UI", }), }); const loginBody = await login.text(); if (!login.ok) { return { ok: false, step: "login", status: login.status, body: loginBody }; } const onboarding = await fetch(`${apiBase}/users/me/onboarding`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ legal_name: "Voice UI", affiliation: "한신대학교", department: "상담심리학과", grade_level: "3학년", phone: "010-6666-6666", contact_address: "경기도 오산시 한신대학교", nickname: "Voice UI", self_introduction: "브라우저 음성 UI 검증용 사용자입니다.", avatar_url: "", terms_accepted: true, privacy_accepted: true, }), }); const onboardingBody = await onboarding.text(); if (!onboarding.ok) { return { ok: false, step: "onboarding", status: onboarding.status, body: onboardingBody, }; } const me = await fetch(`${apiBase}/auth/me`, { credentials: "include" }); const meBody = await me.text(); if (!me.ok) { return { ok: false, step: "me", status: me.status, body: meBody }; } return { ok: true, me: JSON.parse(meBody) as unknown }; }, { apiBase: api.baseURL, workerIndex: testInfo.workerIndex, }); expect(browserLogin, api.logs()).toMatchObject({ ok: true }); await page.goto(`${web.baseURL}/learn/session/${SEEDED_VOICE_PERSONA_CODE}`); await expect( page.locator(".sx-prestart__actions button").first(), diagnostics.join("\n") || (await page.locator("#root").innerText().catch(() => "")), ).toBeVisible(); await page.locator(".sx-prestart__actions button").first().click(); await expect( page.locator(".sx-page.sx-page--active"), [ ...diagnostics, `apiLogs=${api.logs()}`, `pageText=${await page.locator("#root").innerText().catch(() => "")}`, ].join("\n\n"), ).toBeVisible({ timeout: 20_000 }); const textInput = page.getByLabel("학습자 발화 입력"); const sendButton = page.getByRole("button", { name: "보내기", exact: true }); await textInput.fill("텍스트로 말해도 내담자 음성을 들려주세요."); await sendButton.click(); await expect .poll( () => openai.requests().filter((request) => request === "POST /v1/audio/speech").length, { timeout: 30_000 }, ) .toBeGreaterThan(0); await expect .poll(async () => (await readVoiceUiProbe(page)).audioBufferStarts, { timeout: 30_000 }) .toBeGreaterThan(0); await expect(page.locator(".sx-utt").filter({ hasText: "괜찮아요. 천천히 말해볼게요." })).toBeVisible(); const freshVoiceSession = await page.evaluate(async ({ apiBase, personaCode }) => { const response = await fetch(`${apiBase}/sessions`, { method: "POST", credentials: "include", headers: { "content-type": "application/json" }, body: JSON.stringify({ persona_code: personaCode, theory_mode: "humanistic" }), }); return { ok: response.ok, status: response.status, body: await response.json() as { session_id?: string }, }; }, { apiBase: api.baseURL, personaCode: SEEDED_VOICE_PERSONA_CODE }); expect(freshVoiceSession, api.logs()).toMatchObject({ ok: true }); expect(typeof freshVoiceSession.body.session_id).toBe("string"); await page.goto(`${web.baseURL}/learn/session/${freshVoiceSession.body.session_id}`); await expect(page.locator(".sx-page.sx-page--active")).toBeVisible({ timeout: 20_000 }); const mic = page.locator(".sx-mic"); await expect(mic).toBeEnabled(); await mic.click(); await expect .poll(async () => (await readVoiceUiProbe(page)).getUserMediaCalls, { timeout: 10_000 }) .toBeGreaterThan(0); await expect .poll(async () => (await readVoiceUiProbe(page)).workletModuleLoads, { timeout: 10_000 }) .toBeGreaterThan(0); await expect .poll(async () => (await readVoiceUiProbe(page)).workletNodes, { timeout: 10_000 }) .toBeGreaterThan(0); await expect .poll(async () => (await readVoiceUiProbe(page)).workletChunks, { timeout: 10_000 }) .toBeGreaterThan(0); await expect .poll(async () => { const probe = await readVoiceUiProbe(page); return probe.messages.some( (message) => message.direction === "sent" && message.kind === "text" && message.data?.includes('"audio_start"') && message.data?.includes('"format":"pcm"') && message.data?.includes('"sample_rate":16000'), ); }, { timeout: 10_000 }) .toBeTruthy(); await expect .poll(async () => { const probe = await readVoiceUiProbe(page); return probe.messages.some( (message) => message.direction === "sent" && message.kind === "binary", ); }, { timeout: 10_000 }) .toBeTruthy(); await mic.click(); await expect .poll(async () => { const probe = await readVoiceUiProbe(page); return probe.messages.some( (message) => message.direction === "sent" && message.kind === "text" && message.data?.includes('"audio_end"'), ); }, { timeout: 10_000 }) .toBeTruthy(); const sentAudioEndControls = (await readVoiceUiProbe(page)).messages .filter( (message) => message.direction === "sent" && message.kind === "text" && message.data?.includes('"audio_end"'), ) .map((message) => JSON.parse(message.data ?? "{}") as Record); expect( sentAudioEndControls.some((control) => { const events = Array.isArray(control.provider_events) ? control.provider_events : []; return ( control.format === "pcm" && control.silence_ms === 1250 && control.barge_in === false && events.some((event) => { const record = event as Record; return ( record.type === "voice_activity" && record.source === "browser_audio_worklet" && record.duration_ms === 350 ); }) && events.some((event) => { const record = event as Record; return ( record.type === "silence" && record.source === "browser_audio_worklet" && record.duration_ms === 1250 ); }) ); }), ).toBeTruthy(); await expect .poll(async () => { const events = await parsedVoiceUiEvents(page); return { transcript: events.some((event) => event.type === "transcript"), reply: events.some((event) => event.type === "reply"), ttsEnd: events.some((event) => event.type === "tts_end"), errors: events.filter((event) => event.type === "error" || event.type === "degraded"), }; }, { timeout: 30_000 }) .toEqual({ transcript: true, reply: true, ttsEnd: true, errors: [] }); const events = await parsedVoiceUiEvents(page); const transcript = events.find((event) => event.type === "transcript")?.text; const reply = events.find((event) => event.type === "reply")?.text; expect(typeof transcript).toBe("string"); expect(typeof reply).toBe("string"); await expect(page.locator(".sx-utt").filter({ hasText: String(transcript) })).toBeVisible(); await expect(page.locator(".sx-utt").filter({ hasText: String(reply) })).toBeVisible(); const probe = await readVoiceUiProbe(page); expect(probe.audioContextResumes).toBeGreaterThan(0); expect(probe.audioBufferStarts).toBeGreaterThan(0); expect(probe.audioPlays).toBe(0); expect(probe.mediaPlayRejections).toBe(0); expect(probe.messages.some((message) => message.direction === "received" && message.kind === "binary")).toBe( true, ); expect(openai.requests()).toEqual( expect.arrayContaining(["POST /v1/audio/transcriptions", "POST /v1/audio/speech"]), ); expect(engine.requests()).toContain("POST /v1/generate"); } finally { await web.stop(); await api.stop(); await engine.close(); await openai.close(); } }); });