1146 lines
37 KiB
TypeScript
1146 lines
37 KiB
TypeScript
import { expect, test, type Page } from "@playwright/test";
|
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
|
import { existsSync } from "node:fs";
|
|
import http, { type IncomingMessage, type ServerResponse } from "node:http";
|
|
import net from "node:net";
|
|
|
|
interface TestServer {
|
|
url: string;
|
|
requests: () => string[];
|
|
close: () => Promise<void>;
|
|
}
|
|
|
|
interface SpawnedApi {
|
|
baseURL: string;
|
|
logs: () => string;
|
|
stop: () => Promise<void>;
|
|
}
|
|
|
|
interface SpawnedWeb {
|
|
baseURL: string;
|
|
logs: () => string;
|
|
stop: () => Promise<void>;
|
|
}
|
|
|
|
interface VoiceProbe {
|
|
code: number;
|
|
messages: string[];
|
|
binaryChunks: number;
|
|
}
|
|
|
|
interface VoiceUiProbeMessage {
|
|
direction: "sent" | "received";
|
|
kind: "text" | "binary";
|
|
data?: string;
|
|
byteLength?: number;
|
|
}
|
|
|
|
interface VoiceUiProbeState {
|
|
getUserMediaCalls: number;
|
|
recorderStarts: number;
|
|
recorderStops: number;
|
|
workletModuleLoads: number;
|
|
workletNodes: number;
|
|
workletChunks: number;
|
|
trackStops: number;
|
|
audioPlays: number;
|
|
audioContextResumes: number;
|
|
audioBufferStarts: number;
|
|
mediaPlayRejections: number;
|
|
messages: VoiceUiProbeMessage[];
|
|
closeEvents: number[];
|
|
}
|
|
|
|
// This fixture intentionally starts a DB-offline API with ALLOW_SEED_PERSONA_FALLBACK=true
|
|
// so the voice provider cascade can be exercised without a Postgres dependency.
|
|
const SEEDED_VOICE_PERSONA_CODE = "P1";
|
|
|
|
function readBody(req: IncomingMessage): Promise<Buffer> {
|
|
return new Promise((resolve, reject) => {
|
|
const chunks: Buffer[] = [];
|
|
req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
|
req.on("end", () => resolve(Buffer.concat(chunks)));
|
|
req.on("error", reject);
|
|
});
|
|
}
|
|
|
|
async function freePort(): Promise<number> {
|
|
return new Promise((resolve, reject) => {
|
|
const server = net.createServer();
|
|
server.unref();
|
|
server.on("error", reject);
|
|
server.listen(0, "127.0.0.1", () => {
|
|
const address = server.address();
|
|
const port = typeof address === "object" && address ? address.port : 0;
|
|
server.close(() => resolve(port));
|
|
});
|
|
});
|
|
}
|
|
|
|
async function startHttpServer(
|
|
handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>,
|
|
): Promise<TestServer> {
|
|
const port = await freePort();
|
|
const requests: string[] = [];
|
|
const server = http.createServer((req, res) => {
|
|
requests.push(`${req.method ?? "?"} ${req.url ?? "?"}`);
|
|
void Promise.resolve(handler(req, res)).catch((err) => {
|
|
res.writeHead(500, { "content-type": "text/plain" });
|
|
res.end(String(err));
|
|
});
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(port, "127.0.0.1", resolve));
|
|
return {
|
|
url: `http://127.0.0.1:${port}`,
|
|
requests: () => requests,
|
|
close: () => new Promise((resolve) => server.close(() => resolve())),
|
|
};
|
|
}
|
|
|
|
async function startFakeOpenAI(): Promise<TestServer> {
|
|
return startHttpServer(async (req, res) => {
|
|
await readBody(req);
|
|
if (req.method === "POST" && req.url === "/v1/audio/transcriptions") {
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ text: "요즘 잠을 잘 못 자요.", language: "ko", duration: 1.2 }));
|
|
return;
|
|
}
|
|
if (req.method === "POST" && req.url === "/v1/audio/speech") {
|
|
res.writeHead(200, { "content-type": "audio/mpeg" });
|
|
res.end(Buffer.alloc(8192, 128));
|
|
return;
|
|
}
|
|
res.writeHead(404, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ error: "not found" }));
|
|
});
|
|
}
|
|
|
|
async function startFakeEngine(): Promise<TestServer> {
|
|
return startHttpServer(async (req, res) => {
|
|
await readBody(req);
|
|
if (req.method === "GET" && req.url === "/health") {
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ ok: true, engine: "fake" }));
|
|
return;
|
|
}
|
|
if (req.method === "POST" && req.url === "/v1/generate") {
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(
|
|
JSON.stringify({
|
|
text: "괜찮아요. 천천히 말해볼게요.",
|
|
model: "fake-client",
|
|
provider: "e2e",
|
|
tokens_in: 1,
|
|
tokens_out: 1,
|
|
cost_usd: 0,
|
|
inference_geo: "us",
|
|
structured: null,
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
if (req.method === "POST" && req.url === "/v1/stream") {
|
|
res.writeHead(200, {
|
|
"content-type": "text/event-stream",
|
|
"cache-control": "no-cache",
|
|
});
|
|
res.write(`event: token\ndata: ${JSON.stringify({ text: "괜찮아요. " })}\n\n`);
|
|
res.write(`event: token\ndata: ${JSON.stringify({ text: "천천히 말해볼게요." })}\n\n`);
|
|
res.end(
|
|
`event: done\ndata: ${JSON.stringify({
|
|
provider: "e2e",
|
|
model: "fake-client",
|
|
tokens_in: 1,
|
|
tokens_out: 1,
|
|
cost_usd: 0,
|
|
turns: 1,
|
|
})}\n\n`,
|
|
);
|
|
return;
|
|
}
|
|
res.writeHead(404, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ error: "not found" }));
|
|
});
|
|
}
|
|
|
|
async function waitForApi(baseURL: string, proc: ChildProcessWithoutNullStreams): Promise<void> {
|
|
const started = Date.now();
|
|
let lastError = "";
|
|
while (Date.now() - started < 20_000) {
|
|
if (proc.exitCode !== null) {
|
|
throw new Error(`API exited early with code ${proc.exitCode}: ${lastError}`);
|
|
}
|
|
try {
|
|
const response = await fetch(`${baseURL}/health`);
|
|
if (response.ok) return;
|
|
lastError = await response.text();
|
|
} catch (err) {
|
|
lastError = err instanceof Error ? err.message : String(err);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
throw new Error(`Timed out waiting for API ${baseURL}: ${lastError}`);
|
|
}
|
|
|
|
async function startApi({
|
|
engineURL,
|
|
openAIBaseURL,
|
|
frontendBaseURL = "http://localhost:5173",
|
|
}: {
|
|
engineURL: string;
|
|
openAIBaseURL: string;
|
|
frontendBaseURL?: string;
|
|
}): Promise<SpawnedApi> {
|
|
const port = await freePort();
|
|
const baseURL = `http://127.0.0.1:${port}`;
|
|
const localPython311 = process.env.USERPROFILE
|
|
? `${process.env.USERPROFILE}\\AppData\\Local\\Programs\\Python\\Python311\\python.exe`
|
|
: "";
|
|
const python =
|
|
process.env.E2E_PYTHON ??
|
|
process.env.PYTHON311 ??
|
|
(localPython311 && existsSync(localPython311) ? localPython311 : (process.env.PYTHON ?? "python"));
|
|
const proc = spawn(
|
|
python,
|
|
[
|
|
"-m",
|
|
"uvicorn",
|
|
"app.main:app",
|
|
"--host",
|
|
"127.0.0.1",
|
|
"--port",
|
|
String(port),
|
|
"--log-level",
|
|
"debug",
|
|
],
|
|
{
|
|
cwd: "../api",
|
|
env: {
|
|
...process.env,
|
|
PYTHONUNBUFFERED: "1",
|
|
ENVIRONMENT: "dev",
|
|
AUTH_DEV_LOGIN_ENABLED: "true",
|
|
AUTH_ALLOWED_EMAIL_DOMAINS: '["hs.ac.kr","twentyoz.kr"]',
|
|
ALLOW_SEED_PERSONA_FALLBACK: "true",
|
|
DATABASE_URL: "postgresql://user:pass@127.0.0.1:1/vignette",
|
|
DB_POOL_MIN_SIZE: "0",
|
|
DB_COMMAND_TIMEOUT: "1",
|
|
ENGINE_URL: engineURL,
|
|
ENGINE_MODE: "claude_api",
|
|
ENGINE_TIMEOUT: "10",
|
|
ENGINE_CONNECT_TIMEOUT: "2",
|
|
OPENAI_API_KEY: "e2e-fake-key",
|
|
OPENAI_BASE_URL: `${openAIBaseURL}/v1`,
|
|
VIGNETTE_VOICE_POC_SAMPLE_TTS: "false",
|
|
FRONTEND_BASE_URL: frontendBaseURL,
|
|
CORS_ORIGINS: JSON.stringify([frontendBaseURL]),
|
|
},
|
|
windowsHide: true,
|
|
},
|
|
);
|
|
let logs = "";
|
|
proc.stdout.on("data", (chunk) => {
|
|
logs += String(chunk).slice(-4000);
|
|
});
|
|
proc.stderr.on("data", (chunk) => {
|
|
logs += String(chunk).slice(-4000);
|
|
});
|
|
await waitForApi(baseURL, proc).catch((err) => {
|
|
proc.kill();
|
|
throw new Error(`${err instanceof Error ? err.message : String(err)}\n${logs}`);
|
|
});
|
|
return {
|
|
baseURL,
|
|
logs: () => logs,
|
|
stop: async () => {
|
|
if (proc.exitCode === null) proc.kill();
|
|
await new Promise<void>((resolve) => {
|
|
if (proc.exitCode !== null) {
|
|
resolve();
|
|
return;
|
|
}
|
|
proc.once("exit", () => resolve());
|
|
setTimeout(resolve, 3000);
|
|
});
|
|
},
|
|
};
|
|
}
|
|
|
|
async function waitForWeb(baseURL: string, proc: ChildProcessWithoutNullStreams): Promise<void> {
|
|
const started = Date.now();
|
|
let lastError = "";
|
|
while (Date.now() - started < 30_000) {
|
|
if (proc.exitCode !== null) {
|
|
throw new Error(`Web exited early with code ${proc.exitCode}: ${lastError}`);
|
|
}
|
|
try {
|
|
const response = await fetch(baseURL);
|
|
if (response.ok) return;
|
|
lastError = await response.text();
|
|
} catch (err) {
|
|
lastError = err instanceof Error ? err.message : String(err);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
throw new Error(`Timed out waiting for web ${baseURL}: ${lastError}`);
|
|
}
|
|
|
|
async function startWeb({
|
|
apiBaseURL,
|
|
port,
|
|
}: {
|
|
apiBaseURL: string;
|
|
port: number;
|
|
}): Promise<SpawnedWeb> {
|
|
const baseURL = `http://127.0.0.1:${port}`;
|
|
const proc = spawn(
|
|
process.execPath,
|
|
["node_modules/vite/bin/vite.js", "--host", "127.0.0.1", "--port", String(port)],
|
|
{
|
|
cwd: ".",
|
|
env: {
|
|
...process.env,
|
|
VITE_API_BASE: apiBaseURL,
|
|
},
|
|
windowsHide: true,
|
|
},
|
|
);
|
|
let logs = "";
|
|
proc.stdout.on("data", (chunk) => {
|
|
logs += String(chunk).slice(-4000);
|
|
});
|
|
proc.stderr.on("data", (chunk) => {
|
|
logs += String(chunk).slice(-4000);
|
|
});
|
|
await waitForWeb(baseURL, proc).catch((err) => {
|
|
proc.kill();
|
|
throw new Error(`${err instanceof Error ? err.message : String(err)}\n${logs}`);
|
|
});
|
|
return {
|
|
baseURL,
|
|
logs: () => logs,
|
|
stop: async () => {
|
|
if (proc.exitCode === null) proc.kill();
|
|
await new Promise<void>((resolve) => {
|
|
if (proc.exitCode !== null) {
|
|
resolve();
|
|
return;
|
|
}
|
|
proc.once("exit", () => resolve());
|
|
setTimeout(resolve, 3000);
|
|
});
|
|
},
|
|
};
|
|
}
|
|
|
|
async function probeVoiceCascade(page: Page, apiBaseURL: string, sessionId: string): Promise<VoiceProbe> {
|
|
return page.evaluate(
|
|
({ apiBase, sid }) =>
|
|
new Promise<VoiceProbe>((resolve) => {
|
|
const wsURL = new URL(`/voice/ws?session_id=${encodeURIComponent(sid)}`, apiBase);
|
|
wsURL.protocol = "ws:";
|
|
const ws = new WebSocket(wsURL.href);
|
|
ws.binaryType = "arraybuffer";
|
|
const messages: string[] = [];
|
|
let binaryChunks = 0;
|
|
let sawTtsEnd = false;
|
|
const timeout = window.setTimeout(() => {
|
|
ws.close();
|
|
resolve({ code: -1, messages, binaryChunks });
|
|
}, 20_000);
|
|
|
|
ws.onopen = () => {
|
|
ws.send(JSON.stringify({ type: "audio_start", format: "webm" }));
|
|
ws.send(new Uint8Array([1, 2, 3, 4, 5, 6]).buffer);
|
|
ws.send(JSON.stringify({ type: "audio_end", format: "webm" }));
|
|
};
|
|
ws.onmessage = (event) => {
|
|
if (typeof event.data === "string") {
|
|
messages.push(event.data);
|
|
try {
|
|
const parsed = JSON.parse(event.data) as { type?: string; state?: string };
|
|
if (parsed.type === "tts_end") sawTtsEnd = true;
|
|
if (sawTtsEnd && parsed.type === "state" && parsed.state === "idle") {
|
|
ws.send(JSON.stringify({ type: "close" }));
|
|
}
|
|
} catch {
|
|
messages.push(JSON.stringify({ type: "error", detail: "invalid json from ws" }));
|
|
}
|
|
} else {
|
|
binaryChunks += 1;
|
|
}
|
|
};
|
|
ws.onerror = () => {
|
|
messages.push(JSON.stringify({ type: "error", detail: "browser websocket error" }));
|
|
};
|
|
ws.onclose = (event) => {
|
|
window.clearTimeout(timeout);
|
|
resolve({ code: event.code, messages, binaryChunks });
|
|
};
|
|
}),
|
|
{ apiBase: apiBaseURL, sid: sessionId },
|
|
);
|
|
}
|
|
|
|
async function installSyntheticVoiceCapture(
|
|
page: Page,
|
|
options: { blockMediaElementPlayback?: boolean } = {},
|
|
): Promise<void> {
|
|
await page.addInitScript((opts) => {
|
|
type ProbeMessage = {
|
|
direction: "sent" | "received";
|
|
kind: "text" | "binary";
|
|
data?: string;
|
|
byteLength?: number;
|
|
};
|
|
type ProbeState = {
|
|
getUserMediaCalls: number;
|
|
recorderStarts: number;
|
|
recorderStops: number;
|
|
workletModuleLoads: number;
|
|
workletNodes: number;
|
|
workletChunks: number;
|
|
trackStops: number;
|
|
audioPlays: number;
|
|
audioContextResumes: number;
|
|
audioBufferStarts: number;
|
|
mediaPlayRejections: number;
|
|
messages: ProbeMessage[];
|
|
closeEvents: number[];
|
|
};
|
|
const w = window as Window & { __voiceUiProbe?: ProbeState };
|
|
const probe: ProbeState = {
|
|
getUserMediaCalls: 0,
|
|
recorderStarts: 0,
|
|
recorderStops: 0,
|
|
workletModuleLoads: 0,
|
|
workletNodes: 0,
|
|
workletChunks: 0,
|
|
trackStops: 0,
|
|
audioPlays: 0,
|
|
audioContextResumes: 0,
|
|
audioBufferStarts: 0,
|
|
mediaPlayRejections: 0,
|
|
messages: [],
|
|
closeEvents: [],
|
|
};
|
|
w.__voiceUiProbe = probe;
|
|
|
|
const fakeTrack = {
|
|
kind: "audio",
|
|
readyState: "live",
|
|
stop() {
|
|
probe.trackStops += 1;
|
|
this.readyState = "ended";
|
|
},
|
|
};
|
|
const fakeStream = {
|
|
id: "synthetic-voice-ui-stream",
|
|
active: true,
|
|
getTracks: () => [fakeTrack],
|
|
getAudioTracks: () => [fakeTrack],
|
|
};
|
|
Object.defineProperty(navigator, "mediaDevices", {
|
|
configurable: true,
|
|
value: {
|
|
getUserMedia: async () => {
|
|
probe.getUserMediaCalls += 1;
|
|
return fakeStream;
|
|
},
|
|
},
|
|
});
|
|
|
|
class FakeMediaRecorder extends EventTarget {
|
|
static isTypeSupported() {
|
|
return true;
|
|
}
|
|
|
|
state = "inactive";
|
|
mimeType: string;
|
|
private timer: number | null = null;
|
|
ondataavailable: ((event: Event & { data: Blob }) => void) | null = null;
|
|
onstop: ((event: Event) => void) | null = null;
|
|
|
|
constructor(_stream: unknown, options?: { mimeType?: string }) {
|
|
super();
|
|
this.mimeType = options?.mimeType ?? "audio/webm";
|
|
}
|
|
|
|
start(timeslice?: number) {
|
|
this.state = "recording";
|
|
probe.recorderStarts += 1;
|
|
const emit = () => {
|
|
if (this.state !== "recording") return;
|
|
const data = new Blob([new Uint8Array([1, 2, 3, 4, 5, 6])], {
|
|
type: this.mimeType || "audio/webm",
|
|
});
|
|
const event = new Event("dataavailable") as Event & { data: Blob };
|
|
Object.defineProperty(event, "data", { value: data });
|
|
this.ondataavailable?.(event);
|
|
this.dispatchEvent(event);
|
|
};
|
|
window.setTimeout(emit, 25);
|
|
if (timeslice && timeslice > 0) {
|
|
this.timer = window.setInterval(emit, timeslice);
|
|
}
|
|
}
|
|
|
|
stop() {
|
|
if (this.state === "inactive") return;
|
|
this.state = "inactive";
|
|
if (this.timer !== null) {
|
|
window.clearInterval(this.timer);
|
|
this.timer = null;
|
|
}
|
|
probe.recorderStops += 1;
|
|
const event = new Event("stop");
|
|
this.onstop?.(event);
|
|
this.dispatchEvent(event);
|
|
}
|
|
}
|
|
Object.defineProperty(window, "MediaRecorder", {
|
|
configurable: true,
|
|
value: FakeMediaRecorder,
|
|
});
|
|
|
|
const NativeWebSocket = window.WebSocket;
|
|
const sizeOf = (data: unknown) => {
|
|
if (typeof data === "string") return data.length;
|
|
if (data instanceof Blob) return data.size;
|
|
if (data instanceof ArrayBuffer) return data.byteLength;
|
|
if (ArrayBuffer.isView(data)) return data.byteLength;
|
|
return 0;
|
|
};
|
|
class ProbeWebSocket extends NativeWebSocket {
|
|
constructor(url: string | URL, protocols?: string | string[]) {
|
|
if (protocols === undefined) super(url);
|
|
else super(url, protocols);
|
|
this.addEventListener("message", (event) => {
|
|
if (typeof event.data === "string") {
|
|
probe.messages.push({ direction: "received", kind: "text", data: event.data });
|
|
} else {
|
|
probe.messages.push({
|
|
direction: "received",
|
|
kind: "binary",
|
|
byteLength: sizeOf(event.data),
|
|
});
|
|
}
|
|
});
|
|
this.addEventListener("close", (event) => {
|
|
probe.closeEvents.push(event.code);
|
|
});
|
|
}
|
|
|
|
send(data: string | ArrayBufferLike | Blob | ArrayBufferView) {
|
|
if (typeof data === "string") {
|
|
probe.messages.push({ direction: "sent", kind: "text", data });
|
|
} else {
|
|
probe.messages.push({ direction: "sent", kind: "binary", byteLength: sizeOf(data) });
|
|
}
|
|
return super.send(data);
|
|
}
|
|
}
|
|
for (const key of ["CONNECTING", "OPEN", "CLOSING", "CLOSED"] as const) {
|
|
Object.defineProperty(ProbeWebSocket, key, { value: NativeWebSocket[key] });
|
|
}
|
|
Object.defineProperty(window, "WebSocket", {
|
|
configurable: true,
|
|
value: ProbeWebSocket,
|
|
});
|
|
|
|
class FakeAnalyser {
|
|
fftSize = 1024;
|
|
|
|
connect() {
|
|
return this;
|
|
}
|
|
|
|
disconnect() {
|
|
return undefined;
|
|
}
|
|
|
|
getFloatTimeDomainData(buf: Float32Array) {
|
|
for (let i = 0; i < buf.length; i += 1) buf[i] = 0;
|
|
}
|
|
}
|
|
|
|
class FakeBufferSource {
|
|
buffer: unknown = null;
|
|
onended: ((event: Event) => void) | null = null;
|
|
|
|
connect() {
|
|
return this;
|
|
}
|
|
|
|
disconnect() {
|
|
return undefined;
|
|
}
|
|
|
|
start() {
|
|
probe.audioBufferStarts += 1;
|
|
window.setTimeout(() => {
|
|
this.onended?.(new Event("ended"));
|
|
}, 120);
|
|
}
|
|
|
|
stop() {
|
|
this.onended = null;
|
|
}
|
|
}
|
|
|
|
class FakeWorkletPort {
|
|
onmessage: ((event: MessageEvent) => void) | null = null;
|
|
private closed = false;
|
|
|
|
postMessage(message: unknown) {
|
|
if ((message as { type?: string })?.type === "flush") {
|
|
this.emitChunk();
|
|
}
|
|
}
|
|
|
|
close() {
|
|
this.closed = true;
|
|
}
|
|
|
|
emitChunk() {
|
|
if (this.closed) return;
|
|
probe.workletChunks += 1;
|
|
const pcm = new Int16Array([0, 1024, -1024, 0]);
|
|
this.onmessage?.({
|
|
data: {
|
|
type: "chunk",
|
|
pcm: pcm.buffer,
|
|
metrics: {
|
|
durationMs: 1600,
|
|
voiceMs: 350,
|
|
silenceMs: 1250,
|
|
trailingSilenceMs: 1250,
|
|
rms: 0.08,
|
|
peak: 0.64,
|
|
},
|
|
},
|
|
} as MessageEvent);
|
|
}
|
|
}
|
|
|
|
class FakeAudioWorkletNode {
|
|
port = new FakeWorkletPort();
|
|
|
|
constructor(_ctx: unknown, _name: string, _options?: unknown) {
|
|
probe.workletNodes += 1;
|
|
}
|
|
|
|
connect() {
|
|
return this;
|
|
}
|
|
|
|
disconnect() {
|
|
return undefined;
|
|
}
|
|
|
|
__start() {
|
|
window.setTimeout(() => this.port.emitChunk(), 25);
|
|
}
|
|
}
|
|
|
|
class FakeMediaStreamSource {
|
|
connect(node: { __start?: () => void }) {
|
|
node.__start?.();
|
|
return node;
|
|
}
|
|
|
|
disconnect() {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
class FakeAudioContext {
|
|
state = "running";
|
|
destination = {};
|
|
sampleRate = 16000;
|
|
audioWorklet = {
|
|
addModule: async (_url: string) => {
|
|
probe.workletModuleLoads += 1;
|
|
},
|
|
};
|
|
|
|
async resume() {
|
|
probe.audioContextResumes += 1;
|
|
this.state = "running";
|
|
}
|
|
|
|
async close() {
|
|
this.state = "closed";
|
|
}
|
|
|
|
async decodeAudioData(_data: ArrayBuffer) {
|
|
return { duration: 0.12 };
|
|
}
|
|
|
|
createAnalyser() {
|
|
return new FakeAnalyser();
|
|
}
|
|
|
|
createBufferSource() {
|
|
return new FakeBufferSource();
|
|
}
|
|
|
|
createMediaStreamSource(_stream: unknown) {
|
|
return new FakeMediaStreamSource();
|
|
}
|
|
}
|
|
|
|
Object.defineProperty(window, "AudioWorkletNode", {
|
|
configurable: true,
|
|
value: FakeAudioWorkletNode,
|
|
});
|
|
Object.defineProperty(window, "AudioContext", {
|
|
configurable: true,
|
|
value: FakeAudioContext,
|
|
});
|
|
Object.defineProperty(window, "webkitAudioContext", {
|
|
configurable: true,
|
|
value: FakeAudioContext,
|
|
});
|
|
|
|
HTMLMediaElement.prototype.play = function patchedPlay() {
|
|
if (opts.blockMediaElementPlayback) {
|
|
probe.mediaPlayRejections += 1;
|
|
return Promise.reject(new DOMException("Synthetic autoplay block", "NotAllowedError"));
|
|
}
|
|
probe.audioPlays += 1;
|
|
window.setTimeout(() => {
|
|
this.dispatchEvent(new Event("ended"));
|
|
}, 120);
|
|
return Promise.resolve();
|
|
};
|
|
}, options);
|
|
}
|
|
|
|
async function readVoiceUiProbe(page: Page): Promise<VoiceUiProbeState> {
|
|
return page.evaluate(() => {
|
|
const probe = (window as Window & { __voiceUiProbe?: VoiceUiProbeState }).__voiceUiProbe;
|
|
if (!probe) throw new Error("voice UI probe was not installed");
|
|
return probe;
|
|
});
|
|
}
|
|
|
|
async function parsedVoiceUiEvents(page: Page): Promise<Array<{ type?: string; [key: string]: unknown }>> {
|
|
const probe = await readVoiceUiProbe(page);
|
|
return probe.messages
|
|
.filter((message) => message.direction === "received" && message.kind === "text" && message.data)
|
|
.map((message) => JSON.parse(message.data ?? "{}") as { type?: string; [key: string]: unknown });
|
|
}
|
|
|
|
test.describe("voice cascade success path", () => {
|
|
test("runs STT, client turn, TTS, and audio chunks against controlled providers @single-run", async ({
|
|
page,
|
|
}, testInfo) => {
|
|
test.setTimeout(60_000);
|
|
|
|
const openai = await startFakeOpenAI();
|
|
const engine = await startFakeEngine();
|
|
const api = await startApi({ engineURL: engine.url, openAIBaseURL: openai.url });
|
|
try {
|
|
const health = await page.request.get(`${api.baseURL}/voice/health`);
|
|
expect(health.ok(), await health.text()).toBeTruthy();
|
|
await expect(await health.json()).toMatchObject({ status: "ok", available: true });
|
|
|
|
await page.goto(`${api.baseURL}/health`);
|
|
const browserSetup = await page.evaluate(async ({ apiBase, seededPersonaCode, workerIndex }) => {
|
|
const login = await fetch(`${apiBase}/auth/dev-login`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
email: `voice-success.${workerIndex}@hs.ac.kr`,
|
|
role: "learner",
|
|
display_name: "Voice Success",
|
|
}),
|
|
});
|
|
const loginBody = await login.text();
|
|
if (!login.ok) {
|
|
return { ok: false, step: "login", status: login.status, body: loginBody };
|
|
}
|
|
const onboarding = await fetch(`${apiBase}/users/me/onboarding`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
legal_name: "Voice Success",
|
|
affiliation: "한신대학교",
|
|
department: "상담심리학과",
|
|
grade_level: "3학년",
|
|
phone: "010-5555-5555",
|
|
contact_address: "경기도 오산시 한신대학교",
|
|
nickname: "Voice Success",
|
|
self_introduction: "음성 캐스케이드 성공 경로 검증용 사용자입니다.",
|
|
avatar_url: "",
|
|
terms_accepted: true,
|
|
privacy_accepted: true,
|
|
}),
|
|
});
|
|
const onboardingBody = await onboarding.text();
|
|
if (!onboarding.ok) {
|
|
return {
|
|
ok: false,
|
|
step: "onboarding",
|
|
status: onboarding.status,
|
|
body: onboardingBody,
|
|
};
|
|
}
|
|
const me = await fetch(`${apiBase}/auth/me`, { credentials: "include" });
|
|
const meBody = await me.text();
|
|
if (!me.ok) {
|
|
return { ok: false, step: "me", status: me.status, body: meBody };
|
|
}
|
|
const start = await fetch(`${apiBase}/sessions`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ persona_code: seededPersonaCode, theory_mode: "humanistic" }),
|
|
});
|
|
const startBody = await start.text();
|
|
if (!start.ok) {
|
|
return { ok: false, step: "sessions", status: start.status, body: startBody };
|
|
}
|
|
return {
|
|
ok: true,
|
|
me: JSON.parse(meBody) as unknown,
|
|
started: JSON.parse(startBody) as { session_id: string },
|
|
};
|
|
}, {
|
|
apiBase: api.baseURL,
|
|
seededPersonaCode: SEEDED_VOICE_PERSONA_CODE,
|
|
workerIndex: testInfo.workerIndex,
|
|
});
|
|
expect(browserSetup, api.logs()).toMatchObject({ ok: true });
|
|
if (!browserSetup.ok) throw new Error(JSON.stringify(browserSetup));
|
|
const started = browserSetup.started;
|
|
|
|
const result = await probeVoiceCascade(page, api.baseURL, started.session_id);
|
|
const events = result.messages.map((message) => JSON.parse(message) as { type: string; [key: string]: unknown });
|
|
|
|
expect(
|
|
result.code,
|
|
[
|
|
JSON.stringify(result, null, 2),
|
|
`fakeOpenAI=${JSON.stringify(openai.requests())}`,
|
|
`fakeEngine=${JSON.stringify(engine.requests())}`,
|
|
api.logs(),
|
|
].join("\n\n"),
|
|
).toBe(1000);
|
|
expect(events.some((event) => event.type === "degraded")).toBe(false);
|
|
expect(events.some((event) => event.type === "error")).toBe(false);
|
|
expect(events).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ type: "ready", session_id: started.session_id }),
|
|
expect.objectContaining({ type: "state", state: "listening" }),
|
|
expect.objectContaining({ type: "state", state: "thinking" }),
|
|
expect.objectContaining({ type: "transcript", text: "요즘 잠을 잘 못 자요." }),
|
|
expect.objectContaining({ type: "reply", text: "괜찮아요. 천천히 말해볼게요." }),
|
|
expect.objectContaining({ type: "state", state: "speaking" }),
|
|
expect.objectContaining({ type: "tts_end" }),
|
|
expect.objectContaining({ type: "state", state: "idle" }),
|
|
]),
|
|
);
|
|
expect(result.binaryChunks).toBeGreaterThan(0);
|
|
} finally {
|
|
await api.stop();
|
|
await engine.close();
|
|
await openai.close();
|
|
}
|
|
});
|
|
|
|
test("drives one voice turn through the Session mic UI with synthetic browser audio @single-run", async ({
|
|
page,
|
|
}, testInfo) => {
|
|
test.setTimeout(150_000);
|
|
|
|
const diagnostics: string[] = [];
|
|
page.on("pageerror", (error) => diagnostics.push(`pageerror: ${error.message}`));
|
|
page.on("console", (message) => {
|
|
if (message.type() === "error") diagnostics.push(`console: ${message.text()}`);
|
|
});
|
|
page.on("requestfailed", (request) => {
|
|
diagnostics.push(`requestfailed: ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ""}`);
|
|
});
|
|
page.on("response", (response) => {
|
|
const url = response.url();
|
|
if (
|
|
response.status() >= 400 &&
|
|
(url.includes("/sessions") || url.includes("/voice/ws") || url.includes("/voice/speech"))
|
|
) {
|
|
diagnostics.push(`response: ${response.status()} ${url}`);
|
|
}
|
|
});
|
|
|
|
await installSyntheticVoiceCapture(page, { blockMediaElementPlayback: true });
|
|
|
|
const openai = await startFakeOpenAI();
|
|
const engine = await startFakeEngine();
|
|
const webPort = await freePort();
|
|
const webBaseURL = `http://127.0.0.1:${webPort}`;
|
|
const api = await startApi({
|
|
engineURL: engine.url,
|
|
openAIBaseURL: openai.url,
|
|
frontendBaseURL: webBaseURL,
|
|
});
|
|
const web = await startWeb({ apiBaseURL: api.baseURL, port: webPort });
|
|
try {
|
|
await page.route("**/personas", async (route) => {
|
|
if (route.request().method() !== "GET") {
|
|
await route.fallback();
|
|
return;
|
|
}
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: "application/json",
|
|
headers: {
|
|
"access-control-allow-origin": web.baseURL,
|
|
"access-control-allow-credentials": "true",
|
|
},
|
|
body: JSON.stringify([
|
|
{
|
|
code: SEEDED_VOICE_PERSONA_CODE,
|
|
display_name: "Voice UI fixture",
|
|
difficulty: "hard",
|
|
theory_target: ["humanistic"],
|
|
demographics: { age_band: "teen" },
|
|
presenting_summary: "Synthetic browser audio UI proof",
|
|
voice_preset: "soft-young-fem",
|
|
source: "database",
|
|
degraded: false,
|
|
},
|
|
]),
|
|
});
|
|
});
|
|
|
|
await page.goto(`${web.baseURL}/login`, { waitUntil: "domcontentloaded" });
|
|
const browserLogin = await page.evaluate(async ({ apiBase, workerIndex }) => {
|
|
const login = await fetch(`${apiBase}/auth/dev-login`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
email: `voice-ui.${workerIndex}@hs.ac.kr`,
|
|
role: "learner",
|
|
display_name: "Voice UI",
|
|
}),
|
|
});
|
|
const loginBody = await login.text();
|
|
if (!login.ok) {
|
|
return { ok: false, step: "login", status: login.status, body: loginBody };
|
|
}
|
|
const onboarding = await fetch(`${apiBase}/users/me/onboarding`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
legal_name: "Voice UI",
|
|
affiliation: "한신대학교",
|
|
department: "상담심리학과",
|
|
grade_level: "3학년",
|
|
phone: "010-6666-6666",
|
|
contact_address: "경기도 오산시 한신대학교",
|
|
nickname: "Voice UI",
|
|
self_introduction: "브라우저 음성 UI 검증용 사용자입니다.",
|
|
avatar_url: "",
|
|
terms_accepted: true,
|
|
privacy_accepted: true,
|
|
}),
|
|
});
|
|
const onboardingBody = await onboarding.text();
|
|
if (!onboarding.ok) {
|
|
return {
|
|
ok: false,
|
|
step: "onboarding",
|
|
status: onboarding.status,
|
|
body: onboardingBody,
|
|
};
|
|
}
|
|
const me = await fetch(`${apiBase}/auth/me`, { credentials: "include" });
|
|
const meBody = await me.text();
|
|
if (!me.ok) {
|
|
return { ok: false, step: "me", status: me.status, body: meBody };
|
|
}
|
|
return { ok: true, me: JSON.parse(meBody) as unknown };
|
|
}, {
|
|
apiBase: api.baseURL,
|
|
workerIndex: testInfo.workerIndex,
|
|
});
|
|
expect(browserLogin, api.logs()).toMatchObject({ ok: true });
|
|
|
|
await page.goto(`${web.baseURL}/learn/session/${SEEDED_VOICE_PERSONA_CODE}`);
|
|
await expect(
|
|
page.locator(".sx-prestart__actions button").first(),
|
|
diagnostics.join("\n") || (await page.locator("#root").innerText().catch(() => "")),
|
|
).toBeVisible();
|
|
await page.locator(".sx-prestart__actions button").first().click();
|
|
await expect(
|
|
page.locator(".sx-page.sx-page--active"),
|
|
[
|
|
...diagnostics,
|
|
`apiLogs=${api.logs()}`,
|
|
`pageText=${await page.locator("#root").innerText().catch(() => "")}`,
|
|
].join("\n\n"),
|
|
).toBeVisible({ timeout: 20_000 });
|
|
|
|
const textInput = page.getByLabel("학습자 발화 입력");
|
|
const sendButton = page.getByRole("button", { name: "보내기", exact: true });
|
|
await textInput.fill("텍스트로 말해도 내담자 음성을 들려주세요.");
|
|
await sendButton.click();
|
|
await expect
|
|
.poll(
|
|
() => openai.requests().filter((request) => request === "POST /v1/audio/speech").length,
|
|
{ timeout: 30_000 },
|
|
)
|
|
.toBeGreaterThan(0);
|
|
await expect
|
|
.poll(async () => (await readVoiceUiProbe(page)).audioBufferStarts, { timeout: 30_000 })
|
|
.toBeGreaterThan(0);
|
|
await expect(page.locator(".sx-utt").filter({ hasText: "괜찮아요. 천천히 말해볼게요." })).toBeVisible();
|
|
|
|
const freshVoiceSession = await page.evaluate(async ({ apiBase, personaCode }) => {
|
|
const response = await fetch(`${apiBase}/sessions`, {
|
|
method: "POST",
|
|
credentials: "include",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ persona_code: personaCode, theory_mode: "humanistic" }),
|
|
});
|
|
return {
|
|
ok: response.ok,
|
|
status: response.status,
|
|
body: await response.json() as { session_id?: string },
|
|
};
|
|
}, { apiBase: api.baseURL, personaCode: SEEDED_VOICE_PERSONA_CODE });
|
|
expect(freshVoiceSession, api.logs()).toMatchObject({ ok: true });
|
|
expect(typeof freshVoiceSession.body.session_id).toBe("string");
|
|
await page.goto(`${web.baseURL}/learn/session/${freshVoiceSession.body.session_id}`);
|
|
await expect(page.locator(".sx-page.sx-page--active")).toBeVisible({ timeout: 20_000 });
|
|
|
|
const mic = page.locator(".sx-mic");
|
|
await expect(mic).toBeEnabled();
|
|
await mic.click();
|
|
|
|
await expect
|
|
.poll(async () => (await readVoiceUiProbe(page)).getUserMediaCalls, { timeout: 10_000 })
|
|
.toBeGreaterThan(0);
|
|
await expect
|
|
.poll(async () => (await readVoiceUiProbe(page)).workletModuleLoads, { timeout: 10_000 })
|
|
.toBeGreaterThan(0);
|
|
await expect
|
|
.poll(async () => (await readVoiceUiProbe(page)).workletNodes, { timeout: 10_000 })
|
|
.toBeGreaterThan(0);
|
|
await expect
|
|
.poll(async () => (await readVoiceUiProbe(page)).workletChunks, { timeout: 10_000 })
|
|
.toBeGreaterThan(0);
|
|
await expect
|
|
.poll(async () => {
|
|
const probe = await readVoiceUiProbe(page);
|
|
return probe.messages.some(
|
|
(message) =>
|
|
message.direction === "sent" &&
|
|
message.kind === "text" &&
|
|
message.data?.includes('"audio_start"') &&
|
|
message.data?.includes('"format":"pcm"') &&
|
|
message.data?.includes('"sample_rate":16000'),
|
|
);
|
|
}, { timeout: 10_000 })
|
|
.toBeTruthy();
|
|
await expect
|
|
.poll(async () => {
|
|
const probe = await readVoiceUiProbe(page);
|
|
return probe.messages.some(
|
|
(message) => message.direction === "sent" && message.kind === "binary",
|
|
);
|
|
}, { timeout: 10_000 })
|
|
.toBeTruthy();
|
|
|
|
await mic.click();
|
|
await expect
|
|
.poll(async () => {
|
|
const probe = await readVoiceUiProbe(page);
|
|
return probe.messages.some(
|
|
(message) =>
|
|
message.direction === "sent" &&
|
|
message.kind === "text" &&
|
|
message.data?.includes('"audio_end"'),
|
|
);
|
|
}, { timeout: 10_000 })
|
|
.toBeTruthy();
|
|
const sentAudioEndControls = (await readVoiceUiProbe(page)).messages
|
|
.filter(
|
|
(message) =>
|
|
message.direction === "sent" &&
|
|
message.kind === "text" &&
|
|
message.data?.includes('"audio_end"'),
|
|
)
|
|
.map((message) => JSON.parse(message.data ?? "{}") as Record<string, unknown>);
|
|
expect(
|
|
sentAudioEndControls.some((control) => {
|
|
const events = Array.isArray(control.provider_events) ? control.provider_events : [];
|
|
return (
|
|
control.format === "pcm" &&
|
|
control.silence_ms === 1250 &&
|
|
control.barge_in === false &&
|
|
events.some((event) => {
|
|
const record = event as Record<string, unknown>;
|
|
return (
|
|
record.type === "voice_activity" &&
|
|
record.source === "browser_audio_worklet" &&
|
|
record.duration_ms === 350
|
|
);
|
|
}) &&
|
|
events.some((event) => {
|
|
const record = event as Record<string, unknown>;
|
|
return (
|
|
record.type === "silence" &&
|
|
record.source === "browser_audio_worklet" &&
|
|
record.duration_ms === 1250
|
|
);
|
|
})
|
|
);
|
|
}),
|
|
).toBeTruthy();
|
|
|
|
await expect
|
|
.poll(async () => {
|
|
const events = await parsedVoiceUiEvents(page);
|
|
return {
|
|
transcript: events.some((event) => event.type === "transcript"),
|
|
reply: events.some((event) => event.type === "reply"),
|
|
ttsEnd: events.some((event) => event.type === "tts_end"),
|
|
errors: events.filter((event) => event.type === "error" || event.type === "degraded"),
|
|
};
|
|
}, { timeout: 30_000 })
|
|
.toEqual({ transcript: true, reply: true, ttsEnd: true, errors: [] });
|
|
|
|
const events = await parsedVoiceUiEvents(page);
|
|
const transcript = events.find((event) => event.type === "transcript")?.text;
|
|
const reply = events.find((event) => event.type === "reply")?.text;
|
|
expect(typeof transcript).toBe("string");
|
|
expect(typeof reply).toBe("string");
|
|
await expect(page.locator(".sx-utt").filter({ hasText: String(transcript) })).toBeVisible();
|
|
await expect(page.locator(".sx-utt").filter({ hasText: String(reply) })).toBeVisible();
|
|
|
|
const probe = await readVoiceUiProbe(page);
|
|
expect(probe.audioContextResumes).toBeGreaterThan(0);
|
|
expect(probe.audioBufferStarts).toBeGreaterThan(0);
|
|
expect(probe.audioPlays).toBe(0);
|
|
expect(probe.mediaPlayRejections).toBe(0);
|
|
expect(probe.messages.some((message) => message.direction === "received" && message.kind === "binary")).toBe(
|
|
true,
|
|
);
|
|
expect(openai.requests()).toEqual(
|
|
expect.arrayContaining(["POST /v1/audio/transcriptions", "POST /v1/audio/speech"]),
|
|
);
|
|
expect(engine.requests()).toContain("POST /v1/generate");
|
|
} finally {
|
|
await web.stop();
|
|
await api.stop();
|
|
await engine.close();
|
|
await openai.close();
|
|
}
|
|
});
|
|
});
|