Stabilize runtime auth and E2E coverage
This commit is contained in:
Yun Chan
parent
6a3e3b541c
commit
188e899394
133 changed files with 55987 additions and 6775 deletions
360
apps/web/e2e/voice-success.spec.ts
Normal file
360
apps/web/e2e/voice-success.spec.ts
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
import { expect, test, type Page } from "@playwright/test";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import http, { type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import net from "node:net";
|
||||
|
||||
interface TestServer {
|
||||
url: string;
|
||||
requests: () => string[];
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface SpawnedApi {
|
||||
baseURL: string;
|
||||
logs: () => string;
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface VoiceProbe {
|
||||
code: number;
|
||||
messages: string[];
|
||||
binaryChunks: number;
|
||||
}
|
||||
|
||||
// This fixture intentionally starts a DB-offline API with ALLOW_SEED_PERSONA_FALLBACK=true
|
||||
// so the voice provider cascade can be exercised without a Postgres dependency.
|
||||
const SEEDED_VOICE_PERSONA_CODE = "P1";
|
||||
|
||||
function readBody(req: IncomingMessage): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
||||
req.on("end", () => resolve(Buffer.concat(chunks)));
|
||||
req.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
const port = typeof address === "object" && address ? address.port : 0;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startHttpServer(
|
||||
handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>,
|
||||
): Promise<TestServer> {
|
||||
const port = await freePort();
|
||||
const requests: string[] = [];
|
||||
const server = http.createServer((req, res) => {
|
||||
requests.push(`${req.method ?? "?"} ${req.url ?? "?"}`);
|
||||
void Promise.resolve(handler(req, res)).catch((err) => {
|
||||
res.writeHead(500, { "content-type": "text/plain" });
|
||||
res.end(String(err));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(port, "127.0.0.1", resolve));
|
||||
return {
|
||||
url: `http://127.0.0.1:${port}`,
|
||||
requests: () => requests,
|
||||
close: () => new Promise((resolve) => server.close(() => resolve())),
|
||||
};
|
||||
}
|
||||
|
||||
async function startFakeOpenAI(): Promise<TestServer> {
|
||||
return startHttpServer(async (req, res) => {
|
||||
await readBody(req);
|
||||
if (req.method === "POST" && req.url === "/v1/audio/transcriptions") {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ text: "요즘 잠을 잘 못 자요.", language: "ko", duration: 1.2 }));
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && req.url === "/v1/audio/speech") {
|
||||
res.writeHead(200, { "content-type": "audio/mpeg" });
|
||||
res.end(Buffer.alloc(8192, 128));
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "not found" }));
|
||||
});
|
||||
}
|
||||
|
||||
async function startFakeEngine(): Promise<TestServer> {
|
||||
return startHttpServer(async (req, res) => {
|
||||
await readBody(req);
|
||||
if (req.method === "GET" && req.url === "/health") {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true, engine: "fake" }));
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && req.url === "/v1/generate") {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
text: "괜찮아요. 천천히 말해볼게요.",
|
||||
model: "fake-client",
|
||||
provider: "e2e",
|
||||
tokens_in: 1,
|
||||
tokens_out: 1,
|
||||
cost_usd: 0,
|
||||
inference_geo: "us",
|
||||
structured: null,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "not found" }));
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForApi(baseURL: string, proc: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
const started = Date.now();
|
||||
let lastError = "";
|
||||
while (Date.now() - started < 20_000) {
|
||||
if (proc.exitCode !== null) {
|
||||
throw new Error(`API exited early with code ${proc.exitCode}: ${lastError}`);
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${baseURL}/health`);
|
||||
if (response.ok) return;
|
||||
lastError = await response.text();
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error(`Timed out waiting for API ${baseURL}: ${lastError}`);
|
||||
}
|
||||
|
||||
async function startApi({
|
||||
engineURL,
|
||||
openAIBaseURL,
|
||||
}: {
|
||||
engineURL: string;
|
||||
openAIBaseURL: string;
|
||||
}): Promise<SpawnedApi> {
|
||||
const port = await freePort();
|
||||
const baseURL = `http://127.0.0.1:${port}`;
|
||||
const localPython311 = process.env.USERPROFILE
|
||||
? `${process.env.USERPROFILE}\\AppData\\Local\\Programs\\Python\\Python311\\python.exe`
|
||||
: "";
|
||||
const python =
|
||||
process.env.E2E_PYTHON ??
|
||||
process.env.PYTHON311 ??
|
||||
(localPython311 && existsSync(localPython311) ? localPython311 : (process.env.PYTHON ?? "python"));
|
||||
const proc = spawn(
|
||||
python,
|
||||
[
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"app.main:app",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
String(port),
|
||||
"--log-level",
|
||||
"debug",
|
||||
],
|
||||
{
|
||||
cwd: "../api",
|
||||
env: {
|
||||
...process.env,
|
||||
PYTHONUNBUFFERED: "1",
|
||||
ENVIRONMENT: "dev",
|
||||
AUTH_DEV_LOGIN_ENABLED: "true",
|
||||
AUTH_ALLOWED_EMAIL_DOMAINS: '["hs.ac.kr","twentyoz.kr"]',
|
||||
ALLOW_SEED_PERSONA_FALLBACK: "true",
|
||||
DATABASE_URL: "postgresql://user:pass@127.0.0.1:1/vignette",
|
||||
DB_POOL_MIN_SIZE: "0",
|
||||
DB_COMMAND_TIMEOUT: "1",
|
||||
ENGINE_URL: engineURL,
|
||||
ENGINE_MODE: "claude_api",
|
||||
ENGINE_TIMEOUT: "10",
|
||||
ENGINE_CONNECT_TIMEOUT: "2",
|
||||
OPENAI_API_KEY: "e2e-fake-key",
|
||||
OPENAI_BASE_URL: `${openAIBaseURL}/v1`,
|
||||
FRONTEND_BASE_URL: "http://localhost:5173",
|
||||
CORS_ORIGINS: '["http://localhost:5173"]',
|
||||
},
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
let logs = "";
|
||||
proc.stdout.on("data", (chunk) => {
|
||||
logs += String(chunk).slice(-4000);
|
||||
});
|
||||
proc.stderr.on("data", (chunk) => {
|
||||
logs += String(chunk).slice(-4000);
|
||||
});
|
||||
await waitForApi(baseURL, proc).catch((err) => {
|
||||
proc.kill();
|
||||
throw new Error(`${err instanceof Error ? err.message : String(err)}\n${logs}`);
|
||||
});
|
||||
return {
|
||||
baseURL,
|
||||
logs: () => logs,
|
||||
stop: async () => {
|
||||
if (proc.exitCode === null) proc.kill();
|
||||
await new Promise<void>((resolve) => {
|
||||
if (proc.exitCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
proc.once("exit", () => resolve());
|
||||
setTimeout(resolve, 3000);
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function probeVoiceCascade(page: Page, apiBaseURL: string, sessionId: string): Promise<VoiceProbe> {
|
||||
return page.evaluate(
|
||||
({ apiBase, sid }) =>
|
||||
new Promise<VoiceProbe>((resolve) => {
|
||||
const wsURL = new URL(`/voice/ws?session_id=${encodeURIComponent(sid)}`, apiBase);
|
||||
wsURL.protocol = "ws:";
|
||||
const ws = new WebSocket(wsURL.href);
|
||||
ws.binaryType = "arraybuffer";
|
||||
const messages: string[] = [];
|
||||
let binaryChunks = 0;
|
||||
let sawTtsEnd = false;
|
||||
const timeout = window.setTimeout(() => {
|
||||
ws.close();
|
||||
resolve({ code: -1, messages, binaryChunks });
|
||||
}, 20_000);
|
||||
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({ type: "audio_start", format: "webm" }));
|
||||
ws.send(new Uint8Array([1, 2, 3, 4, 5, 6]).buffer);
|
||||
ws.send(JSON.stringify({ type: "audio_end", format: "webm" }));
|
||||
};
|
||||
ws.onmessage = (event) => {
|
||||
if (typeof event.data === "string") {
|
||||
messages.push(event.data);
|
||||
try {
|
||||
const parsed = JSON.parse(event.data) as { type?: string; state?: string };
|
||||
if (parsed.type === "tts_end") sawTtsEnd = true;
|
||||
if (sawTtsEnd && parsed.type === "state" && parsed.state === "idle") {
|
||||
ws.send(JSON.stringify({ type: "close" }));
|
||||
}
|
||||
} catch {
|
||||
messages.push(JSON.stringify({ type: "error", detail: "invalid json from ws" }));
|
||||
}
|
||||
} else {
|
||||
binaryChunks += 1;
|
||||
}
|
||||
};
|
||||
ws.onerror = () => {
|
||||
messages.push(JSON.stringify({ type: "error", detail: "browser websocket error" }));
|
||||
};
|
||||
ws.onclose = (event) => {
|
||||
window.clearTimeout(timeout);
|
||||
resolve({ code: event.code, messages, binaryChunks });
|
||||
};
|
||||
}),
|
||||
{ apiBase: apiBaseURL, sid: sessionId },
|
||||
);
|
||||
}
|
||||
|
||||
test.describe("voice cascade success path", () => {
|
||||
test("runs STT, client turn, TTS, and audio chunks against controlled providers @single-run", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
const openai = await startFakeOpenAI();
|
||||
const engine = await startFakeEngine();
|
||||
const api = await startApi({ engineURL: engine.url, openAIBaseURL: openai.url });
|
||||
try {
|
||||
const health = await page.request.get(`${api.baseURL}/voice/health`);
|
||||
expect(health.ok(), await health.text()).toBeTruthy();
|
||||
await expect(await health.json()).toMatchObject({ status: "ok", available: true });
|
||||
|
||||
await page.goto(`${api.baseURL}/health`);
|
||||
const browserSetup = await page.evaluate(async ({ apiBase, seededPersonaCode, workerIndex }) => {
|
||||
const login = await fetch(`${apiBase}/auth/dev-login`, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
email: `voice-success.${workerIndex}@hs.ac.kr`,
|
||||
role: "learner",
|
||||
display_name: "Voice Success",
|
||||
}),
|
||||
});
|
||||
const loginBody = await login.text();
|
||||
if (!login.ok) {
|
||||
return { ok: false, step: "login", status: login.status, body: loginBody };
|
||||
}
|
||||
const me = await fetch(`${apiBase}/auth/me`, { credentials: "include" });
|
||||
const meBody = await me.text();
|
||||
if (!me.ok) {
|
||||
return { ok: false, step: "me", status: me.status, body: meBody };
|
||||
}
|
||||
const start = await fetch(`${apiBase}/sessions`, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ persona_code: seededPersonaCode, theory_mode: "humanistic" }),
|
||||
});
|
||||
const startBody = await start.text();
|
||||
if (!start.ok) {
|
||||
return { ok: false, step: "sessions", status: start.status, body: startBody };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
me: JSON.parse(meBody) as unknown,
|
||||
started: JSON.parse(startBody) as { session_id: string },
|
||||
};
|
||||
}, {
|
||||
apiBase: api.baseURL,
|
||||
seededPersonaCode: SEEDED_VOICE_PERSONA_CODE,
|
||||
workerIndex: testInfo.workerIndex,
|
||||
});
|
||||
expect(browserSetup, api.logs()).toMatchObject({ ok: true });
|
||||
if (!browserSetup.ok) throw new Error(JSON.stringify(browserSetup));
|
||||
const started = browserSetup.started;
|
||||
|
||||
const result = await probeVoiceCascade(page, api.baseURL, started.session_id);
|
||||
const events = result.messages.map((message) => JSON.parse(message) as { type: string; [key: string]: unknown });
|
||||
|
||||
expect(
|
||||
result.code,
|
||||
[
|
||||
JSON.stringify(result, null, 2),
|
||||
`fakeOpenAI=${JSON.stringify(openai.requests())}`,
|
||||
`fakeEngine=${JSON.stringify(engine.requests())}`,
|
||||
api.logs(),
|
||||
].join("\n\n"),
|
||||
).toBe(1000);
|
||||
expect(events.some((event) => event.type === "degraded")).toBe(false);
|
||||
expect(events.some((event) => event.type === "error")).toBe(false);
|
||||
expect(events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ type: "ready", session_id: started.session_id }),
|
||||
expect.objectContaining({ type: "state", state: "listening" }),
|
||||
expect.objectContaining({ type: "state", state: "thinking" }),
|
||||
expect.objectContaining({ type: "transcript", text: "요즘 잠을 잘 못 자요." }),
|
||||
expect.objectContaining({ type: "reply", text: "괜찮아요. 천천히 말해볼게요." }),
|
||||
expect.objectContaining({ type: "state", state: "speaking" }),
|
||||
expect.objectContaining({ type: "tts_chunk", seq: 0 }),
|
||||
expect.objectContaining({ type: "tts_end" }),
|
||||
expect.objectContaining({ type: "state", state: "idle" }),
|
||||
]),
|
||||
);
|
||||
expect(result.binaryChunks).toBeGreaterThan(0);
|
||||
} finally {
|
||||
await api.stop();
|
||||
await engine.close();
|
||||
await openai.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue