개선관리 요구사항과 Google 로그인을 완료
This commit is contained in:
Yun Chan
parent
cc0a15b7c6
commit
2a39636163
112 changed files with 10166 additions and 527 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { expect, test, type Page } from "@playwright/test";
|
||||
import {
|
||||
completeAlliancePreCheckpoint,
|
||||
|
|
@ -10,6 +11,8 @@ import {
|
|||
|
||||
interface SessionStartResponse {
|
||||
session_id: string;
|
||||
case_id: string;
|
||||
session_no: number;
|
||||
degraded: boolean;
|
||||
}
|
||||
|
||||
|
|
@ -173,6 +176,126 @@ interface PrepostMeasuresResponse {
|
|||
measures: PrepostMeasureItem[];
|
||||
}
|
||||
|
||||
interface ControlledFailureGateway {
|
||||
url: string;
|
||||
clientRequests: () => number;
|
||||
evaluatorRequests: () => number;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
async function startControlledEvaluationFailureGateway(
|
||||
model: string,
|
||||
fallbackProvider: string,
|
||||
): Promise<ControlledFailureGateway> {
|
||||
let clientRequestCount = 0;
|
||||
let evaluatorRequestCount = 0;
|
||||
|
||||
const sendJson = (response: ServerResponse, status: number, payload: unknown) => {
|
||||
response.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
||||
response.end(JSON.stringify(payload));
|
||||
};
|
||||
const readJson = async (request: IncomingMessage) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const raw = Buffer.concat(chunks).toString("utf8");
|
||||
return raw ? (JSON.parse(raw) as Record<string, unknown>) : {};
|
||||
};
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
const provider = url.searchParams.get("provider") || fallbackProvider;
|
||||
if (request.method === "GET" && url.pathname === "/v1/capabilities") {
|
||||
sendJson(response, 200, {
|
||||
provider,
|
||||
available: true,
|
||||
source: "live_cli",
|
||||
models: [
|
||||
{
|
||||
id: model,
|
||||
label: "REQ-007 controlled gateway",
|
||||
description: "client succeeds while evaluator returns HTTP 500",
|
||||
reasoning_efforts: ["low", "medium", "high", "xhigh", "max", "ultra"],
|
||||
default_reasoning_effort: "high",
|
||||
is_default: true,
|
||||
},
|
||||
],
|
||||
default_model: model,
|
||||
default_reasoning_effort: "high",
|
||||
detail: "controlled gateway ready",
|
||||
fetched_at: Date.now() / 1000,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request.method === "GET" && ["/ready", "/health"].includes(url.pathname)) {
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
status: "ok",
|
||||
detail: "controlled gateway ready",
|
||||
cached: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request.method === "DELETE" && url.pathname.startsWith("/session/")) {
|
||||
sendJson(response, 200, { closed: true });
|
||||
return;
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/v1/generate") {
|
||||
const payload = await readJson(request);
|
||||
if (payload.ai_role === "evaluator") {
|
||||
evaluatorRequestCount += 1;
|
||||
sendJson(response, 500, { detail: "controlled evaluator failure" });
|
||||
return;
|
||||
}
|
||||
clientRequestCount += 1;
|
||||
sendJson(response, 200, {
|
||||
text: "조금 더 이야기해볼게요.",
|
||||
model,
|
||||
provider: String(payload.provider || fallbackProvider),
|
||||
tokens_in: 7,
|
||||
tokens_out: 9,
|
||||
cost_usd: 0,
|
||||
});
|
||||
return;
|
||||
}
|
||||
sendJson(response, 404, { detail: `unexpected controlled gateway path: ${url.pathname}` });
|
||||
} catch (error) {
|
||||
sendJson(response, 500, {
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (error: Error) => reject(error);
|
||||
server.once("error", onError);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", onError);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
throw new Error("controlled gateway did not expose a TCP port");
|
||||
}
|
||||
|
||||
return {
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
clientRequests: () => clientRequestCount,
|
||||
evaluatorRequests: () => evaluatorRequestCount,
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
});
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function expectResponseOk(response: { ok: () => boolean; text: () => Promise<string> }) {
|
||||
if (!response.ok()) {
|
||||
expect(response.ok(), await response.text()).toBeTruthy();
|
||||
|
|
@ -1357,6 +1480,158 @@ test.describe("session persistence", () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("keeps the failed review durable while the next persona session accepts a stored turn @single-run", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180_000);
|
||||
|
||||
await withGlobalEngineConfigLock("req-007-evaluation-failure-continuity", async () => {
|
||||
await signInAsAdmin(page);
|
||||
const engineResponse = await page.request.get("/api/admin/engine-config");
|
||||
await expectResponseOk(engineResponse);
|
||||
const currentEngine = (await engineResponse.json()) as AdminEngineConfigResponse;
|
||||
expect(currentEngine.source, "REQ-007 continuity proof requires the real DB lane").toBe(
|
||||
"database",
|
||||
);
|
||||
expect(currentEngine.durable, "REQ-007 engine configuration must be durable").toBe(true);
|
||||
const gateway = await startControlledEvaluationFailureGateway(
|
||||
currentEngine.model,
|
||||
currentEngine.engine_mode,
|
||||
);
|
||||
|
||||
try {
|
||||
const controlledPatch = await page.request.patch("/api/admin/engine-config", {
|
||||
data: {
|
||||
engine_mode: currentEngine.engine_mode,
|
||||
engine_url: gateway.url,
|
||||
model: currentEngine.model,
|
||||
reasoning_effort: currentEngine.reasoning_effort,
|
||||
},
|
||||
});
|
||||
await expectResponseOk(controlledPatch);
|
||||
|
||||
await signInAsLearner(page);
|
||||
const persona = await fetchAvailablePersona(page);
|
||||
const firstStartResponse = await page.request.post("/api/sessions", {
|
||||
data: { persona_code: persona.code, theory_mode: "humanistic" },
|
||||
});
|
||||
await expectResponseOk(firstStartResponse);
|
||||
expect(firstStartResponse.status()).toBe(201);
|
||||
const first = (await firstStartResponse.json()) as SessionStartResponse;
|
||||
expect(first.degraded).toBe(false);
|
||||
|
||||
const firstTurnResponse = await page.request.post(
|
||||
`/api/sessions/${first.session_id}/turn`,
|
||||
{
|
||||
data: { text: "오늘 가장 버거웠던 순간부터 이야기해도 괜찮을까요?" },
|
||||
},
|
||||
);
|
||||
await expectResponseOk(firstTurnResponse);
|
||||
|
||||
const endResponse = await page.request.post(`/api/sessions/${first.session_id}/end`);
|
||||
await expectResponseOk(endResponse);
|
||||
expect(endResponse.status()).toBe(200);
|
||||
|
||||
const endedDetailResponse = await page.request.get(`/api/sessions/${first.session_id}`);
|
||||
await expectResponseOk(endedDetailResponse);
|
||||
const endedDetail = (await endedDetailResponse.json()) as SessionDetailResponse;
|
||||
expect(endedDetail.status).toBe("ended");
|
||||
expect(endedDetail.ended_at).toBeTruthy();
|
||||
expect(endedDetail.turns?.length ?? 0).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const nextStartResponse = await page.request.post("/api/sessions", {
|
||||
data: { persona_code: persona.code, theory_mode: "humanistic" },
|
||||
});
|
||||
await expectResponseOk(nextStartResponse);
|
||||
expect(nextStartResponse.status()).toBe(201);
|
||||
const next = (await nextStartResponse.json()) as SessionStartResponse;
|
||||
expect(next.degraded).toBe(false);
|
||||
expect(next.session_id).not.toBe(first.session_id);
|
||||
expect(next.case_id).toBe(first.case_id);
|
||||
expect(next.session_no).toBe(first.session_no + 1);
|
||||
|
||||
const nextTurnResponse = await page.request.post(
|
||||
`/api/sessions/${next.session_id}/turn`,
|
||||
{
|
||||
data: { text: "지난 회기의 이야기를 이어서 천천히 살펴볼까요?" },
|
||||
},
|
||||
);
|
||||
await expectResponseOk(nextTurnResponse);
|
||||
const nextTurn = (await nextTurnResponse.json()) as {
|
||||
client_reply?: string | null;
|
||||
output_error?: string | null;
|
||||
};
|
||||
expect(nextTurn.client_reply).toBe("조금 더 이야기해볼게요.");
|
||||
expect(nextTurn.output_error ?? null).toBeNull();
|
||||
|
||||
const nextDetailResponse = await page.request.get(`/api/sessions/${next.session_id}`);
|
||||
await expectResponseOk(nextDetailResponse);
|
||||
const nextDetail = (await nextDetailResponse.json()) as SessionDetailResponse;
|
||||
expect(nextDetail.status).toBe("active");
|
||||
expect(nextDetail.turns?.length ?? 0).toBeGreaterThanOrEqual(2);
|
||||
expect(gateway.clientRequests()).toBeGreaterThanOrEqual(2);
|
||||
await expect
|
||||
.poll(() => gateway.evaluatorRequests(), {
|
||||
timeout: 30_000,
|
||||
message: "the ended session must reach the controlled evaluator failure",
|
||||
})
|
||||
.toBeGreaterThan(0);
|
||||
|
||||
await signInAsTeacher(page);
|
||||
await expect
|
||||
.poll(async () => evaluationState(page, first.session_id), {
|
||||
timeout: 30_000,
|
||||
intervals: [250, 500, 1_000],
|
||||
message: "the evaluator HTTP 500 must become a durable error row",
|
||||
})
|
||||
.toMatch(/^error:.*:durable$/);
|
||||
|
||||
const reviewResponse = await page.request.get(`/api/sessions/${first.session_id}/review`);
|
||||
await expectResponseOk(reviewResponse);
|
||||
const review = (await reviewResponse.json()) as SessionReviewResponse;
|
||||
expect(review.reviewReady).toBe(false);
|
||||
expect(review.degraded).toBe(true);
|
||||
expect(review.supervisorState).toBe("평가 실패");
|
||||
expect(review.summary ?? "").toContain("AI 평가 재시도가 필요합니다");
|
||||
|
||||
await page.goto(`/teach/session/${first.session_id}/review`);
|
||||
await expect(page.getByRole("button", { name: "AI 평가 재시도" })).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
const evaluatorRequestsBeforeRetry = gateway.evaluatorRequests();
|
||||
const retryResponse = await page.request.post(
|
||||
`/api/eval/sessions/${first.session_id}/reevaluate`,
|
||||
{ data: { scope: "session_end" } },
|
||||
);
|
||||
expect(retryResponse.status()).toBe(503);
|
||||
await expect
|
||||
.poll(() => gateway.evaluatorRequests(), { timeout: 10_000 })
|
||||
.toBeGreaterThan(evaluatorRequestsBeforeRetry);
|
||||
await expect
|
||||
.poll(async () => evaluationState(page, first.session_id), {
|
||||
timeout: 10_000,
|
||||
})
|
||||
.toMatch(/^error:.*:durable$/);
|
||||
} finally {
|
||||
try {
|
||||
await signInAsAdmin(page);
|
||||
const restoreResponse = await page.request.patch("/api/admin/engine-config", {
|
||||
data: {
|
||||
engine_mode: currentEngine.engine_mode,
|
||||
engine_url: currentEngine.engine_url,
|
||||
model: currentEngine.model,
|
||||
reasoning_effort: currentEngine.reasoning_effort,
|
||||
},
|
||||
});
|
||||
await expectResponseOk(restoreResponse);
|
||||
} finally {
|
||||
await gateway.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("finishes session end evaluation into a durable DB-backed teacher review @single-run", async ({
|
||||
page,
|
||||
}) => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue