433 lines
14 KiB
TypeScript
433 lines
14 KiB
TypeScript
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
import {
|
|
expect,
|
|
test,
|
|
type APIResponse,
|
|
type Page,
|
|
} from "@playwright/test";
|
|
import {
|
|
completeAlliancePreCheckpoint,
|
|
useRealApi,
|
|
} from "./support";
|
|
|
|
interface ReturnedPracticeFixture {
|
|
schema_version: "vignette.returned-practice-browser-fixture.v1";
|
|
login: {
|
|
email: string;
|
|
role: "learner";
|
|
display_name: string;
|
|
cohort_ids: string[];
|
|
};
|
|
source_session_id: string;
|
|
practice_session_id: string;
|
|
deliberate: {
|
|
prescription_id: string;
|
|
criterion_id: string;
|
|
novelty: "familiar" | "unseen_transfer";
|
|
mode:
|
|
| "replay"
|
|
| "branch"
|
|
| "constrained_response"
|
|
| "voice_retry"
|
|
| "difficulty_ladder";
|
|
};
|
|
transfer: {
|
|
prescription_id: string;
|
|
suite_id: string;
|
|
trial_id: string;
|
|
criterion_id: string;
|
|
novelty: "unseen_transfer";
|
|
mode:
|
|
| "counterevidence_forecast"
|
|
| "evidence_recall"
|
|
| "uncertainty_range"
|
|
| "collect_more_evidence";
|
|
};
|
|
}
|
|
|
|
interface SessionDetail {
|
|
session_id: string;
|
|
persona_code: string;
|
|
}
|
|
|
|
interface AuthMe {
|
|
user_id: string;
|
|
}
|
|
|
|
interface DeliberateSubmission {
|
|
idempotent_replay: boolean;
|
|
}
|
|
|
|
interface DeliberateReadModel {
|
|
episodes: Array<{
|
|
session_id?: string | null;
|
|
attempts?: unknown[];
|
|
}>;
|
|
}
|
|
|
|
interface TransferSubmission {
|
|
idempotent_replay: boolean;
|
|
assessment: {
|
|
execution_count: number;
|
|
independent_execution_count: number;
|
|
};
|
|
}
|
|
|
|
interface TransferReadModel {
|
|
actual_executions: Array<{
|
|
original_transfer_trial_record_id: string;
|
|
practice_session_id: string;
|
|
}>;
|
|
}
|
|
|
|
const LIVE_GATE = process.env.E2E_PERIODIC_LEARNER_REAL_CLOSED_LOOP === "1";
|
|
const FIXTURE_PATH = process.env.E2E_RETURNED_PRACTICE_FIXTURE ?? "";
|
|
const RESULT_PATH = process.env.E2E_PERIODIC_LEARNER_RESULT ?? "";
|
|
|
|
function loadFixture(): ReturnedPracticeFixture {
|
|
if (!FIXTURE_PATH) {
|
|
throw new Error("E2E_RETURNED_PRACTICE_FIXTURE is required");
|
|
}
|
|
return JSON.parse(
|
|
readFileSync(path.resolve(FIXTURE_PATH), "utf8"),
|
|
) as ReturnedPracticeFixture;
|
|
}
|
|
|
|
async function expectOk(response: APIResponse) {
|
|
if (response.ok()) return;
|
|
let detail = `HTTP ${response.status()}`;
|
|
try {
|
|
detail = await response.text();
|
|
} catch {
|
|
// Streaming responses may not expose a replayable body through CDP.
|
|
}
|
|
expect(response.ok(), detail).toBeTruthy();
|
|
}
|
|
|
|
async function signInExistingFixture(
|
|
page: Page,
|
|
fixture: ReturnedPracticeFixture,
|
|
): Promise<string> {
|
|
const login = await page.request.post("/api/auth/dev-login", {
|
|
data: fixture.login,
|
|
});
|
|
await expectOk(login);
|
|
const me = await page.request.get("/api/auth/me");
|
|
await expectOk(me);
|
|
return ((await me.json()) as AuthMe).user_id;
|
|
}
|
|
|
|
function launchSearch(
|
|
fixture: ReturnedPracticeFixture,
|
|
kind: "deliberate" | "transfer",
|
|
): string {
|
|
const source = kind === "deliberate" ? fixture.deliberate : fixture.transfer;
|
|
const search = new URLSearchParams({
|
|
launch: kind,
|
|
prescription: source.prescription_id,
|
|
source_session: fixture.source_session_id,
|
|
criterion: source.criterion_id,
|
|
novelty: source.novelty,
|
|
mode: source.mode,
|
|
});
|
|
if (kind === "transfer") {
|
|
search.set("suite", fixture.transfer.suite_id);
|
|
search.set("trial", fixture.transfer.trial_id);
|
|
}
|
|
return search.toString();
|
|
}
|
|
|
|
function runtimeAttemptCount(
|
|
readModel: DeliberateReadModel,
|
|
practiceSessionId: string,
|
|
): number {
|
|
return readModel.episodes
|
|
.filter((episode) => episode.session_id === practiceSessionId)
|
|
.reduce((count, episode) => count + (episode.attempts?.length ?? 0), 0);
|
|
}
|
|
|
|
function actualExecutionCount(
|
|
readModel: TransferReadModel,
|
|
fixture: ReturnedPracticeFixture,
|
|
practiceSessionId: string,
|
|
): number {
|
|
return readModel.actual_executions.filter(
|
|
(execution) =>
|
|
execution.original_transfer_trial_record_id === fixture.transfer.trial_id &&
|
|
execution.practice_session_id === practiceSessionId,
|
|
).length;
|
|
}
|
|
|
|
async function readPracticeCount(page: Page, practiceSessionId: string) {
|
|
const response = await page.request.get("/api/practice/learners/me");
|
|
await expectOk(response);
|
|
return runtimeAttemptCount(
|
|
(await response.json()) as DeliberateReadModel,
|
|
practiceSessionId,
|
|
);
|
|
}
|
|
|
|
async function readTransferCount(
|
|
page: Page,
|
|
fixture: ReturnedPracticeFixture,
|
|
practiceSessionId: string,
|
|
) {
|
|
const response = await page.request.get("/api/calibration/learners/me");
|
|
await expectOk(response);
|
|
return actualExecutionCount(
|
|
(await response.json()) as TransferReadModel,
|
|
fixture,
|
|
practiceSessionId,
|
|
);
|
|
}
|
|
|
|
async function openEvidenceCard(
|
|
page: Page,
|
|
fixture: ReturnedPracticeFixture,
|
|
practiceSessionId: string,
|
|
kind: "deliberate" | "transfer",
|
|
) {
|
|
await page.goto(
|
|
`/learn/session/${practiceSessionId}/review?${launchSearch(fixture, kind)}`,
|
|
);
|
|
const insightsTab = page.locator("#sr-tab-insights");
|
|
await expect(insightsTab).toBeVisible({ timeout: 30_000 });
|
|
if ((await insightsTab.getAttribute("aria-selected")) !== "true") {
|
|
await insightsTab.click();
|
|
}
|
|
const selector =
|
|
kind === "deliberate" ? ".dp-runtime-observation" : ".ct-actual-transfer";
|
|
const card = page.locator("#sr-panel-insights").locator(selector);
|
|
await expect(card).toBeVisible({ timeout: 30_000 });
|
|
return card;
|
|
}
|
|
|
|
function writeResult(result: Record<string, unknown>) {
|
|
if (!RESULT_PATH) {
|
|
throw new Error("E2E_PERIODIC_LEARNER_RESULT is required");
|
|
}
|
|
const resolved = path.resolve(RESULT_PATH);
|
|
mkdirSync(path.dirname(resolved), { recursive: true });
|
|
writeFileSync(resolved, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
test.describe("periodic same-learner real closed loop", () => {
|
|
test.skip(!LIVE_GATE, "Explicit disposable periodic gate only");
|
|
|
|
test("@single-run home recommendation, SSE session, review, G4/G5 POST and reload stay one learner", async ({
|
|
page,
|
|
}) => {
|
|
test.setTimeout(20 * 60_000);
|
|
const fixture = loadFixture();
|
|
expect(fixture.schema_version).toBe(
|
|
"vignette.returned-practice-browser-fixture.v1",
|
|
);
|
|
await useRealApi(page);
|
|
const learnerId = await signInExistingFixture(page, fixture);
|
|
|
|
const preparedSession = await page.request.get(
|
|
`/api/sessions/${fixture.practice_session_id}`,
|
|
);
|
|
await expectOk(preparedSession);
|
|
const practicePersona = ((await preparedSession.json()) as SessionDetail)
|
|
.persona_code;
|
|
expect(practicePersona).toBeTruthy();
|
|
|
|
await page.goto("/learn");
|
|
await expect(
|
|
page.getByRole("heading", { name: "오늘 이어갈 회기를 먼저 봅니다." }),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
await expect(page.getByText("다음 연습 추천", { exact: true })).toBeVisible();
|
|
await expect(page.getByText("리뷰 확인을 우선합니다.", { exact: true })).toBeVisible();
|
|
await page.getByRole("button", { name: "리뷰 확인하기" }).click();
|
|
await expect(page).toHaveURL(/\/learn\/history$/);
|
|
|
|
await page.goto(`/learn/practice?${launchSearch(fixture, "deliberate")}`);
|
|
await expect(
|
|
page.getByRole("heading", { name: /처방을 이어받았습니다/ }),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
const escapedPersona = practicePersona.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
const persona = page.getByRole("option", {
|
|
name: new RegExp(escapedPersona),
|
|
});
|
|
await persona.click();
|
|
await expect(persona).toHaveAttribute("aria-selected", "true");
|
|
await page.getByRole("button", { name: "새 회기 시작" }).click();
|
|
await expect(page).toHaveURL(new RegExp(`/learn/session/${escapedPersona}\\?`));
|
|
await page.getByRole("button", { name: "회기 시작" }).click();
|
|
await completeAlliancePreCheckpoint(page);
|
|
|
|
const activeUrl = new URL(page.url());
|
|
const practiceSessionId = activeUrl.pathname.split("/").filter(Boolean).at(-1);
|
|
expect(practiceSessionId).toBeTruthy();
|
|
expect(practiceSessionId).not.toBe(practicePersona);
|
|
|
|
const learnerText =
|
|
"그 말을 꺼내기까지 많이 외롭고 조심스러웠던 것 같아요. 제가 이해한 마음이 맞을까요?";
|
|
const streamResponse = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" &&
|
|
response.url().endsWith(`/api/sessions/${practiceSessionId}/stream`),
|
|
{ timeout: 6 * 60_000 },
|
|
);
|
|
await page.getByLabel("학습자 발화 입력").fill(learnerText);
|
|
await page.getByRole("button", { name: "보내기" }).click();
|
|
const stream = await streamResponse;
|
|
await expectOk(stream);
|
|
expect(stream.headers()["content-type"] ?? "").toContain("text/event-stream");
|
|
await expect(page.locator(".sx-utt").filter({ hasText: learnerText })).toBeVisible();
|
|
await expect(page.locator(".sx-utt.is-client").last()).toBeVisible({
|
|
timeout: 6 * 60_000,
|
|
});
|
|
|
|
await page.getByRole("button", { name: "회기 종료" }).click();
|
|
await page.getByRole("button", { name: "종료하고 리뷰 보기" }).click();
|
|
await expect(page).toHaveURL(
|
|
new RegExp(`/learn/session/${practiceSessionId}/review\\?`),
|
|
);
|
|
await expect
|
|
.poll(
|
|
async () => {
|
|
const review = await page.request.get(
|
|
`/api/sessions/${practiceSessionId}/review`,
|
|
);
|
|
if (!review.ok()) return false;
|
|
return Boolean(((await review.json()) as { reviewReady?: boolean }).reviewReady);
|
|
},
|
|
{ timeout: 6 * 60_000, intervals: [1_000, 2_000, 5_000] },
|
|
)
|
|
.toBe(true);
|
|
|
|
expect(await readPracticeCount(page, practiceSessionId!)).toBe(0);
|
|
let deliberateCard = await openEvidenceCard(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
"deliberate",
|
|
);
|
|
const g4PostUrl = `/api/practice/${encodeURIComponent(
|
|
fixture.deliberate.prescription_id,
|
|
)}/attempts/from-session/${practiceSessionId}`;
|
|
const g4FirstPost = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" && response.url().endsWith(g4PostUrl),
|
|
{ timeout: 5 * 60_000 },
|
|
);
|
|
await deliberateCard
|
|
.getByRole("button", { name: "이번 회기를 독립 관찰로 반영" })
|
|
.click();
|
|
const g4First = await g4FirstPost;
|
|
await expectOk(g4First);
|
|
expect(((await g4First.json()) as DeliberateSubmission).idempotent_replay).toBe(
|
|
false,
|
|
);
|
|
const g4FirstCount = await readPracticeCount(page, practiceSessionId!);
|
|
expect(g4FirstCount).toBe(1);
|
|
await page.reload();
|
|
deliberateCard = await openEvidenceCard(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
"deliberate",
|
|
);
|
|
const g4ReplayPost = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" && response.url().endsWith(g4PostUrl),
|
|
{ timeout: 5 * 60_000 },
|
|
);
|
|
await deliberateCard
|
|
.getByRole("button", { name: "반영 상태 다시 확인" })
|
|
.click();
|
|
const g4Replay = await g4ReplayPost;
|
|
await expectOk(g4Replay);
|
|
expect(((await g4Replay.json()) as DeliberateSubmission).idempotent_replay).toBe(
|
|
true,
|
|
);
|
|
const g4ReplayCount = await readPracticeCount(page, practiceSessionId!);
|
|
expect(g4ReplayCount).toBe(1);
|
|
|
|
expect(await readTransferCount(page, fixture, practiceSessionId!)).toBe(0);
|
|
let transferCard = await openEvidenceCard(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
"transfer",
|
|
);
|
|
const g5PostUrl = "/api/calibration/transfer-executions";
|
|
const g5FirstPost = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" && response.url().endsWith(g5PostUrl),
|
|
{ timeout: 5 * 60_000 },
|
|
);
|
|
await transferCard
|
|
.getByRole("button", { name: "이 회기를 전이 근거로 확인" })
|
|
.click();
|
|
const g5First = await g5FirstPost;
|
|
await expectOk(g5First);
|
|
const g5FirstBody = (await g5First.json()) as TransferSubmission;
|
|
expect(g5FirstBody.idempotent_replay).toBe(false);
|
|
expect(g5FirstBody.assessment.execution_count).toBe(1);
|
|
expect(g5FirstBody.assessment.independent_execution_count).toBe(1);
|
|
const g5FirstCount = await readTransferCount(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
);
|
|
expect(g5FirstCount).toBe(1);
|
|
await page.reload();
|
|
transferCard = await openEvidenceCard(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
"transfer",
|
|
);
|
|
const g5ReplayPost = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" && response.url().endsWith(g5PostUrl),
|
|
{ timeout: 5 * 60_000 },
|
|
);
|
|
await transferCard
|
|
.getByRole("button", { name: "같은 회기 기록 다시 확인" })
|
|
.click();
|
|
const g5Replay = await g5ReplayPost;
|
|
await expectOk(g5Replay);
|
|
expect(((await g5Replay.json()) as TransferSubmission).idempotent_replay).toBe(
|
|
true,
|
|
);
|
|
const g5ReplayCount = await readTransferCount(
|
|
page,
|
|
fixture,
|
|
practiceSessionId!,
|
|
);
|
|
expect(g5ReplayCount).toBe(1);
|
|
|
|
const meAfter = await page.request.get("/api/auth/me");
|
|
await expectOk(meAfter);
|
|
expect(((await meAfter.json()) as AuthMe).user_id).toBe(learnerId);
|
|
|
|
writeResult({
|
|
schema_version: "vignette.periodic-learner-real-closed-loop.v1",
|
|
learner_id: learnerId,
|
|
source_session_id: fixture.source_session_id,
|
|
practice_session_id: practiceSessionId,
|
|
same_learner: true,
|
|
recommendation_verified: true,
|
|
session_created_in_browser: true,
|
|
sse_turn_verified: true,
|
|
review_ready: true,
|
|
g4: {
|
|
initial_count: 0,
|
|
first_count: g4FirstCount,
|
|
replay_count: g4ReplayCount,
|
|
replay_idempotent: true,
|
|
},
|
|
g5: {
|
|
initial_count: 0,
|
|
first_count: g5FirstCount,
|
|
replay_count: g5ReplayCount,
|
|
replay_idempotent: true,
|
|
},
|
|
});
|
|
});
|
|
});
|