import { readFile } from "node:fs/promises"; import path from "node:path"; import { expect, test, type APIResponse, type Page } from "@playwright/test"; import { expectNoHorizontalOverflow, useRealApi } from "./support"; const DATA_CLASSIFICATION = "synthetic_replay_red_team_coverage_drift"; const INTERNAL_HEADER = "X-Vignette-Continuous-Improvement-Token"; const SOURCE_PATH = path.resolve( process.cwd(), "../api/app/data/continuous_improvement/synthetic_source_pack.v2.json", ); type Role = "admin" | "teacher" | "learner"; interface RepoSourceSpec { data_classification: typeof DATA_CLASSIFICATION; content_kind: "case" | "rupture" | "practice" | "benchmark"; difficulty_level: number; variant_count: number; prompt_version: string; source_packs: Array<{ artifact: { source_id: string; version: string; content_sha256: string; provenance_uri: string; usage_status: "approved"; citation_label: string; }; content: string; }>; } interface AgenticPipelineResponse { submission_id: string; pipeline_id: string; qualification_id: string; candidate_catalog_entry_id: string; state: "pending_human_approval"; human_approval_required: true; catalog_promoted: false; idempotent_replay: boolean; clinical_claim_allowed: false; agent_calls_executed: number; } interface ContinuousImprovementView { content_qualifications: Array<{ qualification_id: string; catalog_entry_id: string; source_provenance_uris: string[]; draft_payload: Record | null; }>; approvals: Array<{ approval_event_id: string; target_kind: string; target_id: string; decision: string; }>; catalog_entries: Array<{ catalog_record_id: string; qualification_id: string; catalog_entry_id: string; status: "approved"; clinical_claim_allowed: false; }>; data_classification: typeof DATA_CLASSIFICATION; silent_auto_promotion_allowed: false; raw_transcript_included: false; pii_included: false; clinical_claim_allowed: false; } interface ApprovedCatalogResponse { entries: Array<{ catalog_record_id: string; qualification_id: string; catalog_entry_id: string; payload: Record; status: "approved"; clinical_claim_allowed: false; }>; data_classification: typeof DATA_CLASSIFICATION; human_approval_required: true; raw_transcript_included: false; pii_included: false; clinical_claim_allowed: false; } async function expectOk(response: APIResponse) { expect(response.ok(), await response.text()).toBeTruthy(); } function suffixFor(testInfo: { workerIndex: number; retry: number }) { return `live-${Date.now().toString(36)}-${testInfo.workerIndex}-${testInfo.retry}`; } async function signIn(page: Page, role: Role, suffix: string) { const domain = role === "admin" ? "twentyoz.kr" : "hs.ac.kr"; const login = await page.request.post("/api/auth/dev-login", { data: { email: `g8-${role}-${suffix}@${domain}`, role, display_name: `G8 ${role}`, cohort_ids: ["e2e-hanshin"], }, }); await expectOk(login); const onboarding = await page.request.post("/api/users/me/onboarding", { data: { legal_name: `G8 ${role}`, affiliation: "한신대학교", department: role === "admin" ? "운영" : "상담심리학과", grade_level: role === "admin" ? "관리자" : role === "teacher" ? "교수" : "3학년", phone: "010-0000-0000", contact_address: "경기도 오산시 한신대학교", nickname: `G8 ${role}`, self_introduction: "합성 지속 개선 게이트 검증 계정입니다.", avatar_url: "", terms_accepted: true, privacy_accepted: true, }, }); await expectOk(onboarding); } function matching( rows: T[], qualificationId: string, ): T[] { return rows.filter((row) => row.qualification_id === qualificationId); } function assertNoSensitivePayload(value: unknown, location = "response") { if (Array.isArray(value)) { value.forEach((item, index) => assertNoSensitivePayload(item, `${location}[${index}]`)); return; } if (!value || typeof value !== "object") return; const record = value as Record; const forbidden = [ "hidden_answer", "raw_transcript", "transcript", "utterance_text", "clinical_diagnosis", "treatment_plan", ]; for (const key of forbidden) { expect(record, `${location} exposed ${key}`).not.toHaveProperty(key); } for (const key of ["raw_transcript_included", "pii_included", "clinical_claim_allowed"]) { if (key in record) expect(record[key], `${location}.${key}`).toBe(false); } for (const [key, child] of Object.entries(record)) { assertNoSensitivePayload(child, `${location}.${key}`); } } test.describe("continuous improvement human catalog gate (real API/DB)", () => { test.beforeEach(async ({ page }) => { await useRealApi(page); }); test("reviews a repo-approved agentic payload in the admin browser and exposes exactly one approved catalog entry @single-run", async ({ page, }, testInfo) => { test.setTimeout(8 * 60_000); test.skip( process.env.E2E_G8_AGENTIC_APPROVAL !== "1", "Set E2E_G8_AGENTIC_APPROVAL=1 for the explicit real-engine approval gate.", ); const internalToken = process.env.E2E_CONTINUOUS_IMPROVEMENT_INTERNAL_TOKEN ?? ""; expect(internalToken.length, "G8 internal token must be injected without logging it").toBeGreaterThanOrEqual( 32, ); const healthResponse = await page.request.get("/api/health"); await expectOk(healthResponse); const health = (await healthResponse.json()) as { db: boolean; engine: boolean }; expect(health).toMatchObject({ db: true, engine: true }); const sourceSpec = JSON.parse(await readFile(SOURCE_PATH, "utf8")) as RepoSourceSpec; expect(sourceSpec.data_classification).toBe(DATA_CLASSIFICATION); expect(sourceSpec.source_packs).toHaveLength(1); expect(sourceSpec.source_packs[0].artifact).toMatchObject({ usage_status: "approved", provenance_uri: "repo://apps/api/app/data/continuous_improvement/synthetic_source_pack.v2.json", }); const suffix = suffixFor(testInfo); const requestBody = { submission_id: crypto.randomUUID(), pipeline_id: crypto.randomUUID(), benchmark_record_id: crypto.randomUUID(), qualification_id: crypto.randomUUID(), data_classification: sourceSpec.data_classification, source_packs: sourceSpec.source_packs, content_kind: sourceSpec.content_kind, difficulty_level: sourceSpec.difficulty_level, variant_count: sourceSpec.variant_count, prompt_version: sourceSpec.prompt_version, }; const agenticPath = "/api/internal/continuous-improvement/agentic-content-pipelines"; const internalHeaders = { [INTERNAL_HEADER]: internalToken }; const create = await page.request.post(agenticPath, { data: requestBody, headers: internalHeaders, timeout: 5 * 60_000, }); await expectOk(create); expect(create.status()).toBe(201); const candidate = (await create.json()) as AgenticPipelineResponse; expect(candidate).toMatchObject({ submission_id: requestBody.submission_id, qualification_id: requestBody.qualification_id, state: "pending_human_approval", human_approval_required: true, catalog_promoted: false, idempotent_replay: false, clinical_claim_allowed: false, }); expect(candidate.agent_calls_executed).toBeGreaterThanOrEqual(7); const agenticReplay = await page.request.post(agenticPath, { data: requestBody, headers: internalHeaders, timeout: 60_000, }); await expectOk(agenticReplay); const replayedCandidate = (await agenticReplay.json()) as AgenticPipelineResponse; expect(replayedCandidate).toMatchObject({ qualification_id: candidate.qualification_id, candidate_catalog_entry_id: candidate.candidate_catalog_entry_id, idempotent_replay: true, agent_calls_executed: 0, }); const changedAgentic = await page.request.post(agenticPath, { data: { ...requestBody, difficulty_level: requestBody.difficulty_level === 5 ? 4 : 5 }, headers: internalHeaders, timeout: 60_000, }); expect(changedAgentic.status(), await changedAgentic.text()).toBe(409); const approvalBody = { submission_id: crypto.randomUUID(), approval_event_id: crypto.randomUUID(), effect_record_id: crypto.randomUUID(), target_kind: "content_qualification", target_id: candidate.qualification_id, decision: "approve_content", reason_code: "repo-approved synthetic payload의 경계와 수련 목표를 브라우저에서 검수함", evidence_refs: [ sourceSpec.source_packs[0].artifact.provenance_uri, `audit://continuous-improvement/human-review/${candidate.qualification_id}`, ], }; for (const role of ["learner", "teacher"] as const) { await signIn(page, role, `${suffix}-${role}`); const blockedRead = await page.request.get("/api/continuous-improvement"); expect(blockedRead.status(), await blockedRead.text()).toBe(403); const blockedCatalog = await page.request.get("/api/continuous-improvement/catalog"); expect(blockedCatalog.status(), await blockedCatalog.text()).toBe(403); const blockedApproval = await page.request.post("/api/continuous-improvement/approvals", { data: approvalBody, }); expect(blockedApproval.status(), await blockedApproval.text()).toBe(403); } await signIn(page, "admin", `${suffix}-admin`); const beforeViewResponse = await page.request.get("/api/continuous-improvement"); await expectOk(beforeViewResponse); const beforeView = (await beforeViewResponse.json()) as ContinuousImprovementView; assertNoSensitivePayload(beforeView); expect(matching(beforeView.catalog_entries, candidate.qualification_id)).toHaveLength(0); const qualification = beforeView.content_qualifications.find( (item) => item.qualification_id === candidate.qualification_id, ); expect(qualification).toBeTruthy(); expect(qualification?.draft_payload).not.toBeNull(); expect(qualification?.source_provenance_uris).toContain( sourceSpec.source_packs[0].artifact.provenance_uri, ); const beforeCatalogResponse = await page.request.get("/api/continuous-improvement/catalog"); await expectOk(beforeCatalogResponse); const beforeCatalog = (await beforeCatalogResponse.json()) as ApprovedCatalogResponse; assertNoSensitivePayload(beforeCatalog); expect(matching(beforeCatalog.entries, candidate.qualification_id)).toHaveLength(0); await page.goto("/admin/continuous-improvement"); await expect(page.getByRole("heading", { name: "승격보다 근거를 먼저 본다" })).toBeVisible(); const card = page.locator(`[data-qualification-id="${candidate.qualification_id}"]`); await expect(card).toBeVisible(); await expect(card.getByText(candidate.candidate_catalog_entry_id)).toBeVisible(); await card.getByText("검수 payload 펼쳐 보기").click(); await expect(card.getByRole("heading", { name: "상황과 도전" })).toBeVisible(); await expect(card).not.toContainText("hidden_answer"); await expect(card).not.toContainText("raw_transcript"); const reason = card.getByLabel("콘텐츠 승인 사유"); const approveButton = card.getByRole("button", { name: "카탈로그 승인" }); await expect(approveButton).toBeDisabled(); await reason.fill(approvalBody.reason_code); await expect(approveButton).toBeEnabled(); const approvalResponsePromise = page.waitForResponse((response) => { if (response.request().method() !== "POST") return false; const url = new URL(response.url()); if (!url.pathname.endsWith("/continuous-improvement/approvals")) return false; const body = response.request().postDataJSON() as { target_id?: string }; return body.target_id === candidate.qualification_id; }); await approveButton.focus(); await expect(approveButton).toBeFocused(); await approveButton.press("Enter"); const approvalResponse = await approvalResponsePromise; await expectOk(approvalResponse); const browserApprovalBody = approvalResponse.request().postDataJSON() as typeof approvalBody; await expect(card.getByText("카탈로그 승인 원장 기록됨")).toBeVisible({ timeout: 15_000 }); await expectNoHorizontalOverflow(page); const afterCatalogResponse = await page.request.get("/api/continuous-improvement/catalog"); await expectOk(afterCatalogResponse); const afterCatalog = (await afterCatalogResponse.json()) as ApprovedCatalogResponse; assertNoSensitivePayload(afterCatalog); const consumed = matching(afterCatalog.entries, candidate.qualification_id); expect(consumed).toHaveLength(1); expect(consumed[0]).toMatchObject({ catalog_entry_id: candidate.candidate_catalog_entry_id, catalog_record_id: browserApprovalBody.effect_record_id, status: "approved", clinical_claim_allowed: false, }); const approvalReplay = await page.request.post("/api/continuous-improvement/approvals", { data: browserApprovalBody, }); await expectOk(approvalReplay); expect(approvalReplay.status()).toBe(201); expect(await approvalReplay.json()).toMatchObject({ idempotent_replay: true }); const changedApproval = await page.request.post("/api/continuous-improvement/approvals", { data: { ...browserApprovalBody, reason_code: `${browserApprovalBody.reason_code} 변경` }, }); expect(changedApproval.status(), await changedApproval.text()).toBe(409); const finalViewResponse = await page.request.get("/api/continuous-improvement"); await expectOk(finalViewResponse); const finalView = (await finalViewResponse.json()) as ContinuousImprovementView; assertNoSensitivePayload(finalView); expect( finalView.approvals.filter( (item) => item.target_kind === "content_qualification" && item.target_id === candidate.qualification_id, ), ).toHaveLength(1); expect(matching(finalView.catalog_entries, candidate.qualification_id)).toHaveLength(1); await testInfo.attach("g8-approved-catalog", { body: await page.screenshot({ fullPage: true }), contentType: "image/png", }); console.log( `G8_LIVE_EVIDENCE ${JSON.stringify({ qualification_id: candidate.qualification_id, catalog_entry_id: candidate.candidate_catalog_entry_id, catalog_record_id: browserApprovalBody.effect_record_id, approval_event_id: browserApprovalBody.approval_event_id, agent_calls_executed: candidate.agent_calls_executed, catalog_before: 0, catalog_after: consumed.length, final_approval_events: finalView.approvals.filter( (item) => item.target_kind === "content_qualification" && item.target_id === candidate.qualification_id, ).length, })}`, ); }); test("approves an existing Claude-qualified pending payload without another model call @single-run", async ({ page, }, testInfo) => { test.setTimeout(2 * 60_000); const qualificationId = process.env.E2E_G8_EXISTING_QUALIFICATION_ID ?? ""; test.skip(!qualificationId, "Set E2E_G8_EXISTING_QUALIFICATION_ID to review an existing candidate."); const healthResponse = await page.request.get("/api/health"); await expectOk(healthResponse); expect((await healthResponse.json()) as { db: boolean; engine: boolean }).toMatchObject({ db: true, engine: true, }); const suffix = suffixFor(testInfo); for (const role of ["learner", "teacher"] as const) { await signIn(page, role, `${suffix}-existing-${role}`); const blockedRead = await page.request.get("/api/continuous-improvement"); expect(blockedRead.status(), await blockedRead.text()).toBe(403); const blockedCatalog = await page.request.get("/api/continuous-improvement/catalog"); expect(blockedCatalog.status(), await blockedCatalog.text()).toBe(403); } await signIn(page, "admin", `${suffix}-existing-admin`); const beforeViewResponse = await page.request.get("/api/continuous-improvement"); await expectOk(beforeViewResponse); const beforeView = (await beforeViewResponse.json()) as ContinuousImprovementView; assertNoSensitivePayload(beforeView); const pending = beforeView.content_qualifications.find( (item) => item.qualification_id === qualificationId, ); expect(pending, "the requested existing qualification must still be pending").toBeTruthy(); expect(pending?.draft_payload).not.toBeNull(); expect(pending?.source_provenance_uris.length).toBeGreaterThan(0); expect(pending?.source_provenance_uris.every((uri) => uri.startsWith("repo://"))).toBeTruthy(); expect(matching(beforeView.catalog_entries, qualificationId)).toHaveLength(0); const beforeCatalogResponse = await page.request.get("/api/continuous-improvement/catalog"); await expectOk(beforeCatalogResponse); const beforeCatalog = (await beforeCatalogResponse.json()) as ApprovedCatalogResponse; assertNoSensitivePayload(beforeCatalog); expect(matching(beforeCatalog.entries, qualificationId)).toHaveLength(0); await page.goto("/admin/continuous-improvement"); const card = page.locator(`[data-qualification-id="${qualificationId}"]`); await expect(card).toBeVisible(); await card.getByText("검수 payload 펼쳐 보기").click(); await expect(card.getByRole("heading", { name: "상황과 도전" })).toBeVisible(); await expect(card).not.toContainText("hidden_answer"); await expect(card).not.toContainText("raw_transcript"); const reasonText = "기존 실제 Claude 합성 후보의 visible payload와 repo 근거를 관리자 브라우저에서 검수함"; const reason = card.getByLabel("콘텐츠 승인 사유"); const approveButton = card.getByRole("button", { name: "카탈로그 승인" }); await expect(approveButton).toBeDisabled(); await reason.fill(reasonText); await expect(approveButton).toBeEnabled(); const approvalResponsePromise = page.waitForResponse((response) => { if (response.request().method() !== "POST") return false; const url = new URL(response.url()); if (!url.pathname.endsWith("/continuous-improvement/approvals")) return false; const body = response.request().postDataJSON() as { target_id?: string }; return body.target_id === qualificationId; }); await approveButton.focus(); await expect(approveButton).toBeFocused(); await approveButton.press("Enter"); const approvalResponse = await approvalResponsePromise; await expectOk(approvalResponse); const approvalBody = approvalResponse.request().postDataJSON() as { submission_id: string; approval_event_id: string; effect_record_id: string; target_kind: "content_qualification"; target_id: string; decision: "approve_content"; reason_code: string; evidence_refs: string[]; }; expect(approvalBody.reason_code).toBe(reasonText); await expect(card.getByText("카탈로그 승인 원장 기록됨")).toBeVisible({ timeout: 15_000 }); await expectNoHorizontalOverflow(page); const afterCatalogResponse = await page.request.get("/api/continuous-improvement/catalog"); await expectOk(afterCatalogResponse); const afterCatalog = (await afterCatalogResponse.json()) as ApprovedCatalogResponse; assertNoSensitivePayload(afterCatalog); const consumed = matching(afterCatalog.entries, qualificationId); expect(consumed).toHaveLength(1); expect(consumed[0]).toMatchObject({ catalog_entry_id: pending?.catalog_entry_id, catalog_record_id: approvalBody.effect_record_id, status: "approved", clinical_claim_allowed: false, }); const approvalReplay = await page.request.post("/api/continuous-improvement/approvals", { data: approvalBody, }); await expectOk(approvalReplay); expect(await approvalReplay.json()).toMatchObject({ idempotent_replay: true }); const changedApproval = await page.request.post("/api/continuous-improvement/approvals", { data: { ...approvalBody, reason_code: `${reasonText} 변경` }, }); expect(changedApproval.status(), await changedApproval.text()).toBe(409); const finalViewResponse = await page.request.get("/api/continuous-improvement"); await expectOk(finalViewResponse); const finalView = (await finalViewResponse.json()) as ContinuousImprovementView; assertNoSensitivePayload(finalView); const approvalCount = finalView.approvals.filter( (item) => item.target_kind === "content_qualification" && item.target_id === qualificationId, ).length; expect(approvalCount).toBe(1); expect(matching(finalView.catalog_entries, qualificationId)).toHaveLength(1); await testInfo.attach("g8-existing-approved-catalog", { body: await page.screenshot({ fullPage: true }), contentType: "image/png", }); console.log( `G8_EXISTING_LIVE_EVIDENCE ${JSON.stringify({ qualification_id: qualificationId, catalog_entry_id: pending?.catalog_entry_id, catalog_record_id: approvalBody.effect_record_id, approval_event_id: approvalBody.approval_event_id, catalog_before: 0, catalog_after: consumed.length, final_approval_events: approvalCount, new_model_calls: 0, })}`, ); }); });