460 lines
20 KiB
TypeScript
460 lines
20 KiB
TypeScript
import { expect, test, type Page } from "@playwright/test";
|
|
import { expectNoHorizontalOverflow } from "./support";
|
|
|
|
const MODEL_GATE_ID = "10000000-0000-4000-8000-000000000001";
|
|
const RELEASE_GATE_ID = "10000000-0000-4000-8000-000000000002";
|
|
const INCIDENT_ID = "10000000-0000-4000-8000-000000000003";
|
|
const CREATED_AT = "2026-08-06T08:30:00Z";
|
|
|
|
type ApprovalDecision =
|
|
| "approve_content"
|
|
| "approve_promotion"
|
|
| "authorize_rollback"
|
|
| "keep_quarantine"
|
|
| "reject";
|
|
|
|
type ApprovalBody = {
|
|
submission_id: string;
|
|
approval_event_id: string;
|
|
effect_record_id: string;
|
|
target_kind: "content_qualification" | "model_change_gate" | "release_gate";
|
|
target_id: string;
|
|
decision: ApprovalDecision;
|
|
reason_code: string;
|
|
evidence_refs: string[];
|
|
};
|
|
|
|
function artifact(
|
|
recordId: string,
|
|
ownerKind: "model_change_gate" | "release_gate",
|
|
ownerId: string,
|
|
kind: "baseline" | "threshold" | "provenance" | "rollback",
|
|
) {
|
|
return {
|
|
artifact_record_id: recordId,
|
|
owner_kind: ownerKind,
|
|
owner_id: ownerId,
|
|
artifact_kind: kind,
|
|
artifact_id: `oas-g8-${kind}-${ownerId.slice(-4)}`,
|
|
content_sha256: kind.slice(0, 1).repeat(64),
|
|
provenance_uri: `audit://synthetic/g8/${ownerId}/${kind}`,
|
|
created_at: CREATED_AT,
|
|
};
|
|
}
|
|
|
|
function baseView() {
|
|
return {
|
|
content_qualifications: [
|
|
{
|
|
qualification_id: "20000000-0000-4000-8000-000000000001",
|
|
pipeline_id: "20000000-0000-4000-8000-000000000002",
|
|
catalog_entry_id: "oas-g8-catalog-synthetic-rupture",
|
|
payload_sha256: "a".repeat(64),
|
|
content_kind: "rupture" as const,
|
|
difficulty_level: 3,
|
|
synthetic_identity_id: "synthetic-identity-synthetic-rupture",
|
|
source_count: 1,
|
|
red_team_review_count: 2,
|
|
benchmark_variant_count: 3,
|
|
benchmark_pass_rate: 1,
|
|
source_provenance_uris: [
|
|
"repo://apps/api/app/data/continuous_improvement/synthetic_source_pack.v2.json",
|
|
],
|
|
draft_payload: {
|
|
title: "합성 관계 균열 수선 연습",
|
|
synthetic_profile: "실존 인물과 무관한 합성 내담자",
|
|
scenario: "상담자가 주제를 너무 빨리 바꿔 합성 내담자가 서두른다고 느낀 상황",
|
|
rupture_or_challenge: "상호작용을 명명하고 내담자의 정정을 초대한다.",
|
|
learner_task: "영향을 방어하지 않고 인정한 뒤 다음 초점을 공동 결정한다.",
|
|
success_criteria: ["상호작용 명명", "정정 초대"],
|
|
source_refs: ["oas-g8-source-repo-synthetic-rupture-v2"],
|
|
grounded_claims: [
|
|
{
|
|
claim: "합성 수련 시나리오",
|
|
source_ref: "oas-g8-source-repo-synthetic-rupture-v2",
|
|
},
|
|
],
|
|
},
|
|
gate_state: "pending_human_approval",
|
|
created_at: CREATED_AT,
|
|
},
|
|
],
|
|
model_change_gates: [
|
|
{
|
|
gate_id: MODEL_GATE_ID,
|
|
gate_decision: "promote",
|
|
reasons: ["synthetic benchmark threshold passed", "coverage drift stable"],
|
|
state: "pending_human_approval",
|
|
created_at: CREATED_AT,
|
|
},
|
|
],
|
|
release_gates: [
|
|
{
|
|
gate_id: RELEASE_GATE_ID,
|
|
release_id: "oas-g8-release-2026-08-06",
|
|
qualified: true,
|
|
state: "pending_human_approval",
|
|
created_at: CREATED_AT,
|
|
},
|
|
],
|
|
gate_artifacts: [
|
|
artifact("30000000-0000-4000-8000-000000000001", "model_change_gate", MODEL_GATE_ID, "baseline"),
|
|
artifact("30000000-0000-4000-8000-000000000002", "model_change_gate", MODEL_GATE_ID, "threshold"),
|
|
artifact("30000000-0000-4000-8000-000000000003", "model_change_gate", MODEL_GATE_ID, "provenance"),
|
|
artifact("30000000-0000-4000-8000-000000000004", "model_change_gate", MODEL_GATE_ID, "rollback"),
|
|
artifact("30000000-0000-4000-8000-000000000005", "release_gate", RELEASE_GATE_ID, "baseline"),
|
|
artifact("30000000-0000-4000-8000-000000000006", "release_gate", RELEASE_GATE_ID, "threshold"),
|
|
artifact("30000000-0000-4000-8000-000000000007", "release_gate", RELEASE_GATE_ID, "provenance"),
|
|
],
|
|
approvals: [],
|
|
catalog_entries: [],
|
|
lifecycle_events: [
|
|
{
|
|
lifecycle_event_id: "40000000-0000-4000-8000-000000000003",
|
|
target_kind: "model_change_gate",
|
|
target_id: "40000000-0000-4000-8000-000000000004",
|
|
event_type: "rollback",
|
|
event_status: "requested",
|
|
evidence_refs: ["audit://synthetic/g8/rollback-requested"],
|
|
created_at: "2026-08-06T08:20:00Z",
|
|
},
|
|
{
|
|
lifecycle_event_id: "40000000-0000-4000-8000-000000000005",
|
|
target_kind: "model_change_gate",
|
|
target_id: "40000000-0000-4000-8000-000000000006",
|
|
event_type: "rollback",
|
|
event_status: "failed",
|
|
evidence_refs: ["audit://synthetic/g8/rollback-failed"],
|
|
created_at: "2026-08-06T08:10:00Z",
|
|
},
|
|
{
|
|
lifecycle_event_id: "40000000-0000-4000-8000-000000000001",
|
|
target_kind: "model_change_gate",
|
|
target_id: "40000000-0000-4000-8000-000000000002",
|
|
event_type: "rollback",
|
|
event_status: "executed",
|
|
evidence_refs: ["audit://synthetic/g8/rollback"],
|
|
created_at: "2026-08-06T08:00:00Z",
|
|
},
|
|
],
|
|
incidents: [
|
|
{
|
|
incident_record_id: INCIDENT_ID,
|
|
incident_id: "oas-g8-incident-provider-timeout",
|
|
error_fingerprint: "e".repeat(64),
|
|
affected_contract: "synthetic.replay.provider-timeout",
|
|
evidence_refs: ["audit://synthetic/g8/incident"],
|
|
pii_included: false,
|
|
created_at: CREATED_AT,
|
|
},
|
|
],
|
|
regression_dag_nodes: [
|
|
["01", "reproduction_test", "passed"],
|
|
["02", "implementation", "passed"],
|
|
["03", "e2e", "pending"],
|
|
["04", "runtime_proof", "pending"],
|
|
].map(([suffix, nodeType, nodeStatus], index, all) => ({
|
|
node_record_id: `50000000-0000-4000-8000-0000000000${suffix}`,
|
|
incident_record_id: INCIDENT_ID,
|
|
node_id: `oas-g8-node-provider-timeout-${nodeType}`,
|
|
node_type: nodeType,
|
|
depends_on_record_ids:
|
|
index === 0 ? [] : [`50000000-0000-4000-8000-0000000000${all[index - 1][0]}`],
|
|
evidence_ref: nodeStatus === "passed" ? `audit://synthetic/g8/${nodeType}` : null,
|
|
node_status: nodeStatus,
|
|
created_at: CREATED_AT,
|
|
})),
|
|
data_classification: "synthetic_replay_red_team_coverage_drift",
|
|
silent_auto_promotion_allowed: false,
|
|
raw_transcript_included: false,
|
|
pii_included: false,
|
|
clinical_claim_allowed: false,
|
|
};
|
|
}
|
|
|
|
type MockOptions = {
|
|
role?: "admin" | "teacher" | "learner";
|
|
status?: number;
|
|
empty?: boolean;
|
|
};
|
|
|
|
async function installMock(page: Page, options: MockOptions = {}) {
|
|
const view = baseView();
|
|
if (options.empty) {
|
|
view.content_qualifications = [];
|
|
view.model_change_gates = [];
|
|
view.release_gates = [];
|
|
view.gate_artifacts = [];
|
|
view.incidents = [];
|
|
view.regression_dag_nodes = [];
|
|
view.lifecycle_events = [];
|
|
}
|
|
const approvalBodies: Record<string, unknown>[] = [];
|
|
|
|
await page.route("**/api/**", async (route) => {
|
|
const request = route.request();
|
|
const path = new URL(request.url()).pathname;
|
|
const fulfill = (body: unknown, status = 200) =>
|
|
route.fulfill({ status, contentType: "application/json", body: JSON.stringify(body) });
|
|
|
|
if (request.method() === "GET" && path.endsWith("/auth/me")) {
|
|
const role = options.role ?? "admin";
|
|
await fulfill({
|
|
user_id: "60000000-0000-4000-8000-000000000001",
|
|
email: `${role}@twentyoz.kr`,
|
|
display_name: `CI ${role}`,
|
|
role,
|
|
admin_access: false,
|
|
super_admin: false,
|
|
account_status: "approved",
|
|
approval_required: false,
|
|
cohort_ids: [],
|
|
consent_at: 1_754_378_000,
|
|
onboarding_completed_at: 1_754_378_000,
|
|
nickname: "",
|
|
self_introduction: "",
|
|
avatar_url: "",
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (request.method() === "GET" && path.endsWith("/continuous-improvement")) {
|
|
if (options.status && options.status !== 200) {
|
|
await fulfill({ detail: "continuous improvement read model unavailable" }, options.status);
|
|
return;
|
|
}
|
|
await fulfill(view);
|
|
return;
|
|
}
|
|
|
|
if (request.method() === "POST" && path.endsWith("/continuous-improvement/approvals")) {
|
|
const body = request.postDataJSON() as ApprovalBody;
|
|
approvalBodies.push(body);
|
|
view.approvals.push({
|
|
approval_event_id: body.approval_event_id,
|
|
target_kind: body.target_kind,
|
|
target_id: body.target_id,
|
|
decision: body.decision,
|
|
reason_code: body.reason_code,
|
|
evidence_refs: body.evidence_refs,
|
|
created_at: CREATED_AT,
|
|
});
|
|
if (body.target_kind === "content_qualification" && body.decision === "approve_content") {
|
|
view.catalog_entries.unshift({
|
|
catalog_record_id: body.effect_record_id,
|
|
qualification_id: body.target_id,
|
|
catalog_entry_id: "oas-g8-catalog-synthetic-rupture",
|
|
status: "approved",
|
|
clinical_claim_allowed: false,
|
|
created_at: CREATED_AT,
|
|
});
|
|
} else if (
|
|
body.decision === "approve_promotion" ||
|
|
body.decision === "authorize_rollback"
|
|
) {
|
|
view.lifecycle_events.unshift({
|
|
lifecycle_event_id: body.effect_record_id,
|
|
target_kind: body.target_kind as "model_change_gate" | "release_gate",
|
|
target_id: body.target_id,
|
|
event_type: body.decision === "authorize_rollback" ? "rollback" : "promotion",
|
|
event_status: body.decision === "authorize_rollback" ? "requested" : "approved",
|
|
evidence_refs: body.evidence_refs,
|
|
created_at: CREATED_AT,
|
|
});
|
|
}
|
|
await fulfill(
|
|
{
|
|
submission_id: body.submission_id,
|
|
approval_event_id: body.approval_event_id,
|
|
target_kind: body.target_kind,
|
|
target_id: body.target_id,
|
|
decision: body.decision,
|
|
effect_record_id: body.effect_record_id,
|
|
idempotent_replay: false,
|
|
},
|
|
201,
|
|
);
|
|
return;
|
|
}
|
|
|
|
await fulfill({ detail: `unmocked ${request.method()} ${path}` }, 404);
|
|
});
|
|
|
|
return { view, approvalBodies };
|
|
}
|
|
|
|
test.describe("continuous improvement admin cockpit", () => {
|
|
test("renders the generated-content, gate, incident and rollback ledgers fail-closed", async ({
|
|
page,
|
|
}, testInfo) => {
|
|
await page.emulateMedia({ colorScheme: "dark", reducedMotion: "reduce" });
|
|
await page.addInitScript(() => localStorage.setItem("vignette.theme", "dark"));
|
|
const mock = await installMock(page);
|
|
|
|
await page.goto("/admin/continuous-improvement");
|
|
|
|
await expect(page.getByRole("heading", { name: "승격보다 근거를 먼저 본다" })).toBeVisible();
|
|
await expect(page.getByText("합성 콘텐츠 검증 파이프라인")).toBeVisible();
|
|
await expect(page.getByText("모델 변경 · 릴리스 게이트")).toBeVisible();
|
|
await expect(page.getByText("사건 회귀 DAG")).toBeVisible();
|
|
await expect(page.getByText("모니터 · 롤백 원장")).toBeVisible();
|
|
await expect(page.getByText("롤백 실행 대기 · 실행기 미구성", { exact: true })).toBeVisible();
|
|
await expect(page.getByText("롤백 실행 실패", { exact: true })).toBeVisible();
|
|
await expect(page.getByText("롤백은 실행됐지만 verification 이벤트가 아직 없어")).toBeVisible();
|
|
const contentCandidate = page.locator(
|
|
'[data-qualification-id="20000000-0000-4000-8000-000000000001"]',
|
|
);
|
|
await expect(contentCandidate.getByText("합성 관계 균열 수선 연습")).toBeVisible();
|
|
await contentCandidate.getByText("검수 payload 펼쳐 보기").click();
|
|
await expect(contentCandidate.getByText("영향을 방어하지 않고 인정한 뒤")).toBeVisible();
|
|
await expect(contentCandidate).not.toContainText("hidden_answer");
|
|
const contentReason = contentCandidate.getByLabel("콘텐츠 결정 사유");
|
|
await expect(contentCandidate.getByRole("button", { name: "결정 선택 필요" })).toBeDisabled();
|
|
await contentCandidate.getByRole("radio", { name: "승인", exact: true }).check();
|
|
await expect(contentCandidate.getByRole("button", { name: "승인 기록" })).toBeDisabled();
|
|
await contentReason.fill("합성 경계와 수련 목표를 직접 검수함");
|
|
await contentReason.press("Enter");
|
|
await expect(contentCandidate.getByText("콘텐츠 승인 원장 기록됨")).toBeVisible();
|
|
await expect(page.locator("html")).toHaveAttribute("data-theme", "dark");
|
|
await page.getByTestId("continuous-improvement-cockpit").screenshot({
|
|
path: testInfo.outputPath("g8-continuous-improvement-before-approval.png"),
|
|
animations: "disabled",
|
|
});
|
|
|
|
const missingGate = page.locator(`[data-gate-id="${RELEASE_GATE_ID}"]`);
|
|
await expect(missingGate.getByRole("radio", { name: "승인", exact: true })).toBeDisabled();
|
|
await expect(missingGate.getByRole("radio", { name: "보류", exact: true })).toBeEnabled();
|
|
await expect(missingGate.getByRole("radio", { name: "반려", exact: true })).toBeEnabled();
|
|
await expect(missingGate.getByText("필수 증거 4종 미완료로 승인 차단")).toBeVisible();
|
|
await missingGate.screenshot({
|
|
path: testInfo.outputPath("g8-human-decision-approval-blocked.png"),
|
|
animations: "disabled",
|
|
});
|
|
|
|
const modelGate = page.locator(`[data-gate-id="${MODEL_GATE_ID}"]`);
|
|
const approvalReason = modelGate.getByLabel("사람 결정 사유");
|
|
await expect(modelGate.getByRole("button", { name: "결정 선택 필요" })).toBeDisabled();
|
|
await expect(modelGate.getByText("승인·보류·반려 중 하나를 명시적으로 선택해야 함")).toBeVisible();
|
|
await modelGate.getByRole("radio", { name: "승인", exact: true }).check();
|
|
const approvalButton = modelGate.getByRole("button", { name: "승인 기록" });
|
|
await expect(approvalButton).toBeDisabled();
|
|
await expect(modelGate.getByText("승인 사유를 먼저 입력해야 함")).toBeVisible();
|
|
await approvalReason.fill("기준선과 rollback runbook을 독립 검토함");
|
|
await expect(approvalButton).toBeEnabled();
|
|
await expect(modelGate.getByText("승인 결정과 사유를 append-only 원장에 기록할 준비됨")).toBeVisible();
|
|
await modelGate.screenshot({
|
|
path: testInfo.outputPath("g8-human-decision-approval-ready.png"),
|
|
animations: "disabled",
|
|
});
|
|
await approvalButton.focus();
|
|
await expect(approvalButton).toBeFocused();
|
|
await approvalReason.focus();
|
|
await approvalReason.press("Enter");
|
|
|
|
await expect(modelGate.getByText("승격 승인 원장 기록됨")).toBeVisible();
|
|
expect(mock.approvalBodies).toHaveLength(2);
|
|
expect(mock.approvalBodies[0]).toMatchObject({
|
|
target_kind: "content_qualification",
|
|
target_id: "20000000-0000-4000-8000-000000000001",
|
|
decision: "approve_content",
|
|
reason_code: "합성 경계와 수련 목표를 직접 검수함",
|
|
});
|
|
expect(mock.approvalBodies[1]).toMatchObject({
|
|
target_kind: "model_change_gate",
|
|
target_id: MODEL_GATE_ID,
|
|
decision: "approve_promotion",
|
|
reason_code: "기준선과 rollback runbook을 독립 검토함",
|
|
});
|
|
expect(mock.approvalBodies[1].evidence_refs).toHaveLength(4);
|
|
|
|
await expectNoHorizontalOverflow(page);
|
|
// index.html watchdog은 3.5초와 8초에 관리자 표면의 canonical marker를 확인한다.
|
|
// 정상 렌더된 지속 개선 콕핏을 dev asset 이름 때문에 진단 화면으로 오인하면 안 된다.
|
|
await page.waitForTimeout(8_600);
|
|
await expect(page.locator("#vignette-boot-diagnostic")).toHaveCount(0);
|
|
await page.screenshot({
|
|
path: testInfo.outputPath("g8-continuous-improvement-cockpit.png"),
|
|
fullPage: true,
|
|
animations: "disabled",
|
|
});
|
|
});
|
|
|
|
test("blocks only unmet approval and records hold, reject and approval as explicit decisions", async ({
|
|
page,
|
|
}) => {
|
|
const mock = await installMock(page);
|
|
await page.goto("/admin/continuous-improvement");
|
|
|
|
const releaseGate = page.locator(`[data-gate-id="${RELEASE_GATE_ID}"]`);
|
|
const releaseApprove = releaseGate.getByRole("radio", { name: "승인", exact: true });
|
|
await expect(releaseApprove).toBeDisabled();
|
|
await expect(releaseGate.getByText("필수 증거 4종 미완료로 승인 차단")).toBeVisible();
|
|
await expect(releaseGate.getByRole("radio", { name: "보류", exact: true })).toBeEnabled();
|
|
await expect(releaseGate.getByRole("radio", { name: "반려", exact: true })).toBeEnabled();
|
|
|
|
await releaseGate.getByRole("radio", { name: "보류", exact: true }).check();
|
|
await releaseGate.getByLabel("사람 결정 사유").fill("rollback 증거 보강 뒤 다시 검토");
|
|
await releaseGate.getByRole("button", { name: "보류 기록" }).click();
|
|
await expect(releaseGate.getByText("보류 원장 기록됨")).toBeVisible();
|
|
|
|
const contentCandidate = page.locator(
|
|
'[data-qualification-id="20000000-0000-4000-8000-000000000001"]',
|
|
);
|
|
await contentCandidate.getByRole("radio", { name: "반려", exact: true }).check();
|
|
await contentCandidate.getByLabel("콘텐츠 결정 사유").fill("검수 기준을 충족하지 못한 후보");
|
|
await contentCandidate.getByRole("button", { name: "반려 기록" }).click();
|
|
await expect(contentCandidate.getByText("반려 원장 기록됨")).toBeVisible();
|
|
|
|
const modelGate = page.locator(`[data-gate-id="${MODEL_GATE_ID}"]`);
|
|
await modelGate.getByRole("radio", { name: "승인", exact: true }).check();
|
|
await modelGate.getByLabel("사람 결정 사유").fill("독립 증거 4종과 승격 조건을 확인함");
|
|
await modelGate.getByRole("button", { name: "승인 기록" }).click();
|
|
await expect(modelGate.getByText("승격 승인 원장 기록됨")).toBeVisible();
|
|
|
|
expect(mock.approvalBodies).toHaveLength(3);
|
|
expect(mock.approvalBodies[0]).toMatchObject({
|
|
target_kind: "release_gate",
|
|
target_id: RELEASE_GATE_ID,
|
|
decision: "keep_quarantine",
|
|
reason_code: "rollback 증거 보강 뒤 다시 검토",
|
|
});
|
|
expect(mock.approvalBodies[1]).toMatchObject({
|
|
target_kind: "content_qualification",
|
|
target_id: "20000000-0000-4000-8000-000000000001",
|
|
decision: "reject",
|
|
reason_code: "검수 기준을 충족하지 못한 후보",
|
|
});
|
|
expect(mock.approvalBodies[2]).toMatchObject({
|
|
target_kind: "model_change_gate",
|
|
target_id: MODEL_GATE_ID,
|
|
decision: "approve_promotion",
|
|
reason_code: "독립 증거 4종과 승격 조건을 확인함",
|
|
});
|
|
await expectNoHorizontalOverflow(page);
|
|
});
|
|
|
|
test("shows empty and degraded states without opening an approval action", async ({ page }) => {
|
|
await installMock(page, { empty: true });
|
|
await page.goto("/admin/continuous-improvement");
|
|
await expect(page.getByText("검토할 콘텐츠 후보가 없어")).toBeVisible();
|
|
await expect(page.getByText("대기 중인 게이트가 없어")).toBeVisible();
|
|
await expect(page.getByText("등록된 운영 사건이 없어")).toBeVisible();
|
|
await expect(page.getByRole("button", { name: /승인 기록|결정 기록/ })).toHaveCount(0);
|
|
|
|
await page.unroute("**/api/**");
|
|
await installMock(page, { status: 503 });
|
|
await page.reload();
|
|
await expect(page.getByRole("heading", { name: "개선 원장을 확인할 수 없어" })).toBeVisible();
|
|
await expect(page.getByText(/승인 동작은 닫힌 상태/)).toBeVisible();
|
|
await expectNoHorizontalOverflow(page);
|
|
});
|
|
|
|
for (const role of ["teacher", "learner"] as const) {
|
|
test(`does not expose the admin cockpit to ${role}`, async ({ page }) => {
|
|
await installMock(page, { role });
|
|
await page.goto("/admin/continuous-improvement");
|
|
await expect(page).toHaveURL(role === "teacher" ? /\/teach$/ : /\/learn$/);
|
|
await expect(page.getByTestId("continuous-improvement-cockpit")).toHaveCount(0);
|
|
});
|
|
}
|
|
});
|