16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
906 lines
31 KiB
Python
906 lines
31 KiB
Python
"""Exercise G8 continuous-improvement gates over live HTTP and PostgreSQL."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import json
|
|
import secrets
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from http.cookiejar import CookieJar
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
|
|
DATA_CLASSIFICATION = "synthetic_replay_red_team_coverage_drift"
|
|
INTERNAL_HEADER = "X-Vignette-Continuous-Improvement-Token"
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ApiResponse:
|
|
status: int
|
|
body: Any
|
|
|
|
|
|
class ApiClient:
|
|
def __init__(self, base_url: str, timeout: float) -> None:
|
|
self.base_url = base_url.rstrip("/")
|
|
self.timeout = timeout
|
|
self._opener = urllib.request.build_opener(
|
|
urllib.request.HTTPCookieProcessor(CookieJar())
|
|
)
|
|
|
|
def request(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
payload: dict[str, Any] | None = None,
|
|
*,
|
|
expected: set[int] | None = None,
|
|
headers: dict[str, str] | None = None,
|
|
) -> ApiResponse:
|
|
data = None
|
|
request_headers = {"Accept": "application/json", **(headers or {})}
|
|
if payload is not None:
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(
|
|
f"{self.base_url}{path}",
|
|
data=data,
|
|
headers=request_headers,
|
|
method=method,
|
|
)
|
|
try:
|
|
with self._opener.open(request, timeout=self.timeout) as response:
|
|
raw = response.read().decode("utf-8")
|
|
result = ApiResponse(
|
|
response.status,
|
|
json.loads(raw) if raw else {},
|
|
)
|
|
except urllib.error.HTTPError as exc:
|
|
raw = exc.read().decode("utf-8", errors="replace")
|
|
try:
|
|
body = json.loads(raw) if raw else {}
|
|
except json.JSONDecodeError:
|
|
body = {"detail": raw[:500]}
|
|
result = ApiResponse(exc.code, body)
|
|
except urllib.error.URLError as exc:
|
|
raise SmokeError(
|
|
f"{method} {path} transport failed: {type(exc.reason).__name__}"
|
|
) from exc
|
|
if result.status not in (expected or {200}):
|
|
detail = result.body.get("detail") if isinstance(result.body, dict) else None
|
|
raise SmokeError(
|
|
f"{method} {path} returned HTTP {result.status}; detail={detail!r}"
|
|
)
|
|
return result
|
|
|
|
|
|
def _sign_in(
|
|
client: ApiClient,
|
|
*,
|
|
suffix: str,
|
|
identity: str,
|
|
role: str,
|
|
) -> None:
|
|
client.request(
|
|
"POST",
|
|
"/auth/dev-login",
|
|
{
|
|
"email": f"dev.e2e.ci.{identity}.{suffix}@hs.ac.kr",
|
|
"role": role,
|
|
"display_name": f"CI {identity.title()}",
|
|
"cohort_ids": ["e2e-hanshin"],
|
|
},
|
|
)
|
|
client.request(
|
|
"POST",
|
|
"/users/me/onboarding",
|
|
{
|
|
"legal_name": f"CI {identity.title()}",
|
|
"affiliation": "한신대학교",
|
|
"department": "상담심리학과",
|
|
"grade_level": "통합검증",
|
|
"phone": "010-0000-0000",
|
|
"contact_address": "경기도 오산시 한신대학교",
|
|
"nickname": f"CI {identity.title()}",
|
|
"self_introduction": "G8 연속 개선 API 검증 fixture입니다.",
|
|
"avatar_url": "",
|
|
"terms_accepted": True,
|
|
"privacy_accepted": True,
|
|
},
|
|
)
|
|
|
|
|
|
def _assert_safe_payload(value: Any, path: str = "response") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden = {
|
|
"aggregate",
|
|
"aggregate_score",
|
|
"global_score",
|
|
"overall_score",
|
|
"raw_transcript",
|
|
"total",
|
|
"total_score",
|
|
"transcript",
|
|
"utterance_text",
|
|
} & set(value)
|
|
if forbidden:
|
|
raise SmokeError(f"{path} exposed forbidden keys: {sorted(forbidden)}")
|
|
for flag in (
|
|
"clinical_claim_allowed",
|
|
"pii_included",
|
|
"raw_transcript_included",
|
|
"silent_auto_promotion_allowed",
|
|
):
|
|
if value.get(flag) is True:
|
|
raise SmokeError(f"{path} enabled forbidden flag {flag}")
|
|
for key, child in value.items():
|
|
_assert_safe_payload(child, f"{path}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, child in enumerate(value):
|
|
_assert_safe_payload(child, f"{path}[{index}]")
|
|
|
|
|
|
def _stable_retry(
|
|
client: ApiClient,
|
|
path: str,
|
|
body: dict[str, Any],
|
|
*,
|
|
headers: dict[str, str] | None = None,
|
|
id_field: str,
|
|
) -> dict[str, Any]:
|
|
created = client.request(
|
|
"POST",
|
|
path,
|
|
body,
|
|
expected={201},
|
|
headers=headers,
|
|
)
|
|
retried = client.request(
|
|
"POST",
|
|
path,
|
|
body,
|
|
expected={201},
|
|
headers=headers,
|
|
)
|
|
if created.body.get(id_field) != retried.body.get(id_field):
|
|
raise SmokeError(f"same submission changed {id_field} for {path}")
|
|
if retried.body.get("idempotent_replay") is not True:
|
|
raise SmokeError(f"same submission was not idempotent for {path}")
|
|
return created.body
|
|
|
|
|
|
def _artifact(kind: str, suffix: str, hash_character: str) -> dict[str, str]:
|
|
return {
|
|
"artifact_record_id": str(uuid4()),
|
|
"artifact_id": f"g8-{kind}-{suffix}",
|
|
"content_sha256": hash_character * 64,
|
|
"provenance_uri": f"audit://continuous-improvement/{suffix}/{kind}",
|
|
}
|
|
|
|
|
|
def _gate_artifacts(prefix: str, suffix: str) -> dict[str, Any]:
|
|
return {
|
|
"baseline": _artifact(f"{prefix}-baseline", suffix, "1"),
|
|
"threshold": _artifact(f"{prefix}-threshold", suffix, "2"),
|
|
"provenance": [_artifact(f"{prefix}-provenance", suffix, "3")],
|
|
"rollback": _artifact(f"{prefix}-rollback", suffix, "4"),
|
|
}
|
|
|
|
|
|
def _content_pipeline(suffix: str) -> dict[str, Any]:
|
|
source_id = f"oas-g8-source-live-{suffix}"
|
|
draft_id = f"oas-g8-draft-live-{suffix}"
|
|
payload_hash = "c" * 64
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"pipeline_id": str(uuid4()),
|
|
"benchmark_record_id": str(uuid4()),
|
|
"qualification_id": str(uuid4()),
|
|
"data_classification": DATA_CLASSIFICATION,
|
|
"draft": {
|
|
"draft_id": draft_id,
|
|
"content_kind": "case",
|
|
"source_refs": [source_id],
|
|
"generation_model": "content-agent-live-v1",
|
|
"prompt_version": "1.0.0",
|
|
"prompt_sha256": "b" * 64,
|
|
"payload_sha256": payload_hash,
|
|
"synthetic_identity_id": f"synthetic-identity-live-{suffix}",
|
|
"difficulty_level": 4,
|
|
"hidden_answer_fingerprint": "d" * 64,
|
|
"visible_answer_overlap_tokens": 0,
|
|
"pii_findings": 0,
|
|
"unsupported_clinical_claims": 0,
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": source_id,
|
|
"version": "1.0.0",
|
|
"content_sha256": "a" * 64,
|
|
"provenance_uri": f"repo://synthetic/g8/{suffix}/source",
|
|
"usage_status": "approved",
|
|
"citation_label": "합성 교육 source pack",
|
|
}
|
|
],
|
|
"reviews": [
|
|
{
|
|
"review_id": f"oas-g8-review-live-{suffix}-safety",
|
|
"draft_id": draft_id,
|
|
"reviewer_agent_id": f"red-team-safety-{suffix}",
|
|
"dimensions": ["safety", "identity", "pii", "grounding"],
|
|
"findings": [],
|
|
"reviewed_payload_sha256": payload_hash,
|
|
},
|
|
{
|
|
"review_id": f"oas-g8-review-live-{suffix}-bias",
|
|
"draft_id": draft_id,
|
|
"reviewer_agent_id": f"red-team-bias-{suffix}",
|
|
"dimensions": ["answer_leakage", "cultural_bias", "difficulty"],
|
|
"findings": [],
|
|
"reviewed_payload_sha256": payload_hash,
|
|
},
|
|
],
|
|
"benchmark": {
|
|
"benchmark_id": f"oas-g8-benchmark-live-{suffix}",
|
|
"draft_id": draft_id,
|
|
"variant_count": 8,
|
|
"variant_pass_rate": 0.875,
|
|
"answer_leakage_count": 0,
|
|
"pii_finding_count": 0,
|
|
"unsupported_claim_count": 0,
|
|
"safety_failure_count": 0,
|
|
"reward_hacking_count": 0,
|
|
"evidence_refs": [f"audit://synthetic/g8/{suffix}/benchmark"],
|
|
},
|
|
}
|
|
|
|
|
|
def _model_gate(suffix: str) -> dict[str, Any]:
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"gate_id": str(uuid4()),
|
|
"baseline_snapshot_record_id": str(uuid4()),
|
|
"candidate_snapshot_record_id": str(uuid4()),
|
|
"data_classification": DATA_CLASSIFICATION,
|
|
"baseline": {
|
|
"snapshot_id": f"oas-g8-model-snapshot-baseline-{suffix}",
|
|
"model": "evaluator-baseline",
|
|
"prompt_version": "1.0.0",
|
|
"benchmark_version": "1.0.0",
|
|
"task_accuracy": 0.9,
|
|
"critical_miss_count": 0,
|
|
"leakage_count": 0,
|
|
"pii_count": 0,
|
|
"calibration_error": 0.12,
|
|
"subgroup_max_gap": 0.1,
|
|
},
|
|
"candidate": {
|
|
"snapshot_id": f"oas-g8-model-snapshot-candidate-{suffix}",
|
|
"model": "evaluator-candidate",
|
|
"prompt_version": "1.1.0",
|
|
"benchmark_version": "1.0.0",
|
|
"task_accuracy": 0.91,
|
|
"critical_miss_count": 1,
|
|
"leakage_count": 0,
|
|
"pii_count": 0,
|
|
"calibration_error": 0.11,
|
|
"subgroup_max_gap": 0.09,
|
|
},
|
|
"artifacts": _gate_artifacts("model", suffix),
|
|
}
|
|
|
|
|
|
def _release_gate(suffix: str) -> dict[str, Any]:
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"gate_id": str(uuid4()),
|
|
"data_classification": DATA_CLASSIFICATION,
|
|
"manifest": {
|
|
"release_id": f"oas-g8-release-live-{suffix}",
|
|
"red_green_passed": True,
|
|
"contract_passed": True,
|
|
"e2e_passed": True,
|
|
"runtime_proof_passed": True,
|
|
"public_proof_passed": True,
|
|
"ssot_synced": True,
|
|
"evidence_refs": [f"audit://continuous-improvement/{suffix}/release"],
|
|
},
|
|
"artifacts": _gate_artifacts("release", suffix),
|
|
}
|
|
|
|
|
|
def _approval(
|
|
*,
|
|
target_kind: str,
|
|
target_id: str,
|
|
decision: str,
|
|
suffix: str,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"approval_event_id": str(uuid4()),
|
|
"effect_record_id": str(uuid4()),
|
|
"target_kind": target_kind,
|
|
"target_id": target_id,
|
|
"decision": decision,
|
|
"reason_code": f"g8_live_{decision}",
|
|
"evidence_refs": [f"audit://continuous-improvement/{suffix}/{decision}"],
|
|
}
|
|
|
|
|
|
def _monitor(
|
|
*,
|
|
target_kind: str,
|
|
target_id: str,
|
|
event_status: str,
|
|
suffix: str,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"lifecycle_event_id": str(uuid4()),
|
|
"data_classification": DATA_CLASSIFICATION,
|
|
"target_kind": target_kind,
|
|
"target_id": target_id,
|
|
"event_status": event_status,
|
|
"evidence_refs": [
|
|
f"audit://continuous-improvement/{suffix}/monitor-{event_status}"
|
|
],
|
|
}
|
|
|
|
|
|
def _incident(suffix: str) -> dict[str, Any]:
|
|
return {
|
|
"submission_id": str(uuid4()),
|
|
"incident_record_id": str(uuid4()),
|
|
"data_classification": DATA_CLASSIFICATION,
|
|
"incident": {
|
|
"incident_id": f"oas-g8-incident-live-{suffix}",
|
|
"error_fingerprint": "e" * 64,
|
|
"affected_contract": "continuous-improvement.synthetic-replay",
|
|
"evidence_refs": [
|
|
f"audit://continuous-improvement/{suffix}/incident"
|
|
],
|
|
"pii_included": False,
|
|
},
|
|
}
|
|
|
|
|
|
def _matching(items: list[dict[str, Any]], key: str, value: str) -> list[dict[str, Any]]:
|
|
return [item for item in items if str(item.get(key)) == value]
|
|
|
|
|
|
def _assert_preapproval_no_effect(
|
|
view: dict[str, Any],
|
|
*,
|
|
qualification_id: str,
|
|
model_gate_id: str,
|
|
release_gate_id: str,
|
|
) -> None:
|
|
if _matching(view["catalog_entries"], "qualification_id", qualification_id):
|
|
raise SmokeError("content entered catalog before human approval")
|
|
targets = {model_gate_id, release_gate_id, qualification_id}
|
|
if any(str(item.get("target_id")) in targets for item in view["approvals"]):
|
|
raise SmokeError("approval existed before the human gate")
|
|
if any(
|
|
str(item.get("target_id")) in {model_gate_id, release_gate_id}
|
|
for item in view["lifecycle_events"]
|
|
):
|
|
raise SmokeError("model/release effect existed before human approval")
|
|
|
|
|
|
def _assert_gate_artifacts(
|
|
view: dict[str, Any],
|
|
*,
|
|
owner_kind: str,
|
|
owner_id: str,
|
|
) -> None:
|
|
artifacts = [
|
|
item
|
|
for item in view["gate_artifacts"]
|
|
if item.get("owner_kind") == owner_kind
|
|
and str(item.get("owner_id")) == owner_id
|
|
]
|
|
kinds = {item.get("artifact_kind") for item in artifacts}
|
|
if len(artifacts) != 4 or kinds != {
|
|
"baseline",
|
|
"threshold",
|
|
"provenance",
|
|
"rollback",
|
|
}:
|
|
raise SmokeError(f"{owner_kind} omitted one of four gate artifacts")
|
|
|
|
|
|
def _assert_incident_dag(
|
|
view: dict[str, Any],
|
|
*,
|
|
incident_record_id: str,
|
|
) -> None:
|
|
nodes = _matching(
|
|
view["regression_dag_nodes"],
|
|
"incident_record_id",
|
|
incident_record_id,
|
|
)
|
|
if len(nodes) != 4:
|
|
raise SmokeError("incident DAG did not persist exactly four nodes")
|
|
by_type = {str(item["node_type"]): item for item in nodes}
|
|
expected = {"reproduction_test", "implementation", "e2e", "runtime_proof"}
|
|
if set(by_type) != expected:
|
|
raise SmokeError("incident DAG node types are incomplete")
|
|
reproduction = by_type["reproduction_test"]
|
|
implementation = by_type["implementation"]
|
|
e2e = by_type["e2e"]
|
|
runtime = by_type["runtime_proof"]
|
|
if reproduction["depends_on_record_ids"]:
|
|
raise SmokeError("incident reproduction node unexpectedly has a dependency")
|
|
chain = (
|
|
(implementation, reproduction),
|
|
(e2e, implementation),
|
|
(runtime, e2e),
|
|
)
|
|
for child, parent in chain:
|
|
if child["depends_on_record_ids"] != [parent["node_record_id"]]:
|
|
raise SmokeError("incident DAG dependency chain is not ordered")
|
|
|
|
|
|
def _rollback_execution_proof(event: dict[str, Any]) -> dict[str, Any]:
|
|
status = str(event.get("event_status") or "")
|
|
if status not in {"requested", "failed", "executed"}:
|
|
raise SmokeError(f"unexpected rollback lifecycle status: {status}")
|
|
evidence_refs = event.get("evidence_refs")
|
|
if not isinstance(evidence_refs, list) or not evidence_refs:
|
|
raise SmokeError("rollback lifecycle omitted durable evidence refs")
|
|
executor_evidence_refs = event.get("executor_evidence_refs")
|
|
if executor_evidence_refs is None:
|
|
executor_evidence_refs = []
|
|
if not isinstance(executor_evidence_refs, list):
|
|
raise SmokeError("rollback executor evidence refs are malformed")
|
|
receipt_id = event.get("executor_receipt_id")
|
|
approval_event_id = event.get("approval_event_id")
|
|
artifact_record_id = event.get("artifact_record_id")
|
|
if not approval_event_id or not artifact_record_id:
|
|
raise SmokeError("rollback lifecycle omitted approval or pinned artifact")
|
|
if status == "executed":
|
|
if not isinstance(receipt_id, str) or not receipt_id.strip():
|
|
raise SmokeError("executed rollback omitted executor receipt id")
|
|
if not executor_evidence_refs:
|
|
raise SmokeError("executed rollback omitted executor evidence refs")
|
|
if not set(executor_evidence_refs).issubset(evidence_refs):
|
|
raise SmokeError("executor evidence is not bound into lifecycle evidence")
|
|
elif receipt_id is not None or executor_evidence_refs:
|
|
raise SmokeError("non-executed rollback carried false receipt evidence")
|
|
return {
|
|
"lifecycle_event_id": event.get("lifecycle_event_id"),
|
|
"target_kind": event.get("target_kind"),
|
|
"target_id": event.get("target_id"),
|
|
"status": status,
|
|
"approval_event_id": approval_event_id,
|
|
"artifact_record_id": artifact_record_id,
|
|
"executor_receipt_id": receipt_id,
|
|
"evidence_refs": evidence_refs,
|
|
"executor_evidence_refs": executor_evidence_refs,
|
|
"executed_receipt_bound": status == "executed",
|
|
"receipt_contract_satisfied": True,
|
|
}
|
|
|
|
|
|
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
if len(args.internal_token) < 32:
|
|
raise SmokeError("--internal-token must contain at least 32 characters")
|
|
root = ApiClient(args.api_base_url, args.request_timeout)
|
|
health = root.request("GET", "/health").body
|
|
if not health.get("db") or not health.get("engine"):
|
|
raise SmokeError("API health is not DB+engine ready")
|
|
|
|
suffix = f"{int(time.time())}-{secrets.token_hex(4)}"
|
|
internal = ApiClient(args.api_base_url, args.request_timeout)
|
|
admin = ApiClient(args.api_base_url, args.request_timeout)
|
|
teacher = ApiClient(args.api_base_url, args.request_timeout)
|
|
learner = ApiClient(args.api_base_url, args.request_timeout)
|
|
_sign_in(admin, suffix=suffix, identity="admin", role="admin")
|
|
_sign_in(teacher, suffix=suffix, identity="teacher", role="teacher")
|
|
_sign_in(learner, suffix=suffix, identity="learner", role="learner")
|
|
|
|
token_headers = {INTERNAL_HEADER: args.internal_token}
|
|
internal.request("GET", "/internal/continuous-improvement", expected={401})
|
|
internal.request(
|
|
"GET",
|
|
"/internal/continuous-improvement",
|
|
expected={403},
|
|
headers={INTERNAL_HEADER: "wrong-token"},
|
|
)
|
|
teacher.request("GET", "/continuous-improvement", expected={403})
|
|
learner.request("GET", "/continuous-improvement", expected={403})
|
|
|
|
content = _content_pipeline(suffix)
|
|
content_path = "/internal/continuous-improvement/content-pipelines"
|
|
content_result = _stable_retry(
|
|
internal,
|
|
content_path,
|
|
content,
|
|
headers=token_headers,
|
|
id_field="qualification_id",
|
|
)
|
|
if (
|
|
content_result.get("state") != "pending_human_approval"
|
|
or content_result.get("human_approval_required") is not True
|
|
or content_result.get("catalog_promoted") is not False
|
|
):
|
|
raise SmokeError("content qualification bypassed the human gate")
|
|
changed_content = copy.deepcopy(content)
|
|
changed_content["benchmark"]["variant_pass_rate"] = 0.9
|
|
internal.request(
|
|
"POST",
|
|
content_path,
|
|
changed_content,
|
|
expected={409},
|
|
headers=token_headers,
|
|
)
|
|
|
|
model_gate = _model_gate(suffix)
|
|
model_path = "/internal/continuous-improvement/model-change-gates"
|
|
model_result = _stable_retry(
|
|
internal,
|
|
model_path,
|
|
model_gate,
|
|
headers=token_headers,
|
|
id_field="gate_id",
|
|
)
|
|
if (
|
|
model_result.get("gate_decision") != "rollback"
|
|
or model_result.get("promotion_executed") is not False
|
|
):
|
|
raise SmokeError("unsafe model candidate did not remain pending rollback")
|
|
changed_model = copy.deepcopy(model_gate)
|
|
changed_model["candidate"]["task_accuracy"] = 0.89
|
|
internal.request(
|
|
"POST",
|
|
model_path,
|
|
changed_model,
|
|
expected={409},
|
|
headers=token_headers,
|
|
)
|
|
|
|
release_gate = _release_gate(suffix)
|
|
release_path = "/internal/continuous-improvement/release-gates"
|
|
release_result = _stable_retry(
|
|
internal,
|
|
release_path,
|
|
release_gate,
|
|
headers=token_headers,
|
|
id_field="gate_id",
|
|
)
|
|
if (
|
|
release_result.get("qualified") is not True
|
|
or release_result.get("promotion_executed") is not False
|
|
):
|
|
raise SmokeError("qualified release bypassed or failed its human gate")
|
|
changed_release = copy.deepcopy(release_gate)
|
|
changed_release["manifest"]["evidence_refs"].append(
|
|
f"audit://continuous-improvement/{suffix}/changed"
|
|
)
|
|
internal.request(
|
|
"POST",
|
|
release_path,
|
|
changed_release,
|
|
expected={409},
|
|
headers=token_headers,
|
|
)
|
|
|
|
incident = _incident(suffix)
|
|
incident_path = "/internal/continuous-improvement/incidents"
|
|
incident_result = _stable_retry(
|
|
internal,
|
|
incident_path,
|
|
incident,
|
|
headers=token_headers,
|
|
id_field="incident_record_id",
|
|
)
|
|
if incident_result.get("node_count") != 4:
|
|
raise SmokeError("incident response omitted the four-node regression DAG")
|
|
changed_incident = copy.deepcopy(incident)
|
|
changed_incident["incident"]["affected_contract"] = "changed.contract"
|
|
internal.request(
|
|
"POST",
|
|
incident_path,
|
|
changed_incident,
|
|
expected={409},
|
|
headers=token_headers,
|
|
)
|
|
|
|
preapproval_view = internal.request(
|
|
"GET",
|
|
"/internal/continuous-improvement",
|
|
headers=token_headers,
|
|
).body
|
|
qualification_id = str(content_result["qualification_id"])
|
|
model_gate_id = str(model_result["gate_id"])
|
|
release_gate_id = str(release_result["gate_id"])
|
|
_assert_preapproval_no_effect(
|
|
preapproval_view,
|
|
qualification_id=qualification_id,
|
|
model_gate_id=model_gate_id,
|
|
release_gate_id=release_gate_id,
|
|
)
|
|
_assert_gate_artifacts(
|
|
preapproval_view,
|
|
owner_kind="model_change_gate",
|
|
owner_id=model_gate_id,
|
|
)
|
|
_assert_gate_artifacts(
|
|
preapproval_view,
|
|
owner_kind="release_gate",
|
|
owner_id=release_gate_id,
|
|
)
|
|
|
|
content_approval = _approval(
|
|
target_kind="content_qualification",
|
|
target_id=qualification_id,
|
|
decision="approve_content",
|
|
suffix=suffix,
|
|
)
|
|
teacher.request(
|
|
"POST",
|
|
"/continuous-improvement/approvals",
|
|
content_approval,
|
|
expected={403},
|
|
)
|
|
learner.request(
|
|
"POST",
|
|
"/continuous-improvement/approvals",
|
|
content_approval,
|
|
expected={403},
|
|
)
|
|
content_approval_result = _stable_retry(
|
|
admin,
|
|
"/continuous-improvement/approvals",
|
|
content_approval,
|
|
id_field="approval_event_id",
|
|
)
|
|
changed_approval = copy.deepcopy(content_approval)
|
|
changed_approval["reason_code"] = "changed_reason_must_conflict"
|
|
admin.request(
|
|
"POST",
|
|
"/continuous-improvement/approvals",
|
|
changed_approval,
|
|
expected={409},
|
|
)
|
|
|
|
model_approval = _approval(
|
|
target_kind="model_change_gate",
|
|
target_id=model_gate_id,
|
|
decision="authorize_rollback",
|
|
suffix=suffix,
|
|
)
|
|
model_approval_result = _stable_retry(
|
|
admin,
|
|
"/continuous-improvement/approvals",
|
|
model_approval,
|
|
id_field="approval_event_id",
|
|
)
|
|
release_approval = _approval(
|
|
target_kind="release_gate",
|
|
target_id=release_gate_id,
|
|
decision="approve_promotion",
|
|
suffix=suffix,
|
|
)
|
|
release_approval_result = _stable_retry(
|
|
admin,
|
|
"/continuous-improvement/approvals",
|
|
release_approval,
|
|
id_field="approval_event_id",
|
|
)
|
|
|
|
monitor_results: dict[str, str] = {}
|
|
for event_status, target_kind, target_id in (
|
|
("healthy", "release_gate", release_gate_id),
|
|
("drift_detected", "model_change_gate", model_gate_id),
|
|
("rollback_recommended", "model_change_gate", model_gate_id),
|
|
):
|
|
body = _monitor(
|
|
target_kind=target_kind,
|
|
target_id=target_id,
|
|
event_status=event_status,
|
|
suffix=suffix,
|
|
)
|
|
result = _stable_retry(
|
|
internal,
|
|
"/internal/continuous-improvement/monitor-events",
|
|
body,
|
|
headers=token_headers,
|
|
id_field="lifecycle_event_id",
|
|
)
|
|
monitor_results[event_status] = str(result["lifecycle_event_id"])
|
|
if event_status == "healthy":
|
|
changed_monitor = copy.deepcopy(body)
|
|
changed_monitor["evidence_refs"].append(
|
|
f"audit://continuous-improvement/{suffix}/changed-monitor"
|
|
)
|
|
internal.request(
|
|
"POST",
|
|
"/internal/continuous-improvement/monitor-events",
|
|
changed_monitor,
|
|
expected={409},
|
|
headers=token_headers,
|
|
)
|
|
|
|
internal_view = internal.request(
|
|
"GET",
|
|
"/internal/continuous-improvement",
|
|
headers=token_headers,
|
|
).body
|
|
admin_view = admin.request("GET", "/continuous-improvement").body
|
|
model_rollbacks = [
|
|
item
|
|
for item in admin_view["lifecycle_events"]
|
|
if str(item.get("target_id")) == model_gate_id
|
|
and item.get("event_type") == "rollback"
|
|
]
|
|
if len(model_rollbacks) != 1:
|
|
raise SmokeError("model rollback authorization did not append one lifecycle event")
|
|
rollback_execution = _rollback_execution_proof(model_rollbacks[0])
|
|
rollback_status = str(rollback_execution["status"])
|
|
if (
|
|
args.expected_rollback_status
|
|
and rollback_status != args.expected_rollback_status
|
|
):
|
|
raise SmokeError(
|
|
"rollback lifecycle status did not match expectation: "
|
|
f"expected={args.expected_rollback_status} actual={rollback_status}"
|
|
)
|
|
if rollback_status == "executed":
|
|
body = _monitor(
|
|
target_kind="model_change_gate",
|
|
target_id=model_gate_id,
|
|
event_status="rollback_verified",
|
|
suffix=suffix,
|
|
)
|
|
result = _stable_retry(
|
|
internal,
|
|
"/internal/continuous-improvement/monitor-events",
|
|
body,
|
|
headers=token_headers,
|
|
id_field="lifecycle_event_id",
|
|
)
|
|
monitor_results["rollback_verified"] = str(result["lifecycle_event_id"])
|
|
internal_view = internal.request(
|
|
"GET",
|
|
"/internal/continuous-improvement",
|
|
headers=token_headers,
|
|
).body
|
|
admin_view = admin.request("GET", "/continuous-improvement").body
|
|
for payload in (internal_view, admin_view):
|
|
_assert_safe_payload(payload)
|
|
|
|
catalog = _matching(
|
|
admin_view["catalog_entries"],
|
|
"qualification_id",
|
|
qualification_id,
|
|
)
|
|
if len(catalog) != 1 or catalog[0].get("status") != "approved":
|
|
raise SmokeError("admin content approval did not append one catalog effect")
|
|
own_approvals = [
|
|
item
|
|
for item in admin_view["approvals"]
|
|
if str(item.get("target_id"))
|
|
in {qualification_id, model_gate_id, release_gate_id}
|
|
]
|
|
if len(own_approvals) != 3:
|
|
raise SmokeError("human approvals were not append-only across three targets")
|
|
|
|
lifecycle = [
|
|
item
|
|
for item in admin_view["lifecycle_events"]
|
|
if str(item.get("target_id")) in {model_gate_id, release_gate_id}
|
|
]
|
|
lifecycle_pairs = {
|
|
(str(item.get("event_type")), str(item.get("event_status")))
|
|
for item in lifecycle
|
|
}
|
|
required_pairs = {
|
|
("promotion", "approved"),
|
|
("rollback", rollback_status),
|
|
("monitor", "healthy"),
|
|
("monitor", "drift_detected"),
|
|
("monitor", "rollback_recommended"),
|
|
}
|
|
if rollback_status == "executed":
|
|
required_pairs.add(("monitor", "rollback_verified"))
|
|
if not required_pairs.issubset(lifecycle_pairs):
|
|
raise SmokeError("promotion/rollback/monitor lifecycle is incomplete")
|
|
_assert_incident_dag(
|
|
admin_view,
|
|
incident_record_id=str(incident_result["incident_record_id"]),
|
|
)
|
|
|
|
return {
|
|
"ok": True,
|
|
"api_base_url": args.api_base_url,
|
|
"executed_at_unix": int(time.time()),
|
|
"fixture_policy": "retained unique dev:e2e identities; no fixture deletion",
|
|
"run_suffix": suffix,
|
|
"identifiers": {
|
|
"pipeline_id": content_result["pipeline_id"],
|
|
"qualification_id": qualification_id,
|
|
"catalog_entry_id": content_result["candidate_catalog_entry_id"],
|
|
"catalog_record_id": content_approval_result["effect_record_id"],
|
|
"model_gate_id": model_gate_id,
|
|
"model_rollback_event_id": model_approval_result["effect_record_id"],
|
|
"release_gate_id": release_gate_id,
|
|
"release_promotion_event_id": release_approval_result[
|
|
"effect_record_id"
|
|
],
|
|
"incident_record_id": incident_result["incident_record_id"],
|
|
"monitor_event_ids": monitor_results,
|
|
},
|
|
"observed_counts": {
|
|
"model_gate_artifacts": 4,
|
|
"release_gate_artifacts": 4,
|
|
"own_human_approvals": len(own_approvals),
|
|
"own_lifecycle_events": len(lifecycle),
|
|
"incident_dag_nodes": 4,
|
|
},
|
|
"rollback_execution": rollback_execution,
|
|
"proof": {
|
|
"health_db_engine_ready": True,
|
|
"missing_internal_token_401": True,
|
|
"wrong_internal_token_403": True,
|
|
"learner_role_blocked": True,
|
|
"teacher_role_blocked": True,
|
|
"source_draft_independent_redteam_benchmark_persisted": True,
|
|
"content_pending_before_human_approval": True,
|
|
"catalog_absent_before_human_approval": True,
|
|
"model_effect_absent_before_human_approval": True,
|
|
"release_effect_absent_before_human_approval": True,
|
|
"model_gate_four_artifact_kinds": True,
|
|
"release_gate_four_artifact_kinds": True,
|
|
"content_admin_approval_append_only": True,
|
|
"model_rollback_admin_authorized": True,
|
|
"model_rollback_status_recorded": True,
|
|
"model_rollback_receipt_contract_satisfied": True,
|
|
"release_promotion_admin_approved": True,
|
|
"monitor_lifecycle_closed": True,
|
|
"incident_four_node_dag_ordered": True,
|
|
"same_submission_retries_stable": True,
|
|
"changed_submission_409": True,
|
|
"raw_transcript_included": False,
|
|
"pii_included": False,
|
|
"clinical_claim_allowed": False,
|
|
"silent_auto_promotion_allowed": False,
|
|
"no_aggregate_score": True,
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--api-base-url", default="http://127.0.0.1:8014")
|
|
parser.add_argument("--internal-token", required=True)
|
|
parser.add_argument("--request-timeout", type=float, default=180.0)
|
|
parser.add_argument(
|
|
"--expected-rollback-status",
|
|
choices=("requested", "failed", "executed"),
|
|
default="",
|
|
)
|
|
parser.add_argument("--out", default="")
|
|
args = parser.parse_args()
|
|
result = run(args)
|
|
text = json.dumps(result, ensure_ascii=False, indent=2)
|
|
if args.out:
|
|
path = Path(args.out)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text + "\n", encoding="utf-8")
|
|
print(text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|