G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
906
scripts/smoke-continuous-improvement-api.py
Normal file
906
scripts/smoke-continuous-improvement-api.py
Normal file
|
|
@ -0,0 +1,906 @@
|
|||
"""Exercise G8 continuous-improvement gates over live HTTP and PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
DATA_CLASSIFICATION = "synthetic_replay_red_team_coverage_drift"
|
||||
INTERNAL_HEADER = "X-Vignette-Continuous-Improvement-Token"
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApiResponse:
|
||||
status: int
|
||||
body: Any
|
||||
|
||||
|
||||
class ApiClient:
|
||||
def __init__(self, base_url: str, timeout: float) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(CookieJar())
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
*,
|
||||
expected: set[int] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> ApiResponse:
|
||||
data = None
|
||||
request_headers = {"Accept": "application/json", **(headers or {})}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=data,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
result = ApiResponse(
|
||||
response.status,
|
||||
json.loads(raw) if raw else {},
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
body = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
body = {"detail": raw[:500]}
|
||||
result = ApiResponse(exc.code, body)
|
||||
except urllib.error.URLError as exc:
|
||||
raise SmokeError(
|
||||
f"{method} {path} transport failed: {type(exc.reason).__name__}"
|
||||
) from exc
|
||||
if result.status not in (expected or {200}):
|
||||
detail = result.body.get("detail") if isinstance(result.body, dict) else None
|
||||
raise SmokeError(
|
||||
f"{method} {path} returned HTTP {result.status}; detail={detail!r}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _sign_in(
|
||||
client: ApiClient,
|
||||
*,
|
||||
suffix: str,
|
||||
identity: str,
|
||||
role: str,
|
||||
) -> None:
|
||||
client.request(
|
||||
"POST",
|
||||
"/auth/dev-login",
|
||||
{
|
||||
"email": f"dev.e2e.ci.{identity}.{suffix}@hs.ac.kr",
|
||||
"role": role,
|
||||
"display_name": f"CI {identity.title()}",
|
||||
"cohort_ids": ["e2e-hanshin"],
|
||||
},
|
||||
)
|
||||
client.request(
|
||||
"POST",
|
||||
"/users/me/onboarding",
|
||||
{
|
||||
"legal_name": f"CI {identity.title()}",
|
||||
"affiliation": "한신대학교",
|
||||
"department": "상담심리학과",
|
||||
"grade_level": "통합검증",
|
||||
"phone": "010-0000-0000",
|
||||
"contact_address": "경기도 오산시 한신대학교",
|
||||
"nickname": f"CI {identity.title()}",
|
||||
"self_introduction": "G8 연속 개선 API 검증 fixture입니다.",
|
||||
"avatar_url": "",
|
||||
"terms_accepted": True,
|
||||
"privacy_accepted": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _assert_safe_payload(value: Any, path: str = "response") -> None:
|
||||
if isinstance(value, dict):
|
||||
forbidden = {
|
||||
"aggregate",
|
||||
"aggregate_score",
|
||||
"global_score",
|
||||
"overall_score",
|
||||
"raw_transcript",
|
||||
"total",
|
||||
"total_score",
|
||||
"transcript",
|
||||
"utterance_text",
|
||||
} & set(value)
|
||||
if forbidden:
|
||||
raise SmokeError(f"{path} exposed forbidden keys: {sorted(forbidden)}")
|
||||
for flag in (
|
||||
"clinical_claim_allowed",
|
||||
"pii_included",
|
||||
"raw_transcript_included",
|
||||
"silent_auto_promotion_allowed",
|
||||
):
|
||||
if value.get(flag) is True:
|
||||
raise SmokeError(f"{path} enabled forbidden flag {flag}")
|
||||
for key, child in value.items():
|
||||
_assert_safe_payload(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_assert_safe_payload(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def _stable_retry(
|
||||
client: ApiClient,
|
||||
path: str,
|
||||
body: dict[str, Any],
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
id_field: str,
|
||||
) -> dict[str, Any]:
|
||||
created = client.request(
|
||||
"POST",
|
||||
path,
|
||||
body,
|
||||
expected={201},
|
||||
headers=headers,
|
||||
)
|
||||
retried = client.request(
|
||||
"POST",
|
||||
path,
|
||||
body,
|
||||
expected={201},
|
||||
headers=headers,
|
||||
)
|
||||
if created.body.get(id_field) != retried.body.get(id_field):
|
||||
raise SmokeError(f"same submission changed {id_field} for {path}")
|
||||
if retried.body.get("idempotent_replay") is not True:
|
||||
raise SmokeError(f"same submission was not idempotent for {path}")
|
||||
return created.body
|
||||
|
||||
|
||||
def _artifact(kind: str, suffix: str, hash_character: str) -> dict[str, str]:
|
||||
return {
|
||||
"artifact_record_id": str(uuid4()),
|
||||
"artifact_id": f"g8-{kind}-{suffix}",
|
||||
"content_sha256": hash_character * 64,
|
||||
"provenance_uri": f"audit://continuous-improvement/{suffix}/{kind}",
|
||||
}
|
||||
|
||||
|
||||
def _gate_artifacts(prefix: str, suffix: str) -> dict[str, Any]:
|
||||
return {
|
||||
"baseline": _artifact(f"{prefix}-baseline", suffix, "1"),
|
||||
"threshold": _artifact(f"{prefix}-threshold", suffix, "2"),
|
||||
"provenance": [_artifact(f"{prefix}-provenance", suffix, "3")],
|
||||
"rollback": _artifact(f"{prefix}-rollback", suffix, "4"),
|
||||
}
|
||||
|
||||
|
||||
def _content_pipeline(suffix: str) -> dict[str, Any]:
|
||||
source_id = f"oas-g8-source-live-{suffix}"
|
||||
draft_id = f"oas-g8-draft-live-{suffix}"
|
||||
payload_hash = "c" * 64
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"pipeline_id": str(uuid4()),
|
||||
"benchmark_record_id": str(uuid4()),
|
||||
"qualification_id": str(uuid4()),
|
||||
"data_classification": DATA_CLASSIFICATION,
|
||||
"draft": {
|
||||
"draft_id": draft_id,
|
||||
"content_kind": "case",
|
||||
"source_refs": [source_id],
|
||||
"generation_model": "content-agent-live-v1",
|
||||
"prompt_version": "1.0.0",
|
||||
"prompt_sha256": "b" * 64,
|
||||
"payload_sha256": payload_hash,
|
||||
"synthetic_identity_id": f"synthetic-identity-live-{suffix}",
|
||||
"difficulty_level": 4,
|
||||
"hidden_answer_fingerprint": "d" * 64,
|
||||
"visible_answer_overlap_tokens": 0,
|
||||
"pii_findings": 0,
|
||||
"unsupported_clinical_claims": 0,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"source_id": source_id,
|
||||
"version": "1.0.0",
|
||||
"content_sha256": "a" * 64,
|
||||
"provenance_uri": f"repo://synthetic/g8/{suffix}/source",
|
||||
"usage_status": "approved",
|
||||
"citation_label": "합성 교육 source pack",
|
||||
}
|
||||
],
|
||||
"reviews": [
|
||||
{
|
||||
"review_id": f"oas-g8-review-live-{suffix}-safety",
|
||||
"draft_id": draft_id,
|
||||
"reviewer_agent_id": f"red-team-safety-{suffix}",
|
||||
"dimensions": ["safety", "identity", "pii", "grounding"],
|
||||
"findings": [],
|
||||
"reviewed_payload_sha256": payload_hash,
|
||||
},
|
||||
{
|
||||
"review_id": f"oas-g8-review-live-{suffix}-bias",
|
||||
"draft_id": draft_id,
|
||||
"reviewer_agent_id": f"red-team-bias-{suffix}",
|
||||
"dimensions": ["answer_leakage", "cultural_bias", "difficulty"],
|
||||
"findings": [],
|
||||
"reviewed_payload_sha256": payload_hash,
|
||||
},
|
||||
],
|
||||
"benchmark": {
|
||||
"benchmark_id": f"oas-g8-benchmark-live-{suffix}",
|
||||
"draft_id": draft_id,
|
||||
"variant_count": 8,
|
||||
"variant_pass_rate": 0.875,
|
||||
"answer_leakage_count": 0,
|
||||
"pii_finding_count": 0,
|
||||
"unsupported_claim_count": 0,
|
||||
"safety_failure_count": 0,
|
||||
"reward_hacking_count": 0,
|
||||
"evidence_refs": [f"audit://synthetic/g8/{suffix}/benchmark"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _model_gate(suffix: str) -> dict[str, Any]:
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"gate_id": str(uuid4()),
|
||||
"baseline_snapshot_record_id": str(uuid4()),
|
||||
"candidate_snapshot_record_id": str(uuid4()),
|
||||
"data_classification": DATA_CLASSIFICATION,
|
||||
"baseline": {
|
||||
"snapshot_id": f"oas-g8-model-snapshot-baseline-{suffix}",
|
||||
"model": "evaluator-baseline",
|
||||
"prompt_version": "1.0.0",
|
||||
"benchmark_version": "1.0.0",
|
||||
"task_accuracy": 0.9,
|
||||
"critical_miss_count": 0,
|
||||
"leakage_count": 0,
|
||||
"pii_count": 0,
|
||||
"calibration_error": 0.12,
|
||||
"subgroup_max_gap": 0.1,
|
||||
},
|
||||
"candidate": {
|
||||
"snapshot_id": f"oas-g8-model-snapshot-candidate-{suffix}",
|
||||
"model": "evaluator-candidate",
|
||||
"prompt_version": "1.1.0",
|
||||
"benchmark_version": "1.0.0",
|
||||
"task_accuracy": 0.91,
|
||||
"critical_miss_count": 1,
|
||||
"leakage_count": 0,
|
||||
"pii_count": 0,
|
||||
"calibration_error": 0.11,
|
||||
"subgroup_max_gap": 0.09,
|
||||
},
|
||||
"artifacts": _gate_artifacts("model", suffix),
|
||||
}
|
||||
|
||||
|
||||
def _release_gate(suffix: str) -> dict[str, Any]:
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"gate_id": str(uuid4()),
|
||||
"data_classification": DATA_CLASSIFICATION,
|
||||
"manifest": {
|
||||
"release_id": f"oas-g8-release-live-{suffix}",
|
||||
"red_green_passed": True,
|
||||
"contract_passed": True,
|
||||
"e2e_passed": True,
|
||||
"runtime_proof_passed": True,
|
||||
"public_proof_passed": True,
|
||||
"ssot_synced": True,
|
||||
"evidence_refs": [f"audit://continuous-improvement/{suffix}/release"],
|
||||
},
|
||||
"artifacts": _gate_artifacts("release", suffix),
|
||||
}
|
||||
|
||||
|
||||
def _approval(
|
||||
*,
|
||||
target_kind: str,
|
||||
target_id: str,
|
||||
decision: str,
|
||||
suffix: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"approval_event_id": str(uuid4()),
|
||||
"effect_record_id": str(uuid4()),
|
||||
"target_kind": target_kind,
|
||||
"target_id": target_id,
|
||||
"decision": decision,
|
||||
"reason_code": f"g8_live_{decision}",
|
||||
"evidence_refs": [f"audit://continuous-improvement/{suffix}/{decision}"],
|
||||
}
|
||||
|
||||
|
||||
def _monitor(
|
||||
*,
|
||||
target_kind: str,
|
||||
target_id: str,
|
||||
event_status: str,
|
||||
suffix: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"lifecycle_event_id": str(uuid4()),
|
||||
"data_classification": DATA_CLASSIFICATION,
|
||||
"target_kind": target_kind,
|
||||
"target_id": target_id,
|
||||
"event_status": event_status,
|
||||
"evidence_refs": [
|
||||
f"audit://continuous-improvement/{suffix}/monitor-{event_status}"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _incident(suffix: str) -> dict[str, Any]:
|
||||
return {
|
||||
"submission_id": str(uuid4()),
|
||||
"incident_record_id": str(uuid4()),
|
||||
"data_classification": DATA_CLASSIFICATION,
|
||||
"incident": {
|
||||
"incident_id": f"oas-g8-incident-live-{suffix}",
|
||||
"error_fingerprint": "e" * 64,
|
||||
"affected_contract": "continuous-improvement.synthetic-replay",
|
||||
"evidence_refs": [
|
||||
f"audit://continuous-improvement/{suffix}/incident"
|
||||
],
|
||||
"pii_included": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _matching(items: list[dict[str, Any]], key: str, value: str) -> list[dict[str, Any]]:
|
||||
return [item for item in items if str(item.get(key)) == value]
|
||||
|
||||
|
||||
def _assert_preapproval_no_effect(
|
||||
view: dict[str, Any],
|
||||
*,
|
||||
qualification_id: str,
|
||||
model_gate_id: str,
|
||||
release_gate_id: str,
|
||||
) -> None:
|
||||
if _matching(view["catalog_entries"], "qualification_id", qualification_id):
|
||||
raise SmokeError("content entered catalog before human approval")
|
||||
targets = {model_gate_id, release_gate_id, qualification_id}
|
||||
if any(str(item.get("target_id")) in targets for item in view["approvals"]):
|
||||
raise SmokeError("approval existed before the human gate")
|
||||
if any(
|
||||
str(item.get("target_id")) in {model_gate_id, release_gate_id}
|
||||
for item in view["lifecycle_events"]
|
||||
):
|
||||
raise SmokeError("model/release effect existed before human approval")
|
||||
|
||||
|
||||
def _assert_gate_artifacts(
|
||||
view: dict[str, Any],
|
||||
*,
|
||||
owner_kind: str,
|
||||
owner_id: str,
|
||||
) -> None:
|
||||
artifacts = [
|
||||
item
|
||||
for item in view["gate_artifacts"]
|
||||
if item.get("owner_kind") == owner_kind
|
||||
and str(item.get("owner_id")) == owner_id
|
||||
]
|
||||
kinds = {item.get("artifact_kind") for item in artifacts}
|
||||
if len(artifacts) != 4 or kinds != {
|
||||
"baseline",
|
||||
"threshold",
|
||||
"provenance",
|
||||
"rollback",
|
||||
}:
|
||||
raise SmokeError(f"{owner_kind} omitted one of four gate artifacts")
|
||||
|
||||
|
||||
def _assert_incident_dag(
|
||||
view: dict[str, Any],
|
||||
*,
|
||||
incident_record_id: str,
|
||||
) -> None:
|
||||
nodes = _matching(
|
||||
view["regression_dag_nodes"],
|
||||
"incident_record_id",
|
||||
incident_record_id,
|
||||
)
|
||||
if len(nodes) != 4:
|
||||
raise SmokeError("incident DAG did not persist exactly four nodes")
|
||||
by_type = {str(item["node_type"]): item for item in nodes}
|
||||
expected = {"reproduction_test", "implementation", "e2e", "runtime_proof"}
|
||||
if set(by_type) != expected:
|
||||
raise SmokeError("incident DAG node types are incomplete")
|
||||
reproduction = by_type["reproduction_test"]
|
||||
implementation = by_type["implementation"]
|
||||
e2e = by_type["e2e"]
|
||||
runtime = by_type["runtime_proof"]
|
||||
if reproduction["depends_on_record_ids"]:
|
||||
raise SmokeError("incident reproduction node unexpectedly has a dependency")
|
||||
chain = (
|
||||
(implementation, reproduction),
|
||||
(e2e, implementation),
|
||||
(runtime, e2e),
|
||||
)
|
||||
for child, parent in chain:
|
||||
if child["depends_on_record_ids"] != [parent["node_record_id"]]:
|
||||
raise SmokeError("incident DAG dependency chain is not ordered")
|
||||
|
||||
|
||||
def _rollback_execution_proof(event: dict[str, Any]) -> dict[str, Any]:
|
||||
status = str(event.get("event_status") or "")
|
||||
if status not in {"requested", "failed", "executed"}:
|
||||
raise SmokeError(f"unexpected rollback lifecycle status: {status}")
|
||||
evidence_refs = event.get("evidence_refs")
|
||||
if not isinstance(evidence_refs, list) or not evidence_refs:
|
||||
raise SmokeError("rollback lifecycle omitted durable evidence refs")
|
||||
executor_evidence_refs = event.get("executor_evidence_refs")
|
||||
if executor_evidence_refs is None:
|
||||
executor_evidence_refs = []
|
||||
if not isinstance(executor_evidence_refs, list):
|
||||
raise SmokeError("rollback executor evidence refs are malformed")
|
||||
receipt_id = event.get("executor_receipt_id")
|
||||
approval_event_id = event.get("approval_event_id")
|
||||
artifact_record_id = event.get("artifact_record_id")
|
||||
if not approval_event_id or not artifact_record_id:
|
||||
raise SmokeError("rollback lifecycle omitted approval or pinned artifact")
|
||||
if status == "executed":
|
||||
if not isinstance(receipt_id, str) or not receipt_id.strip():
|
||||
raise SmokeError("executed rollback omitted executor receipt id")
|
||||
if not executor_evidence_refs:
|
||||
raise SmokeError("executed rollback omitted executor evidence refs")
|
||||
if not set(executor_evidence_refs).issubset(evidence_refs):
|
||||
raise SmokeError("executor evidence is not bound into lifecycle evidence")
|
||||
elif receipt_id is not None or executor_evidence_refs:
|
||||
raise SmokeError("non-executed rollback carried false receipt evidence")
|
||||
return {
|
||||
"lifecycle_event_id": event.get("lifecycle_event_id"),
|
||||
"target_kind": event.get("target_kind"),
|
||||
"target_id": event.get("target_id"),
|
||||
"status": status,
|
||||
"approval_event_id": approval_event_id,
|
||||
"artifact_record_id": artifact_record_id,
|
||||
"executor_receipt_id": receipt_id,
|
||||
"evidence_refs": evidence_refs,
|
||||
"executor_evidence_refs": executor_evidence_refs,
|
||||
"executed_receipt_bound": status == "executed",
|
||||
"receipt_contract_satisfied": True,
|
||||
}
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
if len(args.internal_token) < 32:
|
||||
raise SmokeError("--internal-token must contain at least 32 characters")
|
||||
root = ApiClient(args.api_base_url, args.request_timeout)
|
||||
health = root.request("GET", "/health").body
|
||||
if not health.get("db") or not health.get("engine"):
|
||||
raise SmokeError("API health is not DB+engine ready")
|
||||
|
||||
suffix = f"{int(time.time())}-{secrets.token_hex(4)}"
|
||||
internal = ApiClient(args.api_base_url, args.request_timeout)
|
||||
admin = ApiClient(args.api_base_url, args.request_timeout)
|
||||
teacher = ApiClient(args.api_base_url, args.request_timeout)
|
||||
learner = ApiClient(args.api_base_url, args.request_timeout)
|
||||
_sign_in(admin, suffix=suffix, identity="admin", role="admin")
|
||||
_sign_in(teacher, suffix=suffix, identity="teacher", role="teacher")
|
||||
_sign_in(learner, suffix=suffix, identity="learner", role="learner")
|
||||
|
||||
token_headers = {INTERNAL_HEADER: args.internal_token}
|
||||
internal.request("GET", "/internal/continuous-improvement", expected={401})
|
||||
internal.request(
|
||||
"GET",
|
||||
"/internal/continuous-improvement",
|
||||
expected={403},
|
||||
headers={INTERNAL_HEADER: "wrong-token"},
|
||||
)
|
||||
teacher.request("GET", "/continuous-improvement", expected={403})
|
||||
learner.request("GET", "/continuous-improvement", expected={403})
|
||||
|
||||
content = _content_pipeline(suffix)
|
||||
content_path = "/internal/continuous-improvement/content-pipelines"
|
||||
content_result = _stable_retry(
|
||||
internal,
|
||||
content_path,
|
||||
content,
|
||||
headers=token_headers,
|
||||
id_field="qualification_id",
|
||||
)
|
||||
if (
|
||||
content_result.get("state") != "pending_human_approval"
|
||||
or content_result.get("human_approval_required") is not True
|
||||
or content_result.get("catalog_promoted") is not False
|
||||
):
|
||||
raise SmokeError("content qualification bypassed the human gate")
|
||||
changed_content = copy.deepcopy(content)
|
||||
changed_content["benchmark"]["variant_pass_rate"] = 0.9
|
||||
internal.request(
|
||||
"POST",
|
||||
content_path,
|
||||
changed_content,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
model_gate = _model_gate(suffix)
|
||||
model_path = "/internal/continuous-improvement/model-change-gates"
|
||||
model_result = _stable_retry(
|
||||
internal,
|
||||
model_path,
|
||||
model_gate,
|
||||
headers=token_headers,
|
||||
id_field="gate_id",
|
||||
)
|
||||
if (
|
||||
model_result.get("gate_decision") != "rollback"
|
||||
or model_result.get("promotion_executed") is not False
|
||||
):
|
||||
raise SmokeError("unsafe model candidate did not remain pending rollback")
|
||||
changed_model = copy.deepcopy(model_gate)
|
||||
changed_model["candidate"]["task_accuracy"] = 0.89
|
||||
internal.request(
|
||||
"POST",
|
||||
model_path,
|
||||
changed_model,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
release_gate = _release_gate(suffix)
|
||||
release_path = "/internal/continuous-improvement/release-gates"
|
||||
release_result = _stable_retry(
|
||||
internal,
|
||||
release_path,
|
||||
release_gate,
|
||||
headers=token_headers,
|
||||
id_field="gate_id",
|
||||
)
|
||||
if (
|
||||
release_result.get("qualified") is not True
|
||||
or release_result.get("promotion_executed") is not False
|
||||
):
|
||||
raise SmokeError("qualified release bypassed or failed its human gate")
|
||||
changed_release = copy.deepcopy(release_gate)
|
||||
changed_release["manifest"]["evidence_refs"].append(
|
||||
f"audit://continuous-improvement/{suffix}/changed"
|
||||
)
|
||||
internal.request(
|
||||
"POST",
|
||||
release_path,
|
||||
changed_release,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
incident = _incident(suffix)
|
||||
incident_path = "/internal/continuous-improvement/incidents"
|
||||
incident_result = _stable_retry(
|
||||
internal,
|
||||
incident_path,
|
||||
incident,
|
||||
headers=token_headers,
|
||||
id_field="incident_record_id",
|
||||
)
|
||||
if incident_result.get("node_count") != 4:
|
||||
raise SmokeError("incident response omitted the four-node regression DAG")
|
||||
changed_incident = copy.deepcopy(incident)
|
||||
changed_incident["incident"]["affected_contract"] = "changed.contract"
|
||||
internal.request(
|
||||
"POST",
|
||||
incident_path,
|
||||
changed_incident,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
preapproval_view = internal.request(
|
||||
"GET",
|
||||
"/internal/continuous-improvement",
|
||||
headers=token_headers,
|
||||
).body
|
||||
qualification_id = str(content_result["qualification_id"])
|
||||
model_gate_id = str(model_result["gate_id"])
|
||||
release_gate_id = str(release_result["gate_id"])
|
||||
_assert_preapproval_no_effect(
|
||||
preapproval_view,
|
||||
qualification_id=qualification_id,
|
||||
model_gate_id=model_gate_id,
|
||||
release_gate_id=release_gate_id,
|
||||
)
|
||||
_assert_gate_artifacts(
|
||||
preapproval_view,
|
||||
owner_kind="model_change_gate",
|
||||
owner_id=model_gate_id,
|
||||
)
|
||||
_assert_gate_artifacts(
|
||||
preapproval_view,
|
||||
owner_kind="release_gate",
|
||||
owner_id=release_gate_id,
|
||||
)
|
||||
|
||||
content_approval = _approval(
|
||||
target_kind="content_qualification",
|
||||
target_id=qualification_id,
|
||||
decision="approve_content",
|
||||
suffix=suffix,
|
||||
)
|
||||
teacher.request(
|
||||
"POST",
|
||||
"/continuous-improvement/approvals",
|
||||
content_approval,
|
||||
expected={403},
|
||||
)
|
||||
learner.request(
|
||||
"POST",
|
||||
"/continuous-improvement/approvals",
|
||||
content_approval,
|
||||
expected={403},
|
||||
)
|
||||
content_approval_result = _stable_retry(
|
||||
admin,
|
||||
"/continuous-improvement/approvals",
|
||||
content_approval,
|
||||
id_field="approval_event_id",
|
||||
)
|
||||
changed_approval = copy.deepcopy(content_approval)
|
||||
changed_approval["reason_code"] = "changed_reason_must_conflict"
|
||||
admin.request(
|
||||
"POST",
|
||||
"/continuous-improvement/approvals",
|
||||
changed_approval,
|
||||
expected={409},
|
||||
)
|
||||
|
||||
model_approval = _approval(
|
||||
target_kind="model_change_gate",
|
||||
target_id=model_gate_id,
|
||||
decision="authorize_rollback",
|
||||
suffix=suffix,
|
||||
)
|
||||
model_approval_result = _stable_retry(
|
||||
admin,
|
||||
"/continuous-improvement/approvals",
|
||||
model_approval,
|
||||
id_field="approval_event_id",
|
||||
)
|
||||
release_approval = _approval(
|
||||
target_kind="release_gate",
|
||||
target_id=release_gate_id,
|
||||
decision="approve_promotion",
|
||||
suffix=suffix,
|
||||
)
|
||||
release_approval_result = _stable_retry(
|
||||
admin,
|
||||
"/continuous-improvement/approvals",
|
||||
release_approval,
|
||||
id_field="approval_event_id",
|
||||
)
|
||||
|
||||
monitor_results: dict[str, str] = {}
|
||||
for event_status, target_kind, target_id in (
|
||||
("healthy", "release_gate", release_gate_id),
|
||||
("drift_detected", "model_change_gate", model_gate_id),
|
||||
("rollback_recommended", "model_change_gate", model_gate_id),
|
||||
):
|
||||
body = _monitor(
|
||||
target_kind=target_kind,
|
||||
target_id=target_id,
|
||||
event_status=event_status,
|
||||
suffix=suffix,
|
||||
)
|
||||
result = _stable_retry(
|
||||
internal,
|
||||
"/internal/continuous-improvement/monitor-events",
|
||||
body,
|
||||
headers=token_headers,
|
||||
id_field="lifecycle_event_id",
|
||||
)
|
||||
monitor_results[event_status] = str(result["lifecycle_event_id"])
|
||||
if event_status == "healthy":
|
||||
changed_monitor = copy.deepcopy(body)
|
||||
changed_monitor["evidence_refs"].append(
|
||||
f"audit://continuous-improvement/{suffix}/changed-monitor"
|
||||
)
|
||||
internal.request(
|
||||
"POST",
|
||||
"/internal/continuous-improvement/monitor-events",
|
||||
changed_monitor,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
internal_view = internal.request(
|
||||
"GET",
|
||||
"/internal/continuous-improvement",
|
||||
headers=token_headers,
|
||||
).body
|
||||
admin_view = admin.request("GET", "/continuous-improvement").body
|
||||
model_rollbacks = [
|
||||
item
|
||||
for item in admin_view["lifecycle_events"]
|
||||
if str(item.get("target_id")) == model_gate_id
|
||||
and item.get("event_type") == "rollback"
|
||||
]
|
||||
if len(model_rollbacks) != 1:
|
||||
raise SmokeError("model rollback authorization did not append one lifecycle event")
|
||||
rollback_execution = _rollback_execution_proof(model_rollbacks[0])
|
||||
rollback_status = str(rollback_execution["status"])
|
||||
if (
|
||||
args.expected_rollback_status
|
||||
and rollback_status != args.expected_rollback_status
|
||||
):
|
||||
raise SmokeError(
|
||||
"rollback lifecycle status did not match expectation: "
|
||||
f"expected={args.expected_rollback_status} actual={rollback_status}"
|
||||
)
|
||||
if rollback_status == "executed":
|
||||
body = _monitor(
|
||||
target_kind="model_change_gate",
|
||||
target_id=model_gate_id,
|
||||
event_status="rollback_verified",
|
||||
suffix=suffix,
|
||||
)
|
||||
result = _stable_retry(
|
||||
internal,
|
||||
"/internal/continuous-improvement/monitor-events",
|
||||
body,
|
||||
headers=token_headers,
|
||||
id_field="lifecycle_event_id",
|
||||
)
|
||||
monitor_results["rollback_verified"] = str(result["lifecycle_event_id"])
|
||||
internal_view = internal.request(
|
||||
"GET",
|
||||
"/internal/continuous-improvement",
|
||||
headers=token_headers,
|
||||
).body
|
||||
admin_view = admin.request("GET", "/continuous-improvement").body
|
||||
for payload in (internal_view, admin_view):
|
||||
_assert_safe_payload(payload)
|
||||
|
||||
catalog = _matching(
|
||||
admin_view["catalog_entries"],
|
||||
"qualification_id",
|
||||
qualification_id,
|
||||
)
|
||||
if len(catalog) != 1 or catalog[0].get("status") != "approved":
|
||||
raise SmokeError("admin content approval did not append one catalog effect")
|
||||
own_approvals = [
|
||||
item
|
||||
for item in admin_view["approvals"]
|
||||
if str(item.get("target_id"))
|
||||
in {qualification_id, model_gate_id, release_gate_id}
|
||||
]
|
||||
if len(own_approvals) != 3:
|
||||
raise SmokeError("human approvals were not append-only across three targets")
|
||||
|
||||
lifecycle = [
|
||||
item
|
||||
for item in admin_view["lifecycle_events"]
|
||||
if str(item.get("target_id")) in {model_gate_id, release_gate_id}
|
||||
]
|
||||
lifecycle_pairs = {
|
||||
(str(item.get("event_type")), str(item.get("event_status")))
|
||||
for item in lifecycle
|
||||
}
|
||||
required_pairs = {
|
||||
("promotion", "approved"),
|
||||
("rollback", rollback_status),
|
||||
("monitor", "healthy"),
|
||||
("monitor", "drift_detected"),
|
||||
("monitor", "rollback_recommended"),
|
||||
}
|
||||
if rollback_status == "executed":
|
||||
required_pairs.add(("monitor", "rollback_verified"))
|
||||
if not required_pairs.issubset(lifecycle_pairs):
|
||||
raise SmokeError("promotion/rollback/monitor lifecycle is incomplete")
|
||||
_assert_incident_dag(
|
||||
admin_view,
|
||||
incident_record_id=str(incident_result["incident_record_id"]),
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"api_base_url": args.api_base_url,
|
||||
"executed_at_unix": int(time.time()),
|
||||
"fixture_policy": "retained unique dev:e2e identities; no fixture deletion",
|
||||
"run_suffix": suffix,
|
||||
"identifiers": {
|
||||
"pipeline_id": content_result["pipeline_id"],
|
||||
"qualification_id": qualification_id,
|
||||
"catalog_entry_id": content_result["candidate_catalog_entry_id"],
|
||||
"catalog_record_id": content_approval_result["effect_record_id"],
|
||||
"model_gate_id": model_gate_id,
|
||||
"model_rollback_event_id": model_approval_result["effect_record_id"],
|
||||
"release_gate_id": release_gate_id,
|
||||
"release_promotion_event_id": release_approval_result[
|
||||
"effect_record_id"
|
||||
],
|
||||
"incident_record_id": incident_result["incident_record_id"],
|
||||
"monitor_event_ids": monitor_results,
|
||||
},
|
||||
"observed_counts": {
|
||||
"model_gate_artifacts": 4,
|
||||
"release_gate_artifacts": 4,
|
||||
"own_human_approvals": len(own_approvals),
|
||||
"own_lifecycle_events": len(lifecycle),
|
||||
"incident_dag_nodes": 4,
|
||||
},
|
||||
"rollback_execution": rollback_execution,
|
||||
"proof": {
|
||||
"health_db_engine_ready": True,
|
||||
"missing_internal_token_401": True,
|
||||
"wrong_internal_token_403": True,
|
||||
"learner_role_blocked": True,
|
||||
"teacher_role_blocked": True,
|
||||
"source_draft_independent_redteam_benchmark_persisted": True,
|
||||
"content_pending_before_human_approval": True,
|
||||
"catalog_absent_before_human_approval": True,
|
||||
"model_effect_absent_before_human_approval": True,
|
||||
"release_effect_absent_before_human_approval": True,
|
||||
"model_gate_four_artifact_kinds": True,
|
||||
"release_gate_four_artifact_kinds": True,
|
||||
"content_admin_approval_append_only": True,
|
||||
"model_rollback_admin_authorized": True,
|
||||
"model_rollback_status_recorded": True,
|
||||
"model_rollback_receipt_contract_satisfied": True,
|
||||
"release_promotion_admin_approved": True,
|
||||
"monitor_lifecycle_closed": True,
|
||||
"incident_four_node_dag_ordered": True,
|
||||
"same_submission_retries_stable": True,
|
||||
"changed_submission_409": True,
|
||||
"raw_transcript_included": False,
|
||||
"pii_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
"silent_auto_promotion_allowed": False,
|
||||
"no_aggregate_score": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--api-base-url", default="http://127.0.0.1:8014")
|
||||
parser.add_argument("--internal-token", required=True)
|
||||
parser.add_argument("--request-timeout", type=float, default=180.0)
|
||||
parser.add_argument(
|
||||
"--expected-rollback-status",
|
||||
choices=("requested", "failed", "executed"),
|
||||
default="",
|
||||
)
|
||||
parser.add_argument("--out", default="")
|
||||
args = parser.parse_args()
|
||||
result = run(args)
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.out:
|
||||
path = Path(args.out)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue