"""Exercise the G6 supervision/research ledgers over live HTTP and PostgreSQL.""" from __future__ import annotations import argparse import copy import json import secrets import time import urllib.error import urllib.request from dataclasses import dataclass from http.cookiejar import CookieJar from pathlib import Path from typing import Any from uuid import uuid4 COHORT_ID = "e2e-hanshin" OTHER_COHORT_ID = "e2e-other-cohort" LEARNER_ID = "fa05a889-b546-4e3b-a959-a8efc32f1073" SESSION_ID = "3c834ffd-af5b-4a37-8fd3-ad0690deb165" ATTEMPT_IDS = ( "16a76b48-00ab-4a5f-8c12-e9ae2489db38", "9fd642a9-6476-4d3e-8e3f-c35a1f75f1bf", ) class SmokeError(RuntimeError): pass @dataclass(frozen=True) class ApiResponse: status: int body: Any class ApiClient: def __init__(self, base_url: str, timeout: float) -> None: self.base_url = base_url.rstrip("/") self.timeout = timeout self._opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(CookieJar()) ) def request( self, method: str, path: str, payload: dict[str, Any] | None = None, *, expected: set[int] | None = None, headers: dict[str, str] | None = None, ) -> ApiResponse: data = None request_headers = {"Accept": "application/json", **(headers or {})} if payload is not None: data = json.dumps(payload, ensure_ascii=False).encode("utf-8") request_headers["Content-Type"] = "application/json" request = urllib.request.Request( f"{self.base_url}{path}", data=data, headers=request_headers, method=method, ) try: with self._opener.open(request, timeout=self.timeout) as response: raw = response.read().decode("utf-8") result = ApiResponse(response.status, json.loads(raw) if raw else {}) except urllib.error.HTTPError as exc: raw = exc.read().decode("utf-8", errors="replace") try: body = json.loads(raw) if raw else {} except json.JSONDecodeError: body = {"detail": raw[:500]} result = ApiResponse(exc.code, body) except urllib.error.URLError as exc: raise SmokeError( f"{method} {path} transport failed: {type(exc.reason).__name__}" ) from exc if result.status not in (expected or {200}): detail = result.body.get("detail") if isinstance(result.body, dict) else None raise SmokeError( f"{method} {path} returned HTTP {result.status}; detail={detail!r}" ) return result def _sign_in( client: ApiClient, *, suffix: str, identity: str, role: str, cohort_ids: list[str], ) -> str: client.request( "POST", "/auth/dev-login", { "email": f"dev.e2e.supervision.{identity}.{suffix}@hs.ac.kr", "role": role, "display_name": f"Supervision {identity.title()}", "cohort_ids": cohort_ids, }, ) client.request( "POST", "/users/me/onboarding", { "legal_name": f"Supervision {identity.title()}", "affiliation": "한신대학교", "department": "상담심리학과", "grade_level": "통합검증", "phone": "010-0000-0000", "contact_address": "경기도 오산시 한신대학교", "nickname": f"Supervision {identity.title()}", "self_introduction": "G6 감독·연구 API 검증 fixture입니다.", "avatar_url": "", "terms_accepted": True, "privacy_accepted": True, }, ) user_id = str(client.request("GET", "/auth/me").body.get("user_id") or "") if not user_id: raise SmokeError(f"dev-login omitted user_id for {identity}") return user_id def _pointer(attempt_id: str) -> dict[str, Any]: return { "ledger": "practice_attempt", "event_id": attempt_id, "session_id": SESSION_ID, "route_hint": f"/practice/attempts/{attempt_id}", } def _assert_safe_payload(value: Any, path: str = "response") -> None: if isinstance(value, dict): forbidden = { "total", "total_score", "overall_score", "global_score", "raw_transcript", "transcript", "utterance_text", } & set(value) if forbidden: raise SmokeError(f"{path} exposed forbidden keys: {forbidden}") if value.get("clinical_claim_allowed") is True: raise SmokeError(f"{path} enabled a clinical claim") if value.get("raw_transcript_included") is True: raise SmokeError(f"{path} included raw transcript data") for key, child in value.items(): _assert_safe_payload(child, f"{path}.{key}") elif isinstance(value, list): for index, child in enumerate(value): _assert_safe_payload(child, f"{path}[{index}]") def _stable_retry( client: ApiClient, path: str, body: dict[str, Any], *, headers: dict[str, str] | None = None, id_field: str, ) -> dict[str, Any]: created = client.request("POST", path, body, expected={201}, headers=headers) retried = client.request("POST", path, body, expected={201}, headers=headers) if created.body.get(id_field) != retried.body.get(id_field) or retried.body.get( "idempotent_replay" ) is not True: raise SmokeError(f"same submission retry was not stable for {path}") return created.body def run(args: argparse.Namespace) -> dict[str, Any]: if len(args.internal_token) < 32: raise SmokeError("--internal-token must contain at least 32 characters") root = ApiClient(args.api_base_url, args.request_timeout) health = root.request("GET", "/health") if not health.body.get("db") or not health.body.get("engine"): raise SmokeError("API health is not DB+engine ready") pack = json.loads(Path(args.benchmark_path).read_text(encoding="utf-8")) suffix = f"{int(time.time())}.{secrets.token_hex(3)}" teacher = ApiClient(args.api_base_url, args.request_timeout) learner = ApiClient(args.api_base_url, args.request_timeout) other_teacher = ApiClient(args.api_base_url, args.request_timeout) _sign_in( teacher, suffix=suffix, identity="teacher", role="teacher", cohort_ids=[COHORT_ID], ) _sign_in( learner, suffix=suffix, identity="learner", role="learner", cohort_ids=[COHORT_ID], ) _sign_in( other_teacher, suffix=suffix, identity="other-teacher", role="teacher", cohort_ids=[OTHER_COHORT_ID], ) internal = ApiClient(args.api_base_url, args.request_timeout) token_headers = { "X-Vignette-Supervision-Research-Token": args.internal_token } internal.request( "GET", "/internal/supervision-research/supervisor-view", expected={401} ) internal.request( "GET", "/internal/supervision-research/research-view", expected={403}, headers={"X-Vignette-Supervision-Research-Token": "wrong-token"}, ) live_suffix = secrets.token_hex(5) signal = copy.deepcopy(pack["attention_signals"][4]) signal["signal_id"] = f"oas-g6-signal-live-{live_suffix}" signal["learner_ref"] = "learner-live" signal["evidence"] = [_pointer(ATTEMPT_IDS[0])] attention = { "submission_id": str(uuid4()), "snapshot_id": str(uuid4()), "cohort_id": COHORT_ID, "signals": [signal], "learners": [{"learner_ref": "learner-live", "learner_id": LEARNER_ID}], } attention_path = "/internal/supervision-research/attention-snapshots" attention_result = _stable_retry( internal, attention_path, attention, headers=token_headers, id_field="snapshot_id", ) changed_attention = copy.deepcopy(attention) changed_attention["signals"][0]["uncertainty"] = 0.31 internal.request( "POST", attention_path, changed_attention, expected={409}, headers=token_headers, ) gap = { "submission_id": str(uuid4()), "gap_snapshot_id": str(uuid4()), "cohort_id": COHORT_ID, "competency_id": "competency.empathy.reflection", "gap_kind": "growth_stagnation", "status": "observed", "uncertainty": 0.25, "affected_learner_count": 1, "evidence": [{"learner_id": LEARNER_ID, "pointer": _pointer(ATTEMPT_IDS[0])}], } gap_result = _stable_retry( internal, "/internal/supervision-research/curriculum-gaps", gap, headers=token_headers, id_field="gap_snapshot_id", ) disagreement = copy.deepcopy(pack["disagreements"][0]) disagreement["disagreement_id"] = f"oas-g6-disagreement-live-{live_suffix}" disagreement["ai_evidence"] = [_pointer(ATTEMPT_IDS[0])] disagreement["teacher_correction_evidence"] = [_pointer(ATTEMPT_IDS[1])] disagreement_body = { "submission_id": str(uuid4()), "disagreement_record_id": str(uuid4()), "dataset_row_id": str(uuid4()), "audit_event_id": str(uuid4()), "learner_id": LEARNER_ID, "cohort_id": COHORT_ID, "disagreement": disagreement, } disagreement_path = "/supervision-research/teacher-disagreements" disagreement_result = _stable_retry( teacher, disagreement_path, disagreement_body, id_field="disagreement_record_id", ) changed_disagreement = copy.deepcopy(disagreement_body) changed_disagreement["disagreement"]["correction_reason_code"] = ( "changed_content_must_conflict" ) teacher.request("POST", disagreement_path, changed_disagreement, expected={409}) baseline = copy.deepcopy(pack["baseline_batch"]) candidate = copy.deepcopy(pack["candidate_batch"]) baseline["batch_id"] = f"oas-g6-batch-baseline-{live_suffix}" candidate["batch_id"] = f"oas-g6-batch-candidate-{live_suffix}" for index, item in enumerate(baseline["observations"]): item["evidence_event_id"] = ATTEMPT_IDS[index % 2] for index, item in enumerate(candidate["observations"]): item["evidence_event_id"] = ATTEMPT_IDS[index % 2] comparison = { "submission_id": str(uuid4()), "drift_report_id": str(uuid4()), "baseline_submission_id": str(uuid4()), "baseline_batch_record_id": str(uuid4()), "candidate_submission_id": str(uuid4()), "candidate_batch_record_id": str(uuid4()), "cohort_id": COHORT_ID, "baseline": baseline, "candidate": candidate, "evidence": [ { "evidence_event_id": attempt_id, "learner_id": LEARNER_ID, "pointer": _pointer(attempt_id), } for attempt_id in ATTEMPT_IDS ], } comparison_path = "/internal/supervision-research/evaluation-comparisons" comparison_result = _stable_retry( internal, comparison_path, comparison, headers=token_headers, id_field="drift_report_id", ) if comparison_result.get("status") != "drift_flagged": raise SmokeError("version comparison did not retain the expected drift signal") changed_comparison = copy.deepcopy(comparison) changed_comparison["candidate"]["model"] = "changed-model" internal.request( "POST", comparison_path, changed_comparison, expected={409}, headers=token_headers, ) manifest = { "submission_id": str(uuid4()), "manifest_id": str(uuid4()), "cohort_id": COHORT_ID, "artifacts": pack["phase3_artifacts"], "sources": [ { "domain": artifact["domain"], "learner_id": LEARNER_ID, "pointer": _pointer(ATTEMPT_IDS[index % 2]), } for index, artifact in enumerate(pack["phase3_artifacts"]) ], } manifest_result = _stable_retry( internal, "/internal/supervision-research/phase3-manifests", manifest, headers=token_headers, id_field="manifest_id", ) if manifest_result.get("artifact_count") != 4: raise SmokeError("Phase 3 manifest omitted one of four evidence domains") internal_supervisor = internal.request( "GET", "/internal/supervision-research/supervisor-view", headers=token_headers, ).body internal_research = internal.request( "GET", "/internal/supervision-research/research-view", headers=token_headers, ).body teacher_supervisor = teacher.request( "GET", "/supervision-research/supervision-view" ).body teacher_research = teacher.request( "GET", "/supervision-research/research-view" ).body learner.request("GET", "/supervision-research/supervision-view", expected={403}) learner.request("GET", "/supervision-research/research-view", expected={403}) other_supervisor = other_teacher.request( "GET", "/supervision-research/supervision-view" ).body other_research = other_teacher.request( "GET", "/supervision-research/research-view" ).body if not teacher_supervisor.get("attention_items") or not teacher_supervisor.get( "curriculum_gaps" ): raise SmokeError("same-cohort teacher did not receive supervision ledgers") if not teacher_research.get("calibration_dataset") or not teacher_research.get( "drift_reports" ) or not teacher_research.get("phase3_manifests"): raise SmokeError("same-cohort teacher did not receive research ledgers") target_dataset = next( ( item for item in teacher_research["calibration_dataset"] if str(item.get("disagreement_record_id")) == disagreement_result["disagreement_record_id"] ), None, ) if not target_dataset or not all( target_dataset.get(key) for key in ("ai_model", "prompt_version", "instrument_id", "instrument_version") ): raise SmokeError("calibration dataset omitted version metadata") target_drift = next( ( item for item in teacher_research["drift_reports"] if str(item.get("drift_report_id")) == comparison_result["drift_report_id"] ), None, ) if not target_drift or not target_drift.get("baseline_model") or not target_drift.get( "candidate_model" ) or not target_drift.get("subgroup_metrics"): raise SmokeError("drift report omitted model versions or subgroup metrics") target_manifest = next( ( item for item in teacher_research["phase3_manifests"] if str(item.get("manifest_id")) == manifest_result["manifest_id"] ), None, ) if not target_manifest or len(target_manifest.get("artifacts") or []) != 4: raise SmokeError("Phase 3 read model omitted four-domain artifact provenance") if any(other_supervisor.values()) or any(other_research.values()): raise SmokeError("cross-cohort teacher received G6 ledger rows") for payload in ( internal_supervisor, internal_research, teacher_supervisor, teacher_research, ): _assert_safe_payload(payload) target_item = next( ( item for item in teacher_supervisor.get("attention_items", []) if str(item.get("snapshot_id")) == attention_result["snapshot_id"] ), None, ) if not target_item or len(target_item.get("drilldown_routes") or []) > 3: raise SmokeError("attention queue did not preserve the <=3 drilldown contract") return { "ok": True, "api_base_url": args.api_base_url, "fixture_policy": "retained unique dev:e2e identities; no fixture deletion", "source_learner_id": LEARNER_ID, "source_session_id": SESSION_ID, "source_attempt_ids": list(ATTEMPT_IDS), "attention_snapshot_id": attention_result["snapshot_id"], "curriculum_gap_snapshot_id": gap_result["gap_snapshot_id"], "disagreement_record_id": disagreement_result["disagreement_record_id"], "drift_report_id": comparison_result["drift_report_id"], "phase3_manifest_id": manifest_result["manifest_id"], "proof": { "missing_internal_token_rejected": True, "wrong_internal_token_rejected": True, "attention_retry_stable": True, "changed_attention_rejected": True, "attention_drilldowns_lte_3": True, "curriculum_gap_retry_stable": True, "teacher_disagreement_metadata_only": True, "disagreement_retry_stable": True, "changed_disagreement_rejected": True, "versioned_drift_flagged": True, "comparison_retry_stable": True, "changed_comparison_rejected": True, "phase3_four_domains": True, "manifest_retry_stable": True, "learner_role_rejected": True, "cross_cohort_teacher_empty": True, "supervisor_research_ai_views_separated": True, "version_metadata_hydrated": True, "subgroup_metrics_hydrated": True, "phase3_artifact_provenance_hydrated": True, "raw_transcript_included": False, "clinical_claim_allowed": False, "no_aggregate_score": True, }, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--api-base-url", default="http://127.0.0.1:8011") parser.add_argument("--internal-token", required=True) parser.add_argument("--request-timeout", type=float, default=180.0) parser.add_argument( "--benchmark-path", default="apps/api/app/data/supervision_research_benchmark_g6.v1.json", ) parser.add_argument("--out", default="") args = parser.parse_args() result = run(args) text = json.dumps(result, ensure_ascii=False, indent=2) if args.out: path = Path(args.out) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text + "\n", encoding="utf-8") print(text) if __name__ == "__main__": main()