G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
525
scripts/smoke-supervision-research-api.py
Normal file
525
scripts/smoke-supervision-research-api.py
Normal file
|
|
@ -0,0 +1,525 @@
|
|||
"""Exercise the G6 supervision/research ledgers over live HTTP and PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
COHORT_ID = "e2e-hanshin"
|
||||
OTHER_COHORT_ID = "e2e-other-cohort"
|
||||
LEARNER_ID = "fa05a889-b546-4e3b-a959-a8efc32f1073"
|
||||
SESSION_ID = "3c834ffd-af5b-4a37-8fd3-ad0690deb165"
|
||||
ATTEMPT_IDS = (
|
||||
"16a76b48-00ab-4a5f-8c12-e9ae2489db38",
|
||||
"9fd642a9-6476-4d3e-8e3f-c35a1f75f1bf",
|
||||
)
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApiResponse:
|
||||
status: int
|
||||
body: Any
|
||||
|
||||
|
||||
class ApiClient:
|
||||
def __init__(self, base_url: str, timeout: float) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(CookieJar())
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
*,
|
||||
expected: set[int] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> ApiResponse:
|
||||
data = None
|
||||
request_headers = {"Accept": "application/json", **(headers or {})}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=data,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
result = ApiResponse(response.status, json.loads(raw) if raw else {})
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
body = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
body = {"detail": raw[:500]}
|
||||
result = ApiResponse(exc.code, body)
|
||||
except urllib.error.URLError as exc:
|
||||
raise SmokeError(
|
||||
f"{method} {path} transport failed: {type(exc.reason).__name__}"
|
||||
) from exc
|
||||
if result.status not in (expected or {200}):
|
||||
detail = result.body.get("detail") if isinstance(result.body, dict) else None
|
||||
raise SmokeError(
|
||||
f"{method} {path} returned HTTP {result.status}; detail={detail!r}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _sign_in(
|
||||
client: ApiClient,
|
||||
*,
|
||||
suffix: str,
|
||||
identity: str,
|
||||
role: str,
|
||||
cohort_ids: list[str],
|
||||
) -> str:
|
||||
client.request(
|
||||
"POST",
|
||||
"/auth/dev-login",
|
||||
{
|
||||
"email": f"dev.e2e.supervision.{identity}.{suffix}@hs.ac.kr",
|
||||
"role": role,
|
||||
"display_name": f"Supervision {identity.title()}",
|
||||
"cohort_ids": cohort_ids,
|
||||
},
|
||||
)
|
||||
client.request(
|
||||
"POST",
|
||||
"/users/me/onboarding",
|
||||
{
|
||||
"legal_name": f"Supervision {identity.title()}",
|
||||
"affiliation": "한신대학교",
|
||||
"department": "상담심리학과",
|
||||
"grade_level": "통합검증",
|
||||
"phone": "010-0000-0000",
|
||||
"contact_address": "경기도 오산시 한신대학교",
|
||||
"nickname": f"Supervision {identity.title()}",
|
||||
"self_introduction": "G6 감독·연구 API 검증 fixture입니다.",
|
||||
"avatar_url": "",
|
||||
"terms_accepted": True,
|
||||
"privacy_accepted": True,
|
||||
},
|
||||
)
|
||||
user_id = str(client.request("GET", "/auth/me").body.get("user_id") or "")
|
||||
if not user_id:
|
||||
raise SmokeError(f"dev-login omitted user_id for {identity}")
|
||||
return user_id
|
||||
|
||||
|
||||
def _pointer(attempt_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"ledger": "practice_attempt",
|
||||
"event_id": attempt_id,
|
||||
"session_id": SESSION_ID,
|
||||
"route_hint": f"/practice/attempts/{attempt_id}",
|
||||
}
|
||||
|
||||
|
||||
def _assert_safe_payload(value: Any, path: str = "response") -> None:
|
||||
if isinstance(value, dict):
|
||||
forbidden = {
|
||||
"total",
|
||||
"total_score",
|
||||
"overall_score",
|
||||
"global_score",
|
||||
"raw_transcript",
|
||||
"transcript",
|
||||
"utterance_text",
|
||||
} & set(value)
|
||||
if forbidden:
|
||||
raise SmokeError(f"{path} exposed forbidden keys: {forbidden}")
|
||||
if value.get("clinical_claim_allowed") is True:
|
||||
raise SmokeError(f"{path} enabled a clinical claim")
|
||||
if value.get("raw_transcript_included") is True:
|
||||
raise SmokeError(f"{path} included raw transcript data")
|
||||
for key, child in value.items():
|
||||
_assert_safe_payload(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_assert_safe_payload(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def _stable_retry(
|
||||
client: ApiClient,
|
||||
path: str,
|
||||
body: dict[str, Any],
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
id_field: str,
|
||||
) -> dict[str, Any]:
|
||||
created = client.request("POST", path, body, expected={201}, headers=headers)
|
||||
retried = client.request("POST", path, body, expected={201}, headers=headers)
|
||||
if created.body.get(id_field) != retried.body.get(id_field) or retried.body.get(
|
||||
"idempotent_replay"
|
||||
) is not True:
|
||||
raise SmokeError(f"same submission retry was not stable for {path}")
|
||||
return created.body
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
if len(args.internal_token) < 32:
|
||||
raise SmokeError("--internal-token must contain at least 32 characters")
|
||||
root = ApiClient(args.api_base_url, args.request_timeout)
|
||||
health = root.request("GET", "/health")
|
||||
if not health.body.get("db") or not health.body.get("engine"):
|
||||
raise SmokeError("API health is not DB+engine ready")
|
||||
|
||||
pack = json.loads(Path(args.benchmark_path).read_text(encoding="utf-8"))
|
||||
suffix = f"{int(time.time())}.{secrets.token_hex(3)}"
|
||||
teacher = ApiClient(args.api_base_url, args.request_timeout)
|
||||
learner = ApiClient(args.api_base_url, args.request_timeout)
|
||||
other_teacher = ApiClient(args.api_base_url, args.request_timeout)
|
||||
_sign_in(
|
||||
teacher,
|
||||
suffix=suffix,
|
||||
identity="teacher",
|
||||
role="teacher",
|
||||
cohort_ids=[COHORT_ID],
|
||||
)
|
||||
_sign_in(
|
||||
learner,
|
||||
suffix=suffix,
|
||||
identity="learner",
|
||||
role="learner",
|
||||
cohort_ids=[COHORT_ID],
|
||||
)
|
||||
_sign_in(
|
||||
other_teacher,
|
||||
suffix=suffix,
|
||||
identity="other-teacher",
|
||||
role="teacher",
|
||||
cohort_ids=[OTHER_COHORT_ID],
|
||||
)
|
||||
|
||||
internal = ApiClient(args.api_base_url, args.request_timeout)
|
||||
token_headers = {
|
||||
"X-Vignette-Supervision-Research-Token": args.internal_token
|
||||
}
|
||||
internal.request(
|
||||
"GET", "/internal/supervision-research/supervisor-view", expected={401}
|
||||
)
|
||||
internal.request(
|
||||
"GET",
|
||||
"/internal/supervision-research/research-view",
|
||||
expected={403},
|
||||
headers={"X-Vignette-Supervision-Research-Token": "wrong-token"},
|
||||
)
|
||||
|
||||
live_suffix = secrets.token_hex(5)
|
||||
signal = copy.deepcopy(pack["attention_signals"][4])
|
||||
signal["signal_id"] = f"oas-g6-signal-live-{live_suffix}"
|
||||
signal["learner_ref"] = "learner-live"
|
||||
signal["evidence"] = [_pointer(ATTEMPT_IDS[0])]
|
||||
attention = {
|
||||
"submission_id": str(uuid4()),
|
||||
"snapshot_id": str(uuid4()),
|
||||
"cohort_id": COHORT_ID,
|
||||
"signals": [signal],
|
||||
"learners": [{"learner_ref": "learner-live", "learner_id": LEARNER_ID}],
|
||||
}
|
||||
attention_path = "/internal/supervision-research/attention-snapshots"
|
||||
attention_result = _stable_retry(
|
||||
internal,
|
||||
attention_path,
|
||||
attention,
|
||||
headers=token_headers,
|
||||
id_field="snapshot_id",
|
||||
)
|
||||
changed_attention = copy.deepcopy(attention)
|
||||
changed_attention["signals"][0]["uncertainty"] = 0.31
|
||||
internal.request(
|
||||
"POST",
|
||||
attention_path,
|
||||
changed_attention,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
gap = {
|
||||
"submission_id": str(uuid4()),
|
||||
"gap_snapshot_id": str(uuid4()),
|
||||
"cohort_id": COHORT_ID,
|
||||
"competency_id": "competency.empathy.reflection",
|
||||
"gap_kind": "growth_stagnation",
|
||||
"status": "observed",
|
||||
"uncertainty": 0.25,
|
||||
"affected_learner_count": 1,
|
||||
"evidence": [{"learner_id": LEARNER_ID, "pointer": _pointer(ATTEMPT_IDS[0])}],
|
||||
}
|
||||
gap_result = _stable_retry(
|
||||
internal,
|
||||
"/internal/supervision-research/curriculum-gaps",
|
||||
gap,
|
||||
headers=token_headers,
|
||||
id_field="gap_snapshot_id",
|
||||
)
|
||||
|
||||
disagreement = copy.deepcopy(pack["disagreements"][0])
|
||||
disagreement["disagreement_id"] = f"oas-g6-disagreement-live-{live_suffix}"
|
||||
disagreement["ai_evidence"] = [_pointer(ATTEMPT_IDS[0])]
|
||||
disagreement["teacher_correction_evidence"] = [_pointer(ATTEMPT_IDS[1])]
|
||||
disagreement_body = {
|
||||
"submission_id": str(uuid4()),
|
||||
"disagreement_record_id": str(uuid4()),
|
||||
"dataset_row_id": str(uuid4()),
|
||||
"audit_event_id": str(uuid4()),
|
||||
"learner_id": LEARNER_ID,
|
||||
"cohort_id": COHORT_ID,
|
||||
"disagreement": disagreement,
|
||||
}
|
||||
disagreement_path = "/supervision-research/teacher-disagreements"
|
||||
disagreement_result = _stable_retry(
|
||||
teacher,
|
||||
disagreement_path,
|
||||
disagreement_body,
|
||||
id_field="disagreement_record_id",
|
||||
)
|
||||
changed_disagreement = copy.deepcopy(disagreement_body)
|
||||
changed_disagreement["disagreement"]["correction_reason_code"] = (
|
||||
"changed_content_must_conflict"
|
||||
)
|
||||
teacher.request("POST", disagreement_path, changed_disagreement, expected={409})
|
||||
|
||||
baseline = copy.deepcopy(pack["baseline_batch"])
|
||||
candidate = copy.deepcopy(pack["candidate_batch"])
|
||||
baseline["batch_id"] = f"oas-g6-batch-baseline-{live_suffix}"
|
||||
candidate["batch_id"] = f"oas-g6-batch-candidate-{live_suffix}"
|
||||
for index, item in enumerate(baseline["observations"]):
|
||||
item["evidence_event_id"] = ATTEMPT_IDS[index % 2]
|
||||
for index, item in enumerate(candidate["observations"]):
|
||||
item["evidence_event_id"] = ATTEMPT_IDS[index % 2]
|
||||
comparison = {
|
||||
"submission_id": str(uuid4()),
|
||||
"drift_report_id": str(uuid4()),
|
||||
"baseline_submission_id": str(uuid4()),
|
||||
"baseline_batch_record_id": str(uuid4()),
|
||||
"candidate_submission_id": str(uuid4()),
|
||||
"candidate_batch_record_id": str(uuid4()),
|
||||
"cohort_id": COHORT_ID,
|
||||
"baseline": baseline,
|
||||
"candidate": candidate,
|
||||
"evidence": [
|
||||
{
|
||||
"evidence_event_id": attempt_id,
|
||||
"learner_id": LEARNER_ID,
|
||||
"pointer": _pointer(attempt_id),
|
||||
}
|
||||
for attempt_id in ATTEMPT_IDS
|
||||
],
|
||||
}
|
||||
comparison_path = "/internal/supervision-research/evaluation-comparisons"
|
||||
comparison_result = _stable_retry(
|
||||
internal,
|
||||
comparison_path,
|
||||
comparison,
|
||||
headers=token_headers,
|
||||
id_field="drift_report_id",
|
||||
)
|
||||
if comparison_result.get("status") != "drift_flagged":
|
||||
raise SmokeError("version comparison did not retain the expected drift signal")
|
||||
changed_comparison = copy.deepcopy(comparison)
|
||||
changed_comparison["candidate"]["model"] = "changed-model"
|
||||
internal.request(
|
||||
"POST",
|
||||
comparison_path,
|
||||
changed_comparison,
|
||||
expected={409},
|
||||
headers=token_headers,
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"submission_id": str(uuid4()),
|
||||
"manifest_id": str(uuid4()),
|
||||
"cohort_id": COHORT_ID,
|
||||
"artifacts": pack["phase3_artifacts"],
|
||||
"sources": [
|
||||
{
|
||||
"domain": artifact["domain"],
|
||||
"learner_id": LEARNER_ID,
|
||||
"pointer": _pointer(ATTEMPT_IDS[index % 2]),
|
||||
}
|
||||
for index, artifact in enumerate(pack["phase3_artifacts"])
|
||||
],
|
||||
}
|
||||
manifest_result = _stable_retry(
|
||||
internal,
|
||||
"/internal/supervision-research/phase3-manifests",
|
||||
manifest,
|
||||
headers=token_headers,
|
||||
id_field="manifest_id",
|
||||
)
|
||||
if manifest_result.get("artifact_count") != 4:
|
||||
raise SmokeError("Phase 3 manifest omitted one of four evidence domains")
|
||||
|
||||
internal_supervisor = internal.request(
|
||||
"GET",
|
||||
"/internal/supervision-research/supervisor-view",
|
||||
headers=token_headers,
|
||||
).body
|
||||
internal_research = internal.request(
|
||||
"GET",
|
||||
"/internal/supervision-research/research-view",
|
||||
headers=token_headers,
|
||||
).body
|
||||
teacher_supervisor = teacher.request(
|
||||
"GET", "/supervision-research/supervision-view"
|
||||
).body
|
||||
teacher_research = teacher.request(
|
||||
"GET", "/supervision-research/research-view"
|
||||
).body
|
||||
learner.request("GET", "/supervision-research/supervision-view", expected={403})
|
||||
learner.request("GET", "/supervision-research/research-view", expected={403})
|
||||
other_supervisor = other_teacher.request(
|
||||
"GET", "/supervision-research/supervision-view"
|
||||
).body
|
||||
other_research = other_teacher.request(
|
||||
"GET", "/supervision-research/research-view"
|
||||
).body
|
||||
if not teacher_supervisor.get("attention_items") or not teacher_supervisor.get(
|
||||
"curriculum_gaps"
|
||||
):
|
||||
raise SmokeError("same-cohort teacher did not receive supervision ledgers")
|
||||
if not teacher_research.get("calibration_dataset") or not teacher_research.get(
|
||||
"drift_reports"
|
||||
) or not teacher_research.get("phase3_manifests"):
|
||||
raise SmokeError("same-cohort teacher did not receive research ledgers")
|
||||
target_dataset = next(
|
||||
(
|
||||
item
|
||||
for item in teacher_research["calibration_dataset"]
|
||||
if str(item.get("disagreement_record_id"))
|
||||
== disagreement_result["disagreement_record_id"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_dataset or not all(
|
||||
target_dataset.get(key)
|
||||
for key in ("ai_model", "prompt_version", "instrument_id", "instrument_version")
|
||||
):
|
||||
raise SmokeError("calibration dataset omitted version metadata")
|
||||
target_drift = next(
|
||||
(
|
||||
item
|
||||
for item in teacher_research["drift_reports"]
|
||||
if str(item.get("drift_report_id")) == comparison_result["drift_report_id"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_drift or not target_drift.get("baseline_model") or not target_drift.get(
|
||||
"candidate_model"
|
||||
) or not target_drift.get("subgroup_metrics"):
|
||||
raise SmokeError("drift report omitted model versions or subgroup metrics")
|
||||
target_manifest = next(
|
||||
(
|
||||
item
|
||||
for item in teacher_research["phase3_manifests"]
|
||||
if str(item.get("manifest_id")) == manifest_result["manifest_id"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_manifest or len(target_manifest.get("artifacts") or []) != 4:
|
||||
raise SmokeError("Phase 3 read model omitted four-domain artifact provenance")
|
||||
if any(other_supervisor.values()) or any(other_research.values()):
|
||||
raise SmokeError("cross-cohort teacher received G6 ledger rows")
|
||||
for payload in (
|
||||
internal_supervisor,
|
||||
internal_research,
|
||||
teacher_supervisor,
|
||||
teacher_research,
|
||||
):
|
||||
_assert_safe_payload(payload)
|
||||
target_item = next(
|
||||
(
|
||||
item
|
||||
for item in teacher_supervisor.get("attention_items", [])
|
||||
if str(item.get("snapshot_id")) == attention_result["snapshot_id"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_item or len(target_item.get("drilldown_routes") or []) > 3:
|
||||
raise SmokeError("attention queue did not preserve the <=3 drilldown contract")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"api_base_url": args.api_base_url,
|
||||
"fixture_policy": "retained unique dev:e2e identities; no fixture deletion",
|
||||
"source_learner_id": LEARNER_ID,
|
||||
"source_session_id": SESSION_ID,
|
||||
"source_attempt_ids": list(ATTEMPT_IDS),
|
||||
"attention_snapshot_id": attention_result["snapshot_id"],
|
||||
"curriculum_gap_snapshot_id": gap_result["gap_snapshot_id"],
|
||||
"disagreement_record_id": disagreement_result["disagreement_record_id"],
|
||||
"drift_report_id": comparison_result["drift_report_id"],
|
||||
"phase3_manifest_id": manifest_result["manifest_id"],
|
||||
"proof": {
|
||||
"missing_internal_token_rejected": True,
|
||||
"wrong_internal_token_rejected": True,
|
||||
"attention_retry_stable": True,
|
||||
"changed_attention_rejected": True,
|
||||
"attention_drilldowns_lte_3": True,
|
||||
"curriculum_gap_retry_stable": True,
|
||||
"teacher_disagreement_metadata_only": True,
|
||||
"disagreement_retry_stable": True,
|
||||
"changed_disagreement_rejected": True,
|
||||
"versioned_drift_flagged": True,
|
||||
"comparison_retry_stable": True,
|
||||
"changed_comparison_rejected": True,
|
||||
"phase3_four_domains": True,
|
||||
"manifest_retry_stable": True,
|
||||
"learner_role_rejected": True,
|
||||
"cross_cohort_teacher_empty": True,
|
||||
"supervisor_research_ai_views_separated": True,
|
||||
"version_metadata_hydrated": True,
|
||||
"subgroup_metrics_hydrated": True,
|
||||
"phase3_artifact_provenance_hydrated": True,
|
||||
"raw_transcript_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
"no_aggregate_score": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--api-base-url", default="http://127.0.0.1:8011")
|
||||
parser.add_argument("--internal-token", required=True)
|
||||
parser.add_argument("--request-timeout", type=float, default=180.0)
|
||||
parser.add_argument(
|
||||
"--benchmark-path",
|
||||
default="apps/api/app/data/supervision_research_benchmark_g6.v1.json",
|
||||
)
|
||||
parser.add_argument("--out", default="")
|
||||
args = parser.parse_args()
|
||||
result = run(args)
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.out:
|
||||
path = Path(args.out)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue