G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
976
scripts/smoke-calibration-transfer-api.py
Normal file
976
scripts/smoke-calibration-transfer-api.py
Normal file
|
|
@ -0,0 +1,976 @@
|
|||
"""Exercise the G5 self-prediction lock and reveal boundary over live HTTP/DB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
COHORT_ID = "e2e-hanshin"
|
||||
INSTRUMENT_ID = "calibration-mirror-g5"
|
||||
INSTRUMENT_VERSION = "1.0.0"
|
||||
TRANSFER_INSTRUMENT_ID = "unseen-transfer-g5"
|
||||
API_ENV = Path(__file__).resolve().parents[1] / "apps" / "api" / ".env"
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApiResponse:
|
||||
status: int
|
||||
body: Any
|
||||
|
||||
|
||||
class ApiClient:
|
||||
def __init__(self, base_url: str, timeout: float) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(CookieJar())
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
*,
|
||||
expected: set[int] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> ApiResponse:
|
||||
data = None
|
||||
request_headers = {"Accept": "application/json", **(headers or {})}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=data,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
result = ApiResponse(response.status, json.loads(raw) if raw else {})
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
body = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
body = {"detail": raw[:500]}
|
||||
result = ApiResponse(exc.code, body)
|
||||
except urllib.error.URLError as exc:
|
||||
raise SmokeError(
|
||||
f"{method} {path} transport failed: {type(exc.reason).__name__}"
|
||||
) from exc
|
||||
if result.status not in (expected or {200}):
|
||||
detail = result.body.get("detail") if isinstance(result.body, dict) else None
|
||||
raise SmokeError(
|
||||
f"{method} {path} returned HTTP {result.status}; detail={detail!r}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _sign_in(
|
||||
client: ApiClient,
|
||||
*,
|
||||
suffix: str,
|
||||
identity: str,
|
||||
role: str,
|
||||
cohort_ids: list[str],
|
||||
) -> str:
|
||||
client.request(
|
||||
"POST",
|
||||
"/auth/dev-login",
|
||||
{
|
||||
"email": f"dev.e2e.calibration.{identity}.{suffix}@hs.ac.kr",
|
||||
"role": role,
|
||||
"display_name": f"Calibration {identity.title()}",
|
||||
"cohort_ids": cohort_ids,
|
||||
},
|
||||
)
|
||||
client.request(
|
||||
"POST",
|
||||
"/users/me/onboarding",
|
||||
{
|
||||
"legal_name": f"Calibration {identity.title()}",
|
||||
"affiliation": "한신대학교",
|
||||
"department": "상담심리학과",
|
||||
"grade_level": "통합검증",
|
||||
"phone": "010-0000-0000",
|
||||
"contact_address": "경기도 오산시 한신대학교",
|
||||
"nickname": f"Calibration {identity.title()}",
|
||||
"self_introduction": "G5 자기보정·전이 API 검증 fixture입니다.",
|
||||
"avatar_url": "",
|
||||
"terms_accepted": True,
|
||||
"privacy_accepted": True,
|
||||
},
|
||||
)
|
||||
me = client.request("GET", "/auth/me")
|
||||
user_id = str(me.body.get("user_id") or "")
|
||||
if not user_id:
|
||||
raise SmokeError(f"dev-login omitted user_id for {identity}")
|
||||
return user_id
|
||||
|
||||
|
||||
def _choose_distinct_personas(client: ApiClient) -> tuple[str, str]:
|
||||
response = client.request("GET", "/personas")
|
||||
usable = [
|
||||
item
|
||||
for item in response.body
|
||||
if isinstance(item, dict)
|
||||
and item.get("source") == "database"
|
||||
and not item.get("degraded")
|
||||
and item.get("code")
|
||||
]
|
||||
codes = list(dict.fromkeys(str(item["code"]) for item in usable))
|
||||
if len(codes) < 2:
|
||||
raise SmokeError(
|
||||
"persona catalog requires two distinct non-degraded database personas"
|
||||
)
|
||||
source = "P1" if "P1" in codes else codes[0]
|
||||
practice = next(code for code in codes if code != source)
|
||||
return source, practice
|
||||
|
||||
|
||||
def _load_api_env() -> None:
|
||||
if not API_ENV.exists():
|
||||
return
|
||||
for raw_line in API_ENV.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def _wait_for_session_review(
|
||||
client: ApiClient,
|
||||
session_id: str,
|
||||
*,
|
||||
timeout: float,
|
||||
interval: float,
|
||||
) -> dict[str, Any]:
|
||||
if timeout <= 0 or interval <= 0:
|
||||
raise SmokeError("review poll timeout and interval must be positive")
|
||||
deadline = time.monotonic() + timeout
|
||||
poll_count = 0
|
||||
while True:
|
||||
poll_count += 1
|
||||
detail = client.request("GET", f"/sessions/{session_id}")
|
||||
if detail.body.get("review_ready") is True:
|
||||
review = client.request("GET", f"/sessions/{session_id}/review")
|
||||
if review.body.get("reviewReady") is not True:
|
||||
raise SmokeError(
|
||||
"session detail was review-ready but review payload was not ready"
|
||||
)
|
||||
return {"poll_count": poll_count, "review": review.body}
|
||||
now = time.monotonic()
|
||||
if now >= deadline:
|
||||
raise SmokeError(
|
||||
f"session_end evaluator did not become ready within {timeout:.1f}s "
|
||||
f"for session {session_id}"
|
||||
)
|
||||
time.sleep(min(interval, deadline - now))
|
||||
|
||||
|
||||
def _durable_turn_ids(review: dict[str, Any]) -> list[str]:
|
||||
ids = [str(item["turn_id"]) for item in review.get("turns", []) if item.get("turn_id")]
|
||||
if len(ids) < 2:
|
||||
raise SmokeError("session review did not expose both durable turn UUIDs")
|
||||
return ids[:2]
|
||||
|
||||
|
||||
def _assert_no_aggregate_score(value: Any, path: str = "response") -> None:
|
||||
if isinstance(value, dict):
|
||||
forbidden = {"total", "total_score", "overall_score"} & set(value)
|
||||
if forbidden:
|
||||
raise SmokeError(f"{path} exposed aggregate score keys: {forbidden}")
|
||||
for key, child in value.items():
|
||||
_assert_no_aggregate_score(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_assert_no_aggregate_score(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def _assert_no_raw_transcript(value: Any, path: str = "response") -> None:
|
||||
forbidden = {
|
||||
"raw_transcript",
|
||||
"transcript",
|
||||
"text_masked",
|
||||
"utterance_text",
|
||||
"counselor_text",
|
||||
"client_text",
|
||||
}
|
||||
if isinstance(value, dict):
|
||||
contaminated = forbidden & {str(key).lower() for key in value}
|
||||
if contaminated:
|
||||
raise SmokeError(f"{path} exposed raw-text keys: {contaminated}")
|
||||
for key, child in value.items():
|
||||
_assert_no_raw_transcript(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_assert_no_raw_transcript(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def _actual_execution_request(
|
||||
*, original_transfer_trial_record_id: str, practice_session_id: str
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"original_transfer_trial_record_id": original_transfer_trial_record_id,
|
||||
"practice_session_id": practice_session_id,
|
||||
}
|
||||
|
||||
|
||||
def _build_transfer_suite(
|
||||
*, fixture_suffix: str, evidence_turn_ids: list[str]
|
||||
) -> dict[str, Any]:
|
||||
if not evidence_turn_ids:
|
||||
raise SmokeError("transfer suite requires durable turn UUID evidence")
|
||||
return {
|
||||
"suite_id": f"oas-g5-suite-live-{fixture_suffix}",
|
||||
"training_phrase_family_ids": [f"training-empathy-{fixture_suffix}"],
|
||||
"trials": [
|
||||
{
|
||||
"trial_id": f"oas-g5-transfer-live-{fixture_suffix}",
|
||||
"competency_id": "competency.empathic_attunement",
|
||||
"scenario_variant_id": f"unseen-live-{fixture_suffix}",
|
||||
"variation": {
|
||||
"context_variant": f"academic-transition-{fixture_suffix}",
|
||||
"relationship_style": "withdrawn",
|
||||
"difficulty_level": 3,
|
||||
"expression_variant": "indirect-emotion",
|
||||
"synthetic_subgroup": "synthetic-live-a",
|
||||
"scenario_family_id": "family-academic-transition",
|
||||
"phrase_family_id": f"novel-empathy-{fixture_suffix}",
|
||||
},
|
||||
"status": "passed",
|
||||
"uncertainty": 0.2,
|
||||
"evidence_refs": evidence_turn_ids,
|
||||
"counterevidence": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _actual_execution_read_proof(
|
||||
read_model: dict[str, Any],
|
||||
*,
|
||||
execution_event_id: str,
|
||||
original_transfer_trial_record_id: str,
|
||||
practice_session_id: str,
|
||||
durable_turn_ids: list[str],
|
||||
) -> dict[str, Any]:
|
||||
execution = next(
|
||||
(
|
||||
item
|
||||
for item in read_model.get("actual_executions", [])
|
||||
if str(item.get("execution_event_id")) == execution_event_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if execution is None:
|
||||
raise SmokeError("learner read model omitted actual transfer execution")
|
||||
if (
|
||||
str(execution.get("original_transfer_trial_record_id"))
|
||||
!= original_transfer_trial_record_id
|
||||
or str(execution.get("practice_session_id")) != practice_session_id
|
||||
):
|
||||
raise SmokeError("actual transfer read model changed its server ledger anchors")
|
||||
durable = set(durable_turn_ids)
|
||||
evidence = {str(item) for item in execution.get("evidence_turn_ids") or []}
|
||||
if not evidence or not evidence.issubset(durable):
|
||||
raise SmokeError("actual transfer evidence is not bound to durable practice turns")
|
||||
labels = execution.get("normalized_evaluator_labels") or {}
|
||||
required_labels = {
|
||||
"technique_codes",
|
||||
"client_state_codes",
|
||||
"appropriateness",
|
||||
"intent_deviation_dimensions",
|
||||
"evaluator_error_count",
|
||||
}
|
||||
if not isinstance(labels, dict) or set(labels) != required_labels:
|
||||
raise SmokeError("actual transfer omitted normalized evaluator labels")
|
||||
model_run_id = str(execution.get("model_run_id") or "")
|
||||
if not model_run_id:
|
||||
raise SmokeError("actual transfer omitted evaluator model-run provenance")
|
||||
if (
|
||||
execution.get("source_kind") != "model_inferred"
|
||||
or execution.get("perspective") != "independent_observer"
|
||||
):
|
||||
raise SmokeError("actual transfer omitted fixed source provenance")
|
||||
if (
|
||||
execution.get("instrument_id") != "unseen-transfer-g5"
|
||||
or execution.get("instrument_version") != "1.0.0"
|
||||
or execution.get("observer_version")
|
||||
!= "calibration-actual-transfer-observer-v1"
|
||||
):
|
||||
raise SmokeError("actual transfer omitted fixed G0 instrument provenance")
|
||||
assessment = next(
|
||||
(
|
||||
item
|
||||
for item in read_model.get("actual_transfer_assessments", [])
|
||||
if execution_event_id
|
||||
in {str(value) for value in item.get("source_execution_event_ids") or []}
|
||||
),
|
||||
None,
|
||||
)
|
||||
if assessment is None:
|
||||
raise SmokeError("learner read model omitted actual transfer assessment")
|
||||
if assessment.get("evidence_source") != "actual_practice_execution":
|
||||
raise SmokeError("actual assessment was mixed with synthetic suite evidence")
|
||||
if assessment.get("actual_transfer_status") != "insufficient_evidence":
|
||||
raise SmokeError("one actual execution must remain insufficient evidence")
|
||||
_assert_no_raw_transcript(execution, "actual_execution")
|
||||
_assert_no_raw_transcript(assessment, "actual_transfer_assessment")
|
||||
return {
|
||||
"execution_event_id": execution_event_id,
|
||||
"model_run_id": model_run_id,
|
||||
"source_kind": "model_inferred",
|
||||
"perspective": "independent_observer",
|
||||
"instrument_id": "unseen-transfer-g5",
|
||||
"instrument_version": "1.0.0",
|
||||
"observer_version": "calibration-actual-transfer-observer-v1",
|
||||
"durable_evidence_turn_ids": sorted(evidence),
|
||||
"actual_transfer_status": "insufficient_evidence",
|
||||
"evidence_source": "actual_practice_execution",
|
||||
}
|
||||
|
||||
|
||||
def _json_object(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
raise SmokeError("Postgres JSON evidence was not an object")
|
||||
|
||||
|
||||
async def _create_transfer_suite_model_run(
|
||||
dsn: str,
|
||||
*,
|
||||
learner_id: str,
|
||||
source_session_id: str,
|
||||
evidence_turn_ids: list[str],
|
||||
) -> str:
|
||||
import asyncpg
|
||||
|
||||
model_run_id = str(uuid4())
|
||||
input_payload = {
|
||||
"source": "calibration_transfer_http_smoke",
|
||||
"session_id": source_session_id,
|
||||
"evidence_turn_ids": evidence_turn_ids,
|
||||
}
|
||||
canonical = json.dumps(input_payload, separators=(",", ":"), sort_keys=True)
|
||||
input_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
bundle_hash = hashlib.sha256(
|
||||
b"calibration-transfer-http-smoke-v1"
|
||||
).hexdigest()
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT set_config('app.ai_context', 'true', true)")
|
||||
await conn.execute(
|
||||
"SELECT set_config('app.current_ai_view', 'evaluator', true)"
|
||||
)
|
||||
await conn.execute("SELECT set_config('app.current_role', 'admin', true)")
|
||||
await conn.execute(
|
||||
"SELECT set_config('app.current_uid', $1, true)", learner_id
|
||||
)
|
||||
await conn.execute(
|
||||
"SELECT set_config('app.current_cohort', $1, true)", COHORT_ID
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO audit.model_run (
|
||||
model_run_id, session_id, turn_id, agent_role, provider, model,
|
||||
prompt_bundle_id, prompt_bundle_version, prompt_bundle_hash,
|
||||
structured_schema_version, input_evidence_hash, status, metadata
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,$3::uuid,'evaluator','vignette-smoke',
|
||||
'calibration-transfer-suite-fixture',
|
||||
'calibration-transfer-http-smoke','1.0.0',$4,
|
||||
'vignette.calibration-transfer-suite-smoke.v1',$5,'ready',$6::jsonb
|
||||
)
|
||||
""",
|
||||
model_run_id,
|
||||
source_session_id,
|
||||
evidence_turn_ids[0],
|
||||
bundle_hash,
|
||||
input_hash,
|
||||
json.dumps(input_payload, ensure_ascii=False),
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
return model_run_id
|
||||
|
||||
|
||||
async def _fetch_actual_transfer_db_proof(
|
||||
dsn: str,
|
||||
*,
|
||||
learner_id: str,
|
||||
execution_event_id: str,
|
||||
original_transfer_trial_record_id: str,
|
||||
practice_session_id: str,
|
||||
durable_turn_ids: list[str],
|
||||
model_run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
import asyncpg
|
||||
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT set_config('app.ai_context', 'true', true)")
|
||||
await conn.execute(
|
||||
"SELECT set_config('app.current_ai_view', 'evaluator', true)"
|
||||
)
|
||||
await conn.execute("SELECT set_config('app.current_role', 'admin', true)")
|
||||
await conn.execute(
|
||||
"SELECT set_config('app.current_uid', $1, true)", learner_id
|
||||
)
|
||||
event = await conn.fetchrow(
|
||||
"""
|
||||
SELECT execution_event_id::text,
|
||||
original_transfer_trial_record_id::text,
|
||||
practice_session_id::text, learner_id::text,
|
||||
evidence_turn_ids, normalized_evaluator_labels,
|
||||
model_run_id::text, source_kind, perspective,
|
||||
instrument_id, instrument_version, observer_version
|
||||
FROM app.calibration_transfer_execution_event
|
||||
WHERE execution_event_id = $1::uuid
|
||||
""",
|
||||
execution_event_id,
|
||||
)
|
||||
model_run = await conn.fetchrow(
|
||||
"""
|
||||
SELECT model_run_id::text, session_id::text, agent_role,
|
||||
provider, model, prompt_bundle_id, prompt_bundle_version,
|
||||
structured_schema_version, input_evidence_hash, status, metadata
|
||||
FROM audit.model_run
|
||||
WHERE model_run_id = $1::uuid
|
||||
""",
|
||||
model_run_id,
|
||||
)
|
||||
columns = {
|
||||
str(row["column_name"])
|
||||
for row in await conn.fetch(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'app'
|
||||
AND table_name = 'calibration_transfer_execution_event'
|
||||
"""
|
||||
)
|
||||
}
|
||||
finally:
|
||||
await conn.close()
|
||||
if event is None or model_run is None:
|
||||
raise SmokeError("Postgres omitted actual transfer event or model run")
|
||||
if (
|
||||
str(event["original_transfer_trial_record_id"])
|
||||
!= original_transfer_trial_record_id
|
||||
or str(event["practice_session_id"]) != practice_session_id
|
||||
or str(event["learner_id"]) != learner_id
|
||||
or str(event["model_run_id"]) != model_run_id
|
||||
or event["source_kind"] != "model_inferred"
|
||||
or event["perspective"] != "independent_observer"
|
||||
or event["instrument_id"] != "unseen-transfer-g5"
|
||||
or event["instrument_version"] != "1.0.0"
|
||||
or event["observer_version"]
|
||||
!= "calibration-actual-transfer-observer-v1"
|
||||
):
|
||||
raise SmokeError("Postgres actual transfer anchors or provenance differ")
|
||||
evidence = {str(item) for item in event["evidence_turn_ids"] or []}
|
||||
if not evidence or not evidence.issubset(set(durable_turn_ids)):
|
||||
raise SmokeError("Postgres actual evidence is not durable-turn bound")
|
||||
labels = _json_object(event["normalized_evaluator_labels"])
|
||||
metadata = _json_object(model_run["metadata"])
|
||||
_assert_no_raw_transcript(labels, "postgres.normalized_evaluator_labels")
|
||||
_assert_no_raw_transcript(metadata, "postgres.model_run.metadata")
|
||||
forbidden_columns = {
|
||||
"raw_transcript",
|
||||
"transcript",
|
||||
"text",
|
||||
"text_masked",
|
||||
"utterance_text",
|
||||
}
|
||||
if forbidden_columns & columns:
|
||||
raise SmokeError("actual transfer ledger contains a raw-text column")
|
||||
if (
|
||||
str(model_run["session_id"]) != practice_session_id
|
||||
or model_run["agent_role"] != "evaluator"
|
||||
or model_run["provider"] != "vignette-runtime"
|
||||
or model_run["model"] != "calibration-actual-transfer-observer"
|
||||
or model_run["prompt_bundle_id"] != "calibration-actual-transfer-observer"
|
||||
or model_run["prompt_bundle_version"]
|
||||
!= "calibration-actual-transfer-observer-v1"
|
||||
or model_run["structured_schema_version"]
|
||||
!= "vignette.calibration-actual-transfer-execution.v1"
|
||||
or model_run["status"] != "ready"
|
||||
or len(str(model_run["input_evidence_hash"])) != 64
|
||||
):
|
||||
raise SmokeError("Postgres actual evaluator model-run provenance is incomplete")
|
||||
return {
|
||||
"durable_turn_ids_match_api": True,
|
||||
"model_run_id_matches_api": True,
|
||||
"model_run_schema": "vignette.calibration-actual-transfer-execution.v1",
|
||||
"instrument_id": "unseen-transfer-g5",
|
||||
"instrument_version": "1.0.0",
|
||||
"observer_version": "calibration-actual-transfer-observer-v1",
|
||||
"raw_text_columns": 0,
|
||||
"normalized_evaluator_labels_only": True,
|
||||
}
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
_load_api_env()
|
||||
dsn = args.database_url or os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
raise SmokeError("DATABASE_URL is required via --database-url or apps/api/.env")
|
||||
if len(args.internal_token) < 32:
|
||||
raise SmokeError("--internal-token must contain at least 32 characters")
|
||||
health = ApiClient(args.api_base_url, args.request_timeout).request("GET", "/health")
|
||||
if not health.body.get("db") or not health.body.get("engine"):
|
||||
raise SmokeError("API health is not DB+engine ready")
|
||||
|
||||
suffix = f"{int(time.time())}.{secrets.token_hex(3)}"
|
||||
learner = ApiClient(args.api_base_url, args.request_timeout)
|
||||
teacher = ApiClient(args.api_base_url, args.request_timeout)
|
||||
other_learner = ApiClient(args.api_base_url, args.request_timeout)
|
||||
other_teacher = ApiClient(args.api_base_url, args.request_timeout)
|
||||
learner_id = _sign_in(
|
||||
learner,
|
||||
suffix=suffix,
|
||||
identity="learner",
|
||||
role="learner",
|
||||
cohort_ids=[COHORT_ID],
|
||||
)
|
||||
_sign_in(
|
||||
teacher,
|
||||
suffix=suffix,
|
||||
identity="teacher",
|
||||
role="teacher",
|
||||
cohort_ids=[COHORT_ID],
|
||||
)
|
||||
_sign_in(
|
||||
other_learner,
|
||||
suffix=suffix,
|
||||
identity="other-learner",
|
||||
role="learner",
|
||||
cohort_ids=[COHORT_ID],
|
||||
)
|
||||
_sign_in(
|
||||
other_teacher,
|
||||
suffix=suffix,
|
||||
identity="other-teacher",
|
||||
role="teacher",
|
||||
cohort_ids=["e2e-other-cohort"],
|
||||
)
|
||||
source_persona, practice_persona = _choose_distinct_personas(learner)
|
||||
|
||||
started = learner.request(
|
||||
"POST",
|
||||
"/sessions",
|
||||
{
|
||||
"persona_code": source_persona,
|
||||
"theory_mode": "humanistic",
|
||||
"goal_stages": ["라포", "탐색"],
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
session_id = str(started.body["session_id"])
|
||||
learner.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/turn",
|
||||
{"text": "이번 장면에서 내가 감정을 정확히 반영할 확률을 먼저 예측해볼게요."},
|
||||
)
|
||||
learner.request("POST", f"/sessions/{session_id}/end")
|
||||
source_review = _wait_for_session_review(
|
||||
learner,
|
||||
session_id,
|
||||
timeout=args.review_poll_timeout,
|
||||
interval=args.review_poll_interval,
|
||||
)
|
||||
turn_ids = _durable_turn_ids(source_review["review"])
|
||||
|
||||
history_id = str(uuid4())
|
||||
revision_id = str(uuid4())
|
||||
revision_submission_id = str(uuid4())
|
||||
block_suffix = secrets.token_hex(5)
|
||||
revision = {
|
||||
"submission_id": revision_submission_id,
|
||||
"prediction_revision_id": revision_id,
|
||||
"history_id": history_id,
|
||||
"session_id": session_id,
|
||||
"competency_id": "competency.empathic_attunement",
|
||||
"practice_block_id": f"oas-g5-block-live-{block_suffix}",
|
||||
"scenario_variant_id": f"scenario-live-{block_suffix}",
|
||||
"phrase_family_id": f"phrase-live-{block_suffix}",
|
||||
"revision_no": 1,
|
||||
"supersedes_prediction_revision_id": None,
|
||||
"predicted_success_probability": 0.72,
|
||||
"confidence": 0.80,
|
||||
"recorded_sequence": 1,
|
||||
"revision_reason": "외부평가를 보기 전 장면 근거로 성공 가능성을 예측함",
|
||||
"instrument_id": INSTRUMENT_ID,
|
||||
"instrument_version": INSTRUMENT_VERSION,
|
||||
"evidence_turn_ids": turn_ids,
|
||||
}
|
||||
revision_path = "/calibration/predictions/revisions"
|
||||
created = learner.request("POST", revision_path, revision, expected={201})
|
||||
retried = learner.request("POST", revision_path, revision, expected={201})
|
||||
if created.body.get("prediction_revision_id") != retried.body.get(
|
||||
"prediction_revision_id"
|
||||
) or retried.body.get("idempotent_replay") is not True:
|
||||
raise SmokeError("same prediction revision retry was not stable")
|
||||
learner.request(
|
||||
"POST",
|
||||
revision_path,
|
||||
dict(revision, predicted_success_probability=0.73),
|
||||
expected={409},
|
||||
)
|
||||
|
||||
lock_submission_id = str(uuid4())
|
||||
lock_id = str(uuid4())
|
||||
lock = {
|
||||
"submission_id": lock_submission_id,
|
||||
"lock_id": lock_id,
|
||||
"prediction_revision_id": revision_id,
|
||||
"locked_sequence": 1,
|
||||
}
|
||||
lock_path = f"/calibration/predictions/{history_id}/lock"
|
||||
locked = learner.request("POST", lock_path, lock, expected={201})
|
||||
locked_retry = learner.request("POST", lock_path, lock, expected={201})
|
||||
if locked.body.get("lock_id") != locked_retry.body.get("lock_id") or locked_retry.body.get(
|
||||
"idempotent_replay"
|
||||
) is not True:
|
||||
raise SmokeError("same prediction lock retry was not stable")
|
||||
|
||||
post_lock_revision = dict(
|
||||
revision,
|
||||
submission_id=str(uuid4()),
|
||||
prediction_revision_id=str(uuid4()),
|
||||
revision_no=2,
|
||||
supersedes_prediction_revision_id=revision_id,
|
||||
predicted_success_probability=0.78,
|
||||
recorded_sequence=2,
|
||||
revision_reason="잠금 뒤 오염 시도를 검증함",
|
||||
)
|
||||
learner.request("POST", revision_path, post_lock_revision, expected={422})
|
||||
|
||||
internal = ApiClient(args.api_base_url, args.request_timeout)
|
||||
observation_submission_id = str(uuid4())
|
||||
observation_id = str(uuid4())
|
||||
observation = {
|
||||
"submission_id": observation_submission_id,
|
||||
"observation_id": observation_id,
|
||||
"history_id": history_id,
|
||||
"status": "passed",
|
||||
"source_kind": "observed_runtime",
|
||||
"perspective": "runtime_observation",
|
||||
"model_run_id": None,
|
||||
"instrument_id": INSTRUMENT_ID,
|
||||
"instrument_version": INSTRUMENT_VERSION,
|
||||
"uncertainty": 0.18,
|
||||
"evidence_turn_ids": turn_ids,
|
||||
"counterevidence": ["single_scene_transfer_not_yet_verified"],
|
||||
"revealed_sequence": 2,
|
||||
}
|
||||
observation_path = "/internal/calibration/performance-observations"
|
||||
token_header = {"X-Vignette-Calibration-Transfer-Token": args.internal_token}
|
||||
observed = internal.request(
|
||||
"POST",
|
||||
observation_path,
|
||||
observation,
|
||||
expected={201},
|
||||
headers=token_header,
|
||||
)
|
||||
observed_retry = internal.request(
|
||||
"POST",
|
||||
observation_path,
|
||||
observation,
|
||||
expected={201},
|
||||
headers=token_header,
|
||||
)
|
||||
if observed.body.get("observation_id") != observed_retry.body.get(
|
||||
"observation_id"
|
||||
) or observed_retry.body.get("idempotent_replay") is not True:
|
||||
raise SmokeError("same performance observation retry was not stable")
|
||||
internal.request(
|
||||
"POST",
|
||||
observation_path,
|
||||
dict(observation, uncertainty=0.19),
|
||||
expected={409},
|
||||
headers=token_header,
|
||||
)
|
||||
|
||||
suite_model_run_id = asyncio.run(
|
||||
_create_transfer_suite_model_run(
|
||||
dsn,
|
||||
learner_id=learner_id,
|
||||
source_session_id=session_id,
|
||||
evidence_turn_ids=turn_ids,
|
||||
)
|
||||
)
|
||||
transfer_suite_record_id = str(uuid4())
|
||||
transfer_suite = _build_transfer_suite(
|
||||
fixture_suffix=block_suffix,
|
||||
evidence_turn_ids=turn_ids,
|
||||
)
|
||||
suite_submission = {
|
||||
"submission_id": str(uuid4()),
|
||||
"transfer_suite_record_id": transfer_suite_record_id,
|
||||
"suite": transfer_suite,
|
||||
"model_run_id": suite_model_run_id,
|
||||
"instrument_id": TRANSFER_INSTRUMENT_ID,
|
||||
"instrument_version": INSTRUMENT_VERSION,
|
||||
}
|
||||
suite_path = f"/internal/sessions/{session_id}/calibration/transfer-suites"
|
||||
suite_created = internal.request(
|
||||
"POST",
|
||||
suite_path,
|
||||
suite_submission,
|
||||
expected={201},
|
||||
headers=token_header,
|
||||
)
|
||||
if (
|
||||
str(suite_created.body.get("transfer_suite_record_id"))
|
||||
!= transfer_suite_record_id
|
||||
or suite_created.body.get("trial_count") != 1
|
||||
):
|
||||
raise SmokeError("internal transfer suite route omitted its authoritative trial")
|
||||
|
||||
suite_read = learner.request("GET", "/calibration/learners/me")
|
||||
suite_projection = next(
|
||||
(
|
||||
item
|
||||
for item in suite_read.body.get("transfer_suites") or []
|
||||
if str(item.get("transfer_suite_record_id")) == transfer_suite_record_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if suite_projection is None or len(suite_projection.get("trials") or []) != 1:
|
||||
raise SmokeError("learner read model omitted authoritative transfer trial")
|
||||
original_transfer_trial_record_id = str(
|
||||
suite_projection["trials"][0]["transfer_trial_record_id"]
|
||||
)
|
||||
actual_path = "/calibration/transfer-executions"
|
||||
learner.request(
|
||||
"POST",
|
||||
actual_path,
|
||||
_actual_execution_request(
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=session_id,
|
||||
),
|
||||
expected={422},
|
||||
)
|
||||
|
||||
pending_started = learner.request(
|
||||
"POST",
|
||||
"/sessions",
|
||||
{
|
||||
"persona_code": practice_persona,
|
||||
"theory_mode": "humanistic",
|
||||
"goal_stages": ["라포", "탐색"],
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
pending_session_id = str(pending_started.body["session_id"])
|
||||
learner.request(
|
||||
"POST",
|
||||
actual_path,
|
||||
_actual_execution_request(
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=pending_session_id,
|
||||
),
|
||||
expected={422},
|
||||
)
|
||||
other_learner.request(
|
||||
"POST",
|
||||
actual_path,
|
||||
_actual_execution_request(
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=pending_session_id,
|
||||
),
|
||||
expected={404},
|
||||
)
|
||||
|
||||
practice_started = learner.request(
|
||||
"POST",
|
||||
"/sessions",
|
||||
{
|
||||
"persona_code": practice_persona,
|
||||
"theory_mode": "humanistic",
|
||||
"goal_stages": ["라포", "탐색"],
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
practice_session_id = str(practice_started.body["session_id"])
|
||||
learner.request(
|
||||
"POST",
|
||||
f"/sessions/{practice_session_id}/turn",
|
||||
{
|
||||
"text": (
|
||||
"그 말을 꺼내기까지 많이 외롭고 조심스러웠던 것 같아요. "
|
||||
"제가 이해한 마음이 맞는지 함께 확인해도 괜찮을까요?"
|
||||
)
|
||||
},
|
||||
)
|
||||
learner.request("POST", f"/sessions/{practice_session_id}/end")
|
||||
practice_review = _wait_for_session_review(
|
||||
learner,
|
||||
practice_session_id,
|
||||
timeout=args.review_poll_timeout,
|
||||
interval=args.review_poll_interval,
|
||||
)
|
||||
practice_turn_ids = _durable_turn_ids(practice_review["review"])
|
||||
|
||||
actual_request = _actual_execution_request(
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=practice_session_id,
|
||||
)
|
||||
actual_created = learner.request(
|
||||
"POST", actual_path, actual_request, expected={201}
|
||||
)
|
||||
actual_retried = learner.request(
|
||||
"POST", actual_path, actual_request, expected={201}
|
||||
)
|
||||
actual_execution = actual_created.body.get("execution") or {}
|
||||
actual_execution_event_id = str(
|
||||
actual_execution.get("execution_event_id") or ""
|
||||
)
|
||||
actual_model_run_id = str(actual_execution.get("model_run_id") or "")
|
||||
if (
|
||||
not actual_execution_event_id
|
||||
or not actual_model_run_id
|
||||
or actual_execution_event_id
|
||||
!= str(
|
||||
(actual_retried.body.get("execution") or {}).get("execution_event_id")
|
||||
)
|
||||
or actual_retried.body.get("idempotent_replay") is not True
|
||||
):
|
||||
raise SmokeError("same actual transfer execution retry was not stable")
|
||||
_assert_no_raw_transcript(actual_created.body, "actual_execution_response")
|
||||
|
||||
learner_read = learner.request("GET", "/calibration/learners/me")
|
||||
if learner_read.body.get("requested_view") != "learner":
|
||||
raise SmokeError("learner calibration read used the wrong role projection")
|
||||
histories = learner_read.body.get("prediction_histories") or []
|
||||
target = next((item for item in histories if item.get("history_id") == history_id), None)
|
||||
if not target or not target.get("lock") or not target.get("external_observation"):
|
||||
raise SmokeError("locked prediction and external observation were not hydrated")
|
||||
actual_read_proof = _actual_execution_read_proof(
|
||||
learner_read.body,
|
||||
execution_event_id=actual_execution_event_id,
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=practice_session_id,
|
||||
durable_turn_ids=practice_turn_ids,
|
||||
)
|
||||
actual_db_proof = asyncio.run(
|
||||
_fetch_actual_transfer_db_proof(
|
||||
dsn,
|
||||
learner_id=learner_id,
|
||||
execution_event_id=actual_execution_event_id,
|
||||
original_transfer_trial_record_id=original_transfer_trial_record_id,
|
||||
practice_session_id=practice_session_id,
|
||||
durable_turn_ids=practice_turn_ids,
|
||||
model_run_id=actual_model_run_id,
|
||||
)
|
||||
)
|
||||
teacher_read = teacher.request("GET", f"/calibration/learners/{learner_id}")
|
||||
if teacher_read.body.get("requested_view") != "supervisor":
|
||||
raise SmokeError("teacher calibration read used the wrong role projection")
|
||||
other_learner.request(
|
||||
"GET", f"/calibration/learners/{learner_id}", expected={403}
|
||||
)
|
||||
other_teacher.request(
|
||||
"GET", f"/calibration/learners/{learner_id}", expected={404}
|
||||
)
|
||||
_assert_no_aggregate_score(learner_read.body)
|
||||
_assert_no_aggregate_score(teacher_read.body)
|
||||
_assert_no_raw_transcript(learner_read.body, "learner_read")
|
||||
_assert_no_raw_transcript(teacher_read.body, "teacher_read")
|
||||
if learner_read.body.get("clinical_claim_allowed") is not False:
|
||||
raise SmokeError("calibration read omitted the non-clinical boundary")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"api_base_url": args.api_base_url,
|
||||
"fixture_policy": "retained unique dev:e2e identities; no fixture deletion",
|
||||
"session_id": session_id,
|
||||
"history_id": history_id,
|
||||
"prediction_revision_id": revision_id,
|
||||
"prediction_lock_id": lock_id,
|
||||
"performance_observation_id": observation_id,
|
||||
"transfer_suite_record_id": transfer_suite_record_id,
|
||||
"original_transfer_trial_record_id": original_transfer_trial_record_id,
|
||||
"practice_session_id": practice_session_id,
|
||||
"actual_transfer_execution_event_id": actual_execution_event_id,
|
||||
"proof": {
|
||||
"durable_turn_uuid_count": len(turn_ids),
|
||||
"source_review_poll_count": source_review["poll_count"],
|
||||
"pre_reveal_prediction_recorded": True,
|
||||
"same_prediction_retry_stable": True,
|
||||
"changed_prediction_retry_rejected": True,
|
||||
"prediction_lock_idempotent": True,
|
||||
"post_lock_revision_rejected": True,
|
||||
"authenticated_external_observation": True,
|
||||
"same_observation_retry_stable": True,
|
||||
"changed_observation_retry_rejected": True,
|
||||
"authoritative_transfer_suite_created_by_internal_route": True,
|
||||
"same_source_session_rejected": True,
|
||||
"unended_practice_session_rejected": True,
|
||||
"other_learner_actual_execution_rejected": True,
|
||||
"different_persona_practice_session": practice_persona,
|
||||
"practice_review_poll_count": practice_review["poll_count"],
|
||||
"same_actual_execution_retry_stable": True,
|
||||
"actual_execution_read_model": actual_read_proof,
|
||||
"actual_execution_postgres": actual_db_proof,
|
||||
"no_raw_transcript_evidence": True,
|
||||
"learner_role_projection": "learner",
|
||||
"teacher_role_projection": "supervisor",
|
||||
"other_learner_rejected": True,
|
||||
"cross_cohort_teacher_rejected": True,
|
||||
"no_aggregate_score": True,
|
||||
"clinical_claim_allowed": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--api-base-url", default="http://127.0.0.1:8008")
|
||||
parser.add_argument("--internal-token", required=True)
|
||||
parser.add_argument("--database-url", default="")
|
||||
parser.add_argument("--request-timeout", type=float, default=180.0)
|
||||
parser.add_argument("--review-poll-timeout", type=float, default=180.0)
|
||||
parser.add_argument("--review-poll-interval", type=float, default=0.5)
|
||||
parser.add_argument("--out", default="")
|
||||
args = parser.parse_args()
|
||||
result = run(args)
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.out:
|
||||
path = Path(args.out)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue