G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
884
scripts/smoke-alliance-pulse-api.py
Normal file
884
scripts/smoke-alliance-pulse-api.py
Normal file
|
|
@ -0,0 +1,884 @@
|
|||
"""Exercise the G1 alliance-pulse HTTP flow and verify its Postgres ledger.
|
||||
|
||||
The smoke creates durable dev/E2E fixtures and intentionally does not delete
|
||||
them. It proves the learner-lock/reveal boundary, the 3x3 read model, cohort
|
||||
teacher access, and append-only supervisor supersession without printing
|
||||
cookies, credentials, e-mail addresses, or transcript text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
ALLIANCE_DIMENSIONS = {"goal", "task", "bond"}
|
||||
AGENT_PERSPECTIVES = {"client_agent_report", "independent_observer"}
|
||||
ALL_PERSPECTIVES = {
|
||||
"learner_self_report",
|
||||
"client_agent_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
}
|
||||
COHORT_ID = "e2e-hanshin"
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApiResponse:
|
||||
status: int
|
||||
body: Any
|
||||
|
||||
|
||||
def _json_object(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
decoded = json.loads(value)
|
||||
if isinstance(decoded, dict):
|
||||
return decoded
|
||||
raise SmokeError(f"expected a JSON object, got {type(value).__name__}")
|
||||
|
||||
|
||||
class ApiClient:
|
||||
def __init__(self, base_url: str, timeout: float) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(CookieJar())
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
*,
|
||||
expected: set[int] | None = None,
|
||||
) -> ApiResponse:
|
||||
data = None
|
||||
headers = {"Accept": "application/json"}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=data,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
body = json.loads(raw) if raw else {}
|
||||
result = ApiResponse(status=response.status, body=body)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
body = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
body = {"detail": raw[:500]}
|
||||
result = ApiResponse(status=exc.code, body=body)
|
||||
except urllib.error.URLError as exc:
|
||||
raise SmokeError(
|
||||
f"{method} {path} transport failed: {type(exc.reason).__name__}"
|
||||
) from exc
|
||||
|
||||
allowed = expected or {200}
|
||||
if result.status not in allowed:
|
||||
detail = result.body.get("detail") if isinstance(result.body, dict) else None
|
||||
raise SmokeError(
|
||||
f"{method} {path} returned HTTP {result.status}; detail={detail!r}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _load_api_env() -> None:
|
||||
env_path = Path("apps/api/.env")
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def _onboarding_payload(display_name: str) -> dict[str, Any]:
|
||||
return {
|
||||
"legal_name": display_name,
|
||||
"affiliation": "한신대학교",
|
||||
"department": "상담심리학과",
|
||||
"grade_level": "통합검증",
|
||||
"phone": "010-0000-0000",
|
||||
"contact_address": "경기도 오산시 한신대학교",
|
||||
"nickname": display_name,
|
||||
"self_introduction": "G1 동맹 펄스 API와 원장을 검증하는 개발 fixture입니다.",
|
||||
"avatar_url": "",
|
||||
"terms_accepted": True,
|
||||
"privacy_accepted": True,
|
||||
}
|
||||
|
||||
|
||||
def _sign_in(
|
||||
client: ApiClient,
|
||||
*,
|
||||
email: str,
|
||||
role: str,
|
||||
display_name: str,
|
||||
cohort_ids: list[str] | None = None,
|
||||
) -> str:
|
||||
requested_cohorts = set(cohort_ids or [COHORT_ID])
|
||||
login = client.request(
|
||||
"POST",
|
||||
"/auth/dev-login",
|
||||
{
|
||||
"email": email,
|
||||
"role": role,
|
||||
"display_name": display_name,
|
||||
"cohort_ids": cohort_ids or [COHORT_ID],
|
||||
},
|
||||
)
|
||||
client.request(
|
||||
"POST", "/users/me/onboarding", _onboarding_payload(display_name)
|
||||
)
|
||||
user_id = str(login.body.get("user_id") or "")
|
||||
if not user_id:
|
||||
raise SmokeError("dev-login response omitted user_id")
|
||||
if login.body.get("role") != role:
|
||||
raise SmokeError(
|
||||
f"dev-login role mismatch: {login.body.get('role')!r} != {role!r}"
|
||||
)
|
||||
returned_cohorts = set(login.body.get("cohort_ids") or [])
|
||||
if returned_cohorts != requested_cohorts:
|
||||
raise SmokeError(
|
||||
f"dev-login cohort mismatch: {sorted(returned_cohorts)} != {sorted(requested_cohorts)}"
|
||||
)
|
||||
return user_id
|
||||
|
||||
|
||||
def _choose_persona(client: ApiClient) -> str:
|
||||
response = client.request("GET", "/personas")
|
||||
if not isinstance(response.body, list):
|
||||
raise SmokeError("persona catalog did not return a list")
|
||||
usable = [
|
||||
item
|
||||
for item in response.body
|
||||
if isinstance(item, dict)
|
||||
and item.get("source") == "database"
|
||||
and not item.get("degraded")
|
||||
and item.get("code")
|
||||
]
|
||||
if not usable:
|
||||
raise SmokeError("persona catalog has no non-degraded database persona")
|
||||
preferred = next((item for item in usable if item.get("code") == "P1"), usable[0])
|
||||
return str(preferred["code"])
|
||||
|
||||
|
||||
async def _create_ended_session_with_checkpoints(
|
||||
client: ApiClient,
|
||||
persona_code: str,
|
||||
*,
|
||||
dsn: str,
|
||||
learner_id: str,
|
||||
poll_timeout: float,
|
||||
poll_interval: float,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
started = client.request(
|
||||
"POST",
|
||||
"/sessions",
|
||||
{
|
||||
"persona_code": persona_code,
|
||||
"theory_mode": "humanistic",
|
||||
"goal_stages": ["라포", "탐색"],
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
session_id = str(started.body.get("session_id") or "")
|
||||
if not session_id:
|
||||
raise SmokeError("session start response omitted session_id")
|
||||
if started.body.get("degraded"):
|
||||
raise SmokeError("session start was degraded; refusing DB integration proof")
|
||||
|
||||
pre = client.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/alliance-pulses",
|
||||
{
|
||||
"checkpoint": "pre",
|
||||
"scores": {"goal": 0.63, "task": 0.59, "bond": 0.68},
|
||||
"evidence_turn_ids": [],
|
||||
},
|
||||
expected={202},
|
||||
)
|
||||
pre_id = str(pre.body.get("pulse_id") or "")
|
||||
if not pre_id:
|
||||
raise SmokeError("pre checkpoint omitted pulse_id")
|
||||
pre_terminal, pre_polls, pre_elapsed = _wait_for_terminal(
|
||||
client,
|
||||
session_id=session_id,
|
||||
pulse_id=pre_id,
|
||||
timeout=poll_timeout,
|
||||
interval=poll_interval,
|
||||
)
|
||||
_assert_pre_degraded_read_model(pre_terminal)
|
||||
|
||||
utterances = (
|
||||
"지금 가장 중요하게 다루고 싶은 이야기를 함께 정해도 괜찮을까요?",
|
||||
"그 목표를 위해 오늘은 어떤 방식으로 이야기를 나누는 게 도움이 될까요?",
|
||||
)
|
||||
first_turn = client.request(
|
||||
"POST", f"/sessions/{session_id}/turn", {"text": utterances[0]}
|
||||
)
|
||||
if not first_turn.body.get("client_reply"):
|
||||
raise SmokeError("first session turn returned no client reply")
|
||||
mid_turn_ids = await _fetch_turn_ids(dsn, session_id=session_id, user_id=learner_id)
|
||||
if len(mid_turn_ids) < 2:
|
||||
raise SmokeError("mid checkpoint requires one persisted learner/client exchange")
|
||||
mid = client.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/alliance-pulses",
|
||||
{
|
||||
"checkpoint": "mid",
|
||||
"scores": {"goal": 0.68, "task": 0.62, "bond": 0.74},
|
||||
"evidence_turn_ids": mid_turn_ids[:2],
|
||||
},
|
||||
expected={202},
|
||||
)
|
||||
mid_id = str(mid.body.get("pulse_id") or "")
|
||||
if not mid_id:
|
||||
raise SmokeError("mid checkpoint omitted pulse_id")
|
||||
mid_terminal, mid_polls, mid_elapsed = _wait_for_terminal(
|
||||
client,
|
||||
session_id=session_id,
|
||||
pulse_id=mid_id,
|
||||
timeout=poll_timeout,
|
||||
interval=poll_interval,
|
||||
)
|
||||
_assert_agent_read_model(mid_terminal)
|
||||
|
||||
for text in utterances[1:]:
|
||||
turn = client.request(
|
||||
"POST", f"/sessions/{session_id}/turn", {"text": text}
|
||||
)
|
||||
if not turn.body.get("client_reply"):
|
||||
raise SmokeError("session turn returned no client reply")
|
||||
ended = client.request("POST", f"/sessions/{session_id}/end")
|
||||
if str(ended.body.get("session_id") or "") != session_id:
|
||||
raise SmokeError("session end response did not preserve session_id")
|
||||
return session_id, {
|
||||
"pre": {
|
||||
"pulse_id": pre_id,
|
||||
"status": pre_terminal["status"],
|
||||
"poll_count": pre_polls,
|
||||
"elapsed_seconds": round(pre_elapsed, 3),
|
||||
},
|
||||
"mid": {
|
||||
"pulse_id": mid_id,
|
||||
"status": mid_terminal["status"],
|
||||
"poll_count": mid_polls,
|
||||
"elapsed_seconds": round(mid_elapsed, 3),
|
||||
"evidence_turn_count": 2,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _measurements(pulse: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
items = pulse.get("measurements") or []
|
||||
if not isinstance(items, list):
|
||||
raise SmokeError("alliance pulse measurements are not a list")
|
||||
return [item for item in items if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _single_pulse(response: ApiResponse, pulse_id: str) -> dict[str, Any]:
|
||||
if not isinstance(response.body, dict):
|
||||
raise SmokeError("alliance pulse list response is not an object")
|
||||
items = response.body.get("items") or []
|
||||
matching = [item for item in items if str(item.get("pulse_id")) == pulse_id]
|
||||
if len(matching) != 1:
|
||||
raise SmokeError(f"expected one pulse {pulse_id}, found {len(matching)}")
|
||||
return matching[0]
|
||||
|
||||
|
||||
def _assert_locked_read_model(pulse: dict[str, Any]) -> None:
|
||||
if pulse.get("status") != "awaiting_agents":
|
||||
raise SmokeError(
|
||||
f"first read did not observe awaiting_agents: {pulse.get('status')!r}"
|
||||
)
|
||||
if pulse.get("revealed_at") is not None:
|
||||
raise SmokeError("awaiting pulse exposed revealed_at")
|
||||
measurements = _measurements(pulse)
|
||||
perspectives = {str(item.get("perspective")) for item in measurements}
|
||||
if perspectives != {"learner_self_report"}:
|
||||
raise SmokeError(
|
||||
f"external perspective leaked before reveal: {sorted(perspectives)}"
|
||||
)
|
||||
dimensions = {str(item.get("dimension")) for item in measurements}
|
||||
if dimensions != ALLIANCE_DIMENSIONS or len(measurements) != 3:
|
||||
raise SmokeError(
|
||||
f"locked self-assessment is not exactly 3 dimensions: {dimensions}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_agent_read_model(pulse: dict[str, Any]) -> None:
|
||||
if pulse.get("status") != "ready":
|
||||
raise SmokeError(
|
||||
"agent run did not produce a ready terminal pulse: "
|
||||
f"status={pulse.get('status')!r}, error_code={pulse.get('error_code')!r}"
|
||||
)
|
||||
if not pulse.get("revealed_at"):
|
||||
raise SmokeError("ready pulse omitted revealed_at")
|
||||
measurements = _measurements(pulse)
|
||||
perspective_dimensions: dict[str, set[str]] = {}
|
||||
for item in measurements:
|
||||
perspective = str(item.get("perspective"))
|
||||
perspective_dimensions.setdefault(perspective, set()).add(
|
||||
str(item.get("dimension"))
|
||||
)
|
||||
if item.get("status") != "ready" or item.get("value") is None:
|
||||
raise SmokeError(
|
||||
f"terminal read model contains non-ready score: {perspective}/{item.get('dimension')}"
|
||||
)
|
||||
expected = {
|
||||
"learner_self_report": ALLIANCE_DIMENSIONS,
|
||||
"client_agent_report": ALLIANCE_DIMENSIONS,
|
||||
"independent_observer": ALLIANCE_DIMENSIONS,
|
||||
}
|
||||
if perspective_dimensions != expected or len(measurements) != 9:
|
||||
raise SmokeError(
|
||||
f"terminal read model is not 3 perspectives x 3 dimensions: {perspective_dimensions}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_pre_degraded_read_model(pulse: dict[str, Any]) -> None:
|
||||
if pulse.get("status") != "degraded" or pulse.get("error_code") != "insufficient_transcript":
|
||||
raise SmokeError(
|
||||
"pre checkpoint without transcript must reveal an explicit insufficient state: "
|
||||
f"status={pulse.get('status')!r}, error_code={pulse.get('error_code')!r}"
|
||||
)
|
||||
if not pulse.get("revealed_at"):
|
||||
raise SmokeError("degraded pre checkpoint omitted revealed_at")
|
||||
measurements = _measurements(pulse)
|
||||
if len(measurements) != 9:
|
||||
raise SmokeError(f"degraded pre checkpoint expected 9 measurements, got {len(measurements)}")
|
||||
by_perspective: dict[str, list[dict[str, Any]]] = {}
|
||||
for item in measurements:
|
||||
by_perspective.setdefault(str(item.get("perspective")), []).append(item)
|
||||
if set(by_perspective) != {
|
||||
"learner_self_report",
|
||||
"client_agent_report",
|
||||
"independent_observer",
|
||||
}:
|
||||
raise SmokeError(f"degraded pre perspectives mismatch: {sorted(by_perspective)}")
|
||||
learner = by_perspective["learner_self_report"]
|
||||
external = by_perspective["client_agent_report"] + by_perspective["independent_observer"]
|
||||
if any(item.get("status") != "ready" or item.get("value") is None for item in learner):
|
||||
raise SmokeError("pre learner baseline was not preserved as ready scores")
|
||||
if any(
|
||||
item.get("status") != "degraded"
|
||||
or item.get("value") is not None
|
||||
or item.get("error_code") != "insufficient_transcript"
|
||||
for item in external
|
||||
):
|
||||
raise SmokeError("pre external perspectives were not kept scoreless and degraded")
|
||||
|
||||
|
||||
def _wait_for_terminal(
|
||||
client: ApiClient,
|
||||
*,
|
||||
session_id: str,
|
||||
pulse_id: str,
|
||||
timeout: float,
|
||||
interval: float,
|
||||
) -> tuple[dict[str, Any], int, float]:
|
||||
started_at = time.monotonic()
|
||||
polls = 0
|
||||
while True:
|
||||
polls += 1
|
||||
pulse = _single_pulse(
|
||||
client.request("GET", f"/sessions/{session_id}/alliance-pulses"),
|
||||
pulse_id,
|
||||
)
|
||||
if pulse.get("status") != "awaiting_agents":
|
||||
return pulse, polls, time.monotonic() - started_at
|
||||
perspectives = {
|
||||
str(item.get("perspective")) for item in _measurements(pulse)
|
||||
}
|
||||
if perspectives - {"learner_self_report"}:
|
||||
raise SmokeError(
|
||||
f"external perspective leaked during awaiting state: {perspectives}"
|
||||
)
|
||||
if time.monotonic() - started_at >= timeout:
|
||||
raise SmokeError(
|
||||
f"alliance pulse stayed awaiting_agents for more than {timeout:g}s"
|
||||
)
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
async def _set_admin_context(conn: asyncpg.Connection[Any], user_id: str) -> None:
|
||||
# ``app.is_ai_context()`` evaluates to NULL when the GUC is absent, which
|
||||
# intentionally makes both RLS branches fail closed. Mirror app.db.acquire
|
||||
# and set the human context explicitly instead of relying on a missing GUC.
|
||||
await conn.execute("SELECT set_config('app.ai_context', '', true)")
|
||||
await conn.execute("SELECT set_config('app.current_role', 'admin', true)")
|
||||
await conn.execute("SELECT set_config('app.current_uid', $1, true)", user_id)
|
||||
await conn.execute("SELECT set_config('app.current_cohort', $1, true)", COHORT_ID)
|
||||
|
||||
|
||||
async def _fetch_turn_ids(
|
||||
dsn: str, *, session_id: str, user_id: str
|
||||
) -> list[str]:
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await _set_admin_context(conn, user_id)
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id::text AS turn_id
|
||||
FROM app.turns
|
||||
WHERE session_id = $1::uuid
|
||||
ORDER BY seq, id
|
||||
""",
|
||||
session_id,
|
||||
)
|
||||
return [str(row["turn_id"]) for row in rows]
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def _fetch_ledger(
|
||||
dsn: str, *, session_id: str, pulse_id: str, user_id: str
|
||||
) -> dict[str, Any]:
|
||||
conn = await asyncpg.connect(dsn)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await _set_admin_context(conn, user_id)
|
||||
pulse = await conn.fetchrow(
|
||||
"""
|
||||
SELECT status, learner_locked_at, revealed_at, created_at, updated_at,
|
||||
error_code
|
||||
FROM app.alliance_pulse
|
||||
WHERE pulse_id = $1::uuid AND session_id = $2::uuid
|
||||
""",
|
||||
pulse_id,
|
||||
session_id,
|
||||
)
|
||||
assessment = await conn.fetchrow(
|
||||
"""
|
||||
SELECT scores, evidence_turn_ids, locked_at, created_at
|
||||
FROM app.self_assessment
|
||||
WHERE pulse_id = $1::uuid AND session_id = $2::uuid
|
||||
""",
|
||||
pulse_id,
|
||||
session_id,
|
||||
)
|
||||
events = await conn.fetch(
|
||||
"""
|
||||
SELECT measurement_id::text, supersedes_id::text, dimension,
|
||||
perspective, source_kind, status, value, model_run_id::text,
|
||||
created_at
|
||||
FROM app.measurement_event
|
||||
WHERE pulse_id = $1::uuid AND session_id = $2::uuid
|
||||
ORDER BY created_at, measurement_id
|
||||
""",
|
||||
pulse_id,
|
||||
session_id,
|
||||
)
|
||||
history = await conn.fetch(
|
||||
"""
|
||||
SELECT from_status, to_status, error_code, revealed_at,
|
||||
changed_by_role, ai_view, changed_at
|
||||
FROM audit.alliance_pulse_status_event
|
||||
WHERE pulse_id = $1::uuid
|
||||
ORDER BY changed_at, status_event_id
|
||||
""",
|
||||
pulse_id,
|
||||
)
|
||||
model_runs = await conn.fetch(
|
||||
"""
|
||||
SELECT mr.model_run_id::text, mr.agent_role, mr.provider, mr.model,
|
||||
mr.prompt_bundle_id, mr.prompt_bundle_version,
|
||||
mr.prompt_bundle_hash, mr.structured_schema_version,
|
||||
mr.input_evidence_hash, mr.status, mr.error_code,
|
||||
mr.metadata, mr.created_at
|
||||
FROM audit.model_run mr
|
||||
WHERE mr.session_id = $2::uuid
|
||||
AND mr.metadata->>'pulse_id' = $1::text
|
||||
ORDER BY mr.agent_role, mr.created_at, mr.model_run_id
|
||||
""",
|
||||
pulse_id,
|
||||
session_id,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
if pulse is None or assessment is None:
|
||||
raise SmokeError("Postgres ledger omitted pulse or self-assessment")
|
||||
return {
|
||||
"pulse": dict(pulse),
|
||||
"assessment": dict(assessment),
|
||||
"events": [dict(row) for row in events],
|
||||
"history": [dict(row) for row in history],
|
||||
"model_runs": [dict(row) for row in model_runs],
|
||||
}
|
||||
|
||||
|
||||
def _assert_postgres_ledger(
|
||||
ledger: dict[str, Any], *, expected_prompt_version: str
|
||||
) -> dict[str, Any]:
|
||||
pulse = ledger["pulse"]
|
||||
assessment = ledger["assessment"]
|
||||
events = ledger["events"]
|
||||
history = ledger["history"]
|
||||
model_runs = ledger["model_runs"]
|
||||
|
||||
if pulse["status"] != "ready" or pulse["error_code"] is not None:
|
||||
raise SmokeError(f"Postgres pulse terminal state is invalid: {pulse}")
|
||||
if not (
|
||||
pulse["created_at"]
|
||||
<= pulse["learner_locked_at"]
|
||||
<= pulse["revealed_at"]
|
||||
<= pulse["updated_at"]
|
||||
):
|
||||
raise SmokeError("Postgres pulse timestamp ordering violated lock-before-reveal")
|
||||
if assessment["locked_at"] != pulse["learner_locked_at"]:
|
||||
raise SmokeError("self-assessment lock and pulse lock timestamps diverged")
|
||||
if set(_json_object(assessment["scores"])) != ALLIANCE_DIMENSIONS:
|
||||
raise SmokeError("self-assessment did not store exactly goal/task/bond")
|
||||
|
||||
by_perspective: dict[str, list[dict[str, Any]]] = {}
|
||||
for event in events:
|
||||
by_perspective.setdefault(str(event["perspective"]), []).append(event)
|
||||
expected_counts = {
|
||||
"learner_self_report": 3,
|
||||
"client_agent_report": 3,
|
||||
"independent_observer": 3,
|
||||
"supervisor_human": 6,
|
||||
}
|
||||
actual_counts = {
|
||||
perspective: len(items) for perspective, items in by_perspective.items()
|
||||
}
|
||||
if actual_counts != expected_counts:
|
||||
raise SmokeError(
|
||||
f"Postgres perspective event counts differ: {actual_counts} != {expected_counts}"
|
||||
)
|
||||
for perspective, items in by_perspective.items():
|
||||
dimensions = {str(item["dimension"]) for item in items}
|
||||
if dimensions != ALLIANCE_DIMENSIONS:
|
||||
raise SmokeError(
|
||||
f"Postgres {perspective} dimensions differ: {dimensions}"
|
||||
)
|
||||
for perspective in AGENT_PERSPECTIVES:
|
||||
if any(not item["model_run_id"] for item in by_perspective[perspective]):
|
||||
raise SmokeError(f"Postgres {perspective} score omitted model_run_id")
|
||||
supervisor_events = by_perspective["supervisor_human"]
|
||||
if sum(item["supersedes_id"] is not None for item in supervisor_events) != 3:
|
||||
raise SmokeError("second supervisor rating did not supersede all 3 first ratings")
|
||||
|
||||
history_states = [str(row["to_status"]) for row in history]
|
||||
if history_states != ["awaiting_agents", "ready"]:
|
||||
raise SmokeError(f"pulse status history differs: {history_states}")
|
||||
if not 2 <= len(model_runs) <= 4:
|
||||
raise SmokeError(
|
||||
f"expected 2-4 attempt-level model runs, got {len(model_runs)}"
|
||||
)
|
||||
ready_runs = [row for row in model_runs if row["status"] == "ready"]
|
||||
if len(ready_runs) != 2 or {row["agent_role"] for row in ready_runs} != {
|
||||
"client",
|
||||
"evaluator",
|
||||
}:
|
||||
raise SmokeError("model-run ledger omitted a ready client/evaluator terminal run")
|
||||
if any(
|
||||
row["prompt_bundle_version"] != expected_prompt_version
|
||||
or not row["prompt_bundle_hash"]
|
||||
or not row["input_evidence_hash"]
|
||||
for row in model_runs
|
||||
):
|
||||
raise SmokeError(
|
||||
"model-run provenance omitted the expected prompt bundle or evidence hashes: "
|
||||
f"expected={expected_prompt_version!r}"
|
||||
)
|
||||
attempt_by_role: dict[str, list[int]] = {}
|
||||
for row in model_runs:
|
||||
attempt = int(_json_object(row["metadata"] or "{}").get("attempt") or 0)
|
||||
attempt_by_role.setdefault(str(row["agent_role"]), []).append(attempt)
|
||||
if set(attempt_by_role) != {"client", "evaluator"} or any(
|
||||
attempts != list(range(1, len(attempts) + 1))
|
||||
for attempts in attempt_by_role.values()
|
||||
):
|
||||
raise SmokeError(f"model-run retry attempts are not contiguous: {attempt_by_role}")
|
||||
return {
|
||||
"pulse_status": pulse["status"],
|
||||
"lock_before_reveal": True,
|
||||
"status_history": history_states,
|
||||
"self_assessment_rows": 1,
|
||||
"model_run_count": len(model_runs),
|
||||
"ready_model_run_count": len(ready_runs),
|
||||
"prompt_bundle_version": expected_prompt_version,
|
||||
"attempts_by_agent_role": attempt_by_role,
|
||||
"measurement_event_count": len(events),
|
||||
"events_by_perspective": actual_counts,
|
||||
"supervisor_supersedes_count": 3,
|
||||
}
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
_load_api_env()
|
||||
dsn = args.database_url or os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
raise SmokeError("DATABASE_URL is required via --database-url or apps/api/.env")
|
||||
|
||||
health_client = ApiClient(args.api_base_url, args.request_timeout)
|
||||
health = health_client.request("GET", "/health")
|
||||
if not health.body.get("db") or not health.body.get("engine"):
|
||||
raise SmokeError(
|
||||
f"API health is not DB+engine ready: status={health.body.get('status')!r}"
|
||||
)
|
||||
|
||||
suffix = f"{int(time.time())}.{secrets.token_hex(3)}"
|
||||
learner_client = ApiClient(args.api_base_url, args.request_timeout)
|
||||
teacher_client = ApiClient(args.api_base_url, args.request_timeout)
|
||||
other_learner_client = ApiClient(args.api_base_url, args.request_timeout)
|
||||
other_teacher_client = ApiClient(args.api_base_url, args.request_timeout)
|
||||
learner_id = _sign_in(
|
||||
learner_client,
|
||||
email=f"dev.e2e.alliance.learner.{suffix}@hs.ac.kr",
|
||||
role="learner",
|
||||
display_name="Alliance Pulse Learner",
|
||||
)
|
||||
teacher_id = _sign_in(
|
||||
teacher_client,
|
||||
email=f"dev.e2e.alliance.teacher.{suffix}@hs.ac.kr",
|
||||
role="teacher",
|
||||
display_name="Alliance Pulse Teacher",
|
||||
)
|
||||
_sign_in(
|
||||
other_learner_client,
|
||||
email=f"dev.e2e.alliance.other-learner.{suffix}@hs.ac.kr",
|
||||
role="learner",
|
||||
display_name="Alliance Pulse Other Learner",
|
||||
)
|
||||
_sign_in(
|
||||
other_teacher_client,
|
||||
email=f"dev.e2e.alliance.other-teacher.{suffix}@hs.ac.kr",
|
||||
role="teacher",
|
||||
display_name="Alliance Pulse Other Teacher",
|
||||
cohort_ids=["e2e-other-cohort"],
|
||||
)
|
||||
persona_code = _choose_persona(learner_client)
|
||||
session_id, checkpoint_proof = await _create_ended_session_with_checkpoints(
|
||||
learner_client,
|
||||
persona_code,
|
||||
dsn=dsn,
|
||||
learner_id=learner_id,
|
||||
poll_timeout=args.poll_timeout,
|
||||
poll_interval=args.poll_interval,
|
||||
)
|
||||
turn_ids = await _fetch_turn_ids(
|
||||
dsn, session_id=session_id, user_id=learner_id
|
||||
)
|
||||
if len(turn_ids) < 4:
|
||||
raise SmokeError(f"Postgres stored only {len(turn_ids)} transcript turns")
|
||||
|
||||
learner_scores = {"goal": 0.72, "task": 0.64, "bond": 0.81}
|
||||
pulse_payload = {
|
||||
"checkpoint": "post",
|
||||
"scores": learner_scores,
|
||||
"evidence_turn_ids": turn_ids[:2],
|
||||
}
|
||||
created = learner_client.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/alliance-pulses",
|
||||
pulse_payload,
|
||||
expected={202},
|
||||
)
|
||||
pulse_id = str(created.body.get("pulse_id") or "")
|
||||
if not pulse_id or created.body.get("status") != "awaiting_agents":
|
||||
raise SmokeError(f"pulse create response violated 202 lock contract: {created.body}")
|
||||
if created.body.get("idempotent_replay") is not False:
|
||||
raise SmokeError("first pulse creation was incorrectly marked as a replay")
|
||||
|
||||
same_payload = learner_client.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/alliance-pulses",
|
||||
pulse_payload,
|
||||
expected={202},
|
||||
)
|
||||
if (
|
||||
str(same_payload.body.get("pulse_id") or "") != pulse_id
|
||||
or same_payload.body.get("idempotent_replay") is not True
|
||||
):
|
||||
raise SmokeError("same pulse payload did not return the stable replay result")
|
||||
changed_payload = {
|
||||
**pulse_payload,
|
||||
"scores": {**learner_scores, "goal": 0.71},
|
||||
}
|
||||
learner_client.request(
|
||||
"POST",
|
||||
f"/sessions/{session_id}/alliance-pulses",
|
||||
changed_payload,
|
||||
expected={409},
|
||||
)
|
||||
|
||||
other_learner_client.request(
|
||||
"GET", f"/sessions/{session_id}/alliance-pulses", expected={404}
|
||||
)
|
||||
other_teacher_client.request(
|
||||
"GET", f"/sessions/{session_id}/alliance-pulses", expected={404}
|
||||
)
|
||||
|
||||
first_read = _single_pulse(
|
||||
learner_client.request(
|
||||
"GET", f"/sessions/{session_id}/alliance-pulses"
|
||||
),
|
||||
pulse_id,
|
||||
)
|
||||
_assert_locked_read_model(first_read)
|
||||
|
||||
terminal, poll_count, elapsed = _wait_for_terminal(
|
||||
learner_client,
|
||||
session_id=session_id,
|
||||
pulse_id=pulse_id,
|
||||
timeout=args.poll_timeout,
|
||||
interval=args.poll_interval,
|
||||
)
|
||||
_assert_agent_read_model(terminal)
|
||||
|
||||
teacher_read = _single_pulse(
|
||||
teacher_client.request(
|
||||
"GET", f"/sessions/{session_id}/alliance-pulses"
|
||||
),
|
||||
pulse_id,
|
||||
)
|
||||
_assert_agent_read_model(teacher_read)
|
||||
if teacher_read != terminal:
|
||||
raise SmokeError("teacher cohort read model differs from learner read model")
|
||||
|
||||
first_supervisor = {
|
||||
"scores": {"goal": 0.76, "task": 0.69, "bond": 0.84},
|
||||
"evidence_turn_ids": [turn_ids[0]],
|
||||
"note": "목표 합의와 관계적 안전감을 축어록 근거로 독립 평정함.",
|
||||
}
|
||||
second_supervisor = {
|
||||
"scores": {"goal": 0.79, "task": 0.73, "bond": 0.86},
|
||||
"evidence_turn_ids": [turn_ids[0], turn_ids[1]],
|
||||
"note": "추가 축어록 근거를 반영해 세 축을 독립적으로 재평정함.",
|
||||
}
|
||||
supervisor_path = (
|
||||
f"/sessions/{session_id}/alliance-pulses/{pulse_id}/supervisor-rating"
|
||||
)
|
||||
for payload in (first_supervisor, second_supervisor):
|
||||
response = teacher_client.request(
|
||||
"POST", supervisor_path, payload, expected={201}
|
||||
)
|
||||
if response.body.get("status") != "recorded":
|
||||
raise SmokeError("supervisor rating response omitted recorded status")
|
||||
|
||||
teacher_after_supervisor = _single_pulse(
|
||||
teacher_client.request(
|
||||
"GET", f"/sessions/{session_id}/alliance-pulses"
|
||||
),
|
||||
pulse_id,
|
||||
)
|
||||
measurements = _measurements(teacher_after_supervisor)
|
||||
perspective_dimensions: dict[str, set[str]] = {}
|
||||
for item in measurements:
|
||||
perspective_dimensions.setdefault(str(item.get("perspective")), set()).add(
|
||||
str(item.get("dimension"))
|
||||
)
|
||||
if set(perspective_dimensions) != ALL_PERSPECTIVES or any(
|
||||
dimensions != ALLIANCE_DIMENSIONS
|
||||
for dimensions in perspective_dimensions.values()
|
||||
):
|
||||
raise SmokeError(
|
||||
f"teacher latest read model is not 4 perspectives x 3 dimensions: {perspective_dimensions}"
|
||||
)
|
||||
if len(measurements) != 12:
|
||||
raise SmokeError(
|
||||
f"teacher latest read model returned {len(measurements)} measurements, expected 12"
|
||||
)
|
||||
|
||||
ledger = await _fetch_ledger(
|
||||
dsn,
|
||||
session_id=session_id,
|
||||
pulse_id=pulse_id,
|
||||
user_id=teacher_id,
|
||||
)
|
||||
ledger_proof = _assert_postgres_ledger(
|
||||
ledger, expected_prompt_version=args.expected_prompt_version
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"api_base_url": args.api_base_url,
|
||||
"health": {
|
||||
"status": health.body.get("status"),
|
||||
"db": health.body.get("db"),
|
||||
"engine": health.body.get("engine"),
|
||||
"engine_mode": health.body.get("engine_mode"),
|
||||
},
|
||||
"fixture_policy": "retained dev:e2e identities; no fixture deletion",
|
||||
"cohort_id": COHORT_ID,
|
||||
"persona_code": persona_code,
|
||||
"session_id": session_id,
|
||||
"pulse_id": pulse_id,
|
||||
"checkpoint_proof": checkpoint_proof,
|
||||
"http_proof": {
|
||||
"create_status": created.status,
|
||||
"same_payload_status": same_payload.status,
|
||||
"same_payload_stable_id": True,
|
||||
"same_payload_idempotent_replay": True,
|
||||
"changed_payload_rejected_status": 409,
|
||||
"first_read_status": first_read["status"],
|
||||
"external_perspectives_hidden_before_reveal": True,
|
||||
"terminal_status": terminal["status"],
|
||||
"terminal_poll_count": poll_count,
|
||||
"terminal_elapsed_seconds": round(elapsed, 3),
|
||||
"learner_terminal_measurements": len(_measurements(terminal)),
|
||||
"teacher_cohort_read": True,
|
||||
"other_learner_rejected_status": 404,
|
||||
"cross_cohort_teacher_rejected_status": 404,
|
||||
"authorization_audit_status_history": ledger_proof["status_history"],
|
||||
"supervisor_write_statuses": [201, 201],
|
||||
"teacher_latest_measurements": len(measurements),
|
||||
},
|
||||
"postgres_proof": ledger_proof,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--api-base-url", default="http://127.0.0.1:8002")
|
||||
parser.add_argument("--database-url", default="")
|
||||
parser.add_argument("--request-timeout", type=float, default=180.0)
|
||||
parser.add_argument("--poll-timeout", type=float, default=300.0)
|
||||
parser.add_argument("--poll-interval", type=float, default=0.5)
|
||||
parser.add_argument("--expected-prompt-version", default="1.4.0")
|
||||
parser.add_argument("--out", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = asyncio.run(run(args))
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
||||
if args.out:
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue