16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
477 lines
20 KiB
Python
477 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove G2 warning and noisy-control trajectories on real PostgreSQL rows.
|
|
|
|
The smoke creates two synthetic educational five-session cases. It submits each
|
|
session's three axes through the production route/store measurement producer, then
|
|
reads the resulting learner/teacher projections and audits the immutable ledger.
|
|
No source score is copied into the evidence JSON and no clinical claim is made.
|
|
|
|
Run only against an expendable development database. Unique fixture rows remain
|
|
because the measurement and trajectory ledgers are intentionally append-only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
API_ROOT = REPO_ROOT / "apps" / "api"
|
|
API_ENV = API_ROOT / ".env"
|
|
|
|
|
|
def _load_api_env() -> None:
|
|
if not API_ENV.exists():
|
|
return
|
|
for raw_line in API_ENV.read_text(encoding="utf-8").splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
|
|
|
|
|
_load_api_env()
|
|
if str(API_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(API_ROOT))
|
|
|
|
from app import db, persona_repository, session_persistence # noqa: E402
|
|
from app.deps import Principal, Role # noqa: E402
|
|
from app.routes import outcome_trajectories # noqa: E402
|
|
from app.routes.outcome_trajectories import ( # noqa: E402
|
|
OutcomeAxisValues,
|
|
OutcomeObservationSubmissionRequest,
|
|
)
|
|
from app.services import memory, state_machine # noqa: E402
|
|
|
|
|
|
COHORT = "g2-trajectory-alert-smoke"
|
|
AXES = ("distress_load", "daily_functioning", "learning_engagement")
|
|
ALERT_STATUSES = {"off_track", "deteriorating"}
|
|
INSTRUMENT_ID = "vignette-session-outcome-checkin"
|
|
INSTRUMENT_VERSION = "1.0.0"
|
|
|
|
# These values are synthetic educational fixture inputs. They intentionally stay
|
|
# inside the executable fixture and are never copied to the persisted evidence JSON.
|
|
WORSENING_PROFILE = (
|
|
{"distress_load": 0.70, "daily_functioning": 0.30, "learning_engagement": 0.40},
|
|
{"distress_load": 0.62, "daily_functioning": 0.40, "learning_engagement": 0.48},
|
|
{"distress_load": 0.72, "daily_functioning": 0.50, "learning_engagement": 0.56},
|
|
{"distress_load": 0.70, "daily_functioning": 0.60, "learning_engagement": 0.64},
|
|
{"distress_load": 0.68, "daily_functioning": 0.68, "learning_engagement": 0.72},
|
|
)
|
|
NOISY_CONTROL_PROFILE = (
|
|
{"distress_load": 0.70, "daily_functioning": 0.30, "learning_engagement": 0.40},
|
|
{"distress_load": 0.62, "daily_functioning": 0.40, "learning_engagement": 0.48},
|
|
{"distress_load": 0.64, "daily_functioning": 0.50, "learning_engagement": 0.56},
|
|
{"distress_load": 0.46, "daily_functioning": 0.60, "learning_engagement": 0.64},
|
|
{"distress_load": 0.38, "daily_functioning": 0.68, "learning_engagement": 0.72},
|
|
)
|
|
EXPECTED_WORSENING = (
|
|
"on_track",
|
|
"on_track",
|
|
"off_track",
|
|
"deteriorating",
|
|
"deteriorating",
|
|
)
|
|
EXPECTED_CONTROL = ("on_track", "on_track", "watch", "on_track", "on_track")
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _principal(user_id: UUID, role: Role) -> Principal:
|
|
return Principal(
|
|
user_id=str(user_id),
|
|
role=role,
|
|
cohort_ids=[COHORT],
|
|
consent_at=1.0,
|
|
profile_completed_at=1.0,
|
|
)
|
|
|
|
|
|
async def _seed_users(*user_roles: tuple[UUID, str]) -> None:
|
|
async with db.acquire(role="admin") as conn:
|
|
for user_id, role in user_roles:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.app_user (
|
|
user_id, external_id, email, display_name, role, cohort,
|
|
consent_at, profile_completed_at, terms_agreed_at,
|
|
privacy_agreed_at
|
|
) VALUES ($1,$2,$3,$4,$5,$6,now(),now(),now(),now())
|
|
""",
|
|
user_id,
|
|
f"dev:e2e:g2-trajectory-alert:{user_id}",
|
|
f"{user_id}@g2-trajectory-alert.invalid",
|
|
f"G2 synthetic {role} fixture",
|
|
role,
|
|
COHORT,
|
|
)
|
|
|
|
|
|
async def _create_ended_session(learner: Principal) -> Any:
|
|
catalog = await persona_repository.get_approved_persona("P1")
|
|
if catalog is None:
|
|
raise SmokeError("approved P1 persona is unavailable")
|
|
session = await session_persistence.create_session(
|
|
learner_id=learner.user_id,
|
|
card=catalog.card,
|
|
theory_mode="humanistic",
|
|
state=state_machine.init_state(params=catalog.card.openness_params()),
|
|
persona_id=catalog.persona_id,
|
|
persona_version=catalog.version,
|
|
goal_stages=["라포", "탐색"],
|
|
)
|
|
if session is None:
|
|
raise SmokeError("durable session creation fell back or failed")
|
|
carry = memory.make_carry_over(
|
|
state=session.state,
|
|
session_id=session.session_id,
|
|
case_id=session.case_id,
|
|
session_no=session.session_no,
|
|
masked_turns=[],
|
|
open_threads=[],
|
|
)
|
|
if not await session_persistence.end_session(session, carry):
|
|
raise SmokeError(f"S{session.session_no} durable end failed")
|
|
return session
|
|
|
|
|
|
def _axis(session: dict[str, Any], axis: str) -> dict[str, Any]:
|
|
for item in session.get("axes") or []:
|
|
if item.get("axis") == axis:
|
|
return item
|
|
raise SmokeError(f"S{session.get('session_no')} omitted {axis}")
|
|
|
|
|
|
def _assert_projection(
|
|
payload: dict[str, Any],
|
|
*,
|
|
expected_statuses: tuple[str, ...],
|
|
control: bool,
|
|
) -> None:
|
|
expected_arc = payload.get("expected_arc") or {}
|
|
assessment = payload.get("assessment") or {}
|
|
sessions = assessment.get("sessions") or []
|
|
statuses = tuple(item.get("status") for item in sessions)
|
|
if statuses != expected_statuses:
|
|
raise SmokeError(f"trajectory statuses differ: {statuses!r}")
|
|
if expected_arc.get("data_classification") != "synthetic_educational":
|
|
raise SmokeError("expected arc lost synthetic educational classification")
|
|
if expected_arc.get("clinical_claim_allowed") is not False:
|
|
raise SmokeError("expected arc allowed a clinical claim")
|
|
if assessment.get("data_classification") != "synthetic_educational":
|
|
raise SmokeError("assessment lost synthetic educational classification")
|
|
if assessment.get("clinical_claim_allowed") is not False:
|
|
raise SmokeError("assessment allowed a clinical claim")
|
|
if len(payload.get("observations") or []) != 15:
|
|
raise SmokeError("five-session projection did not expose 15 axis observations")
|
|
if len(payload.get("source_fingerprint") or "") != 64:
|
|
raise SmokeError("projection omitted its source fingerprint")
|
|
if control:
|
|
promoted = ALERT_STATUSES.intersection(statuses)
|
|
if promoted:
|
|
raise SmokeError(f"one-off noise was promoted to an alert: {promoted}")
|
|
noisy_axis = _axis(sessions[2], "distress_load")
|
|
counterevidence = set(noisy_axis.get("counterevidence") or [])
|
|
required = {"inside_off_track_threshold", "deviation_not_yet_sustained"}
|
|
if not required.issubset(counterevidence):
|
|
raise SmokeError(
|
|
"noisy control omitted false-alert counterevidence: "
|
|
f"{sorted(counterevidence)}"
|
|
)
|
|
else:
|
|
deteriorating_axis = _axis(sessions[3], "distress_load")
|
|
if deteriorating_axis.get("status") != "deteriorating":
|
|
raise SmokeError("S4 distress did not become deteriorating")
|
|
if deteriorating_axis.get("adverse_z_change") is None:
|
|
raise SmokeError("deterioration omitted longitudinal change evidence")
|
|
if not sessions[3].get("next_check_questions"):
|
|
raise SmokeError("deterioration omitted next-check questions")
|
|
|
|
|
|
async def _run_case(
|
|
*,
|
|
learner: Principal,
|
|
teacher: Principal,
|
|
profile: tuple[dict[str, float], ...],
|
|
expected_statuses: tuple[str, ...],
|
|
control: bool,
|
|
) -> dict[str, Any]:
|
|
session_ids: list[str] = []
|
|
submitted_measurement_ids: list[str] = []
|
|
case_id = ""
|
|
for expected_no, scores in enumerate(profile, start=1):
|
|
session = await _create_ended_session(learner)
|
|
if session.session_no != expected_no:
|
|
raise SmokeError(
|
|
f"session continuity differs: {session.session_no} != {expected_no}"
|
|
)
|
|
if case_id and session.case_id != case_id:
|
|
raise SmokeError("five sessions did not stay in one case")
|
|
case_id = session.case_id
|
|
session_ids.append(session.session_id)
|
|
created = await outcome_trajectories.create_outcome_observations(
|
|
UUID(session.session_id),
|
|
OutcomeObservationSubmissionRequest(
|
|
submission_id=uuid4(),
|
|
scores=OutcomeAxisValues(**scores),
|
|
confidences=OutcomeAxisValues(**{axis: 0.9 for axis in AXES}),
|
|
),
|
|
learner,
|
|
)
|
|
submitted_measurement_ids.extend(
|
|
str(item) for item in created.submitted_measurement_ids
|
|
)
|
|
|
|
learner_payload = (
|
|
await outcome_trajectories.get_outcome_trajectory(
|
|
UUID(session_ids[-1]), learner
|
|
)
|
|
).model_dump(mode="json")
|
|
teacher_payload = (
|
|
await outcome_trajectories.get_outcome_trajectory(
|
|
UUID(session_ids[-1]), teacher
|
|
)
|
|
).model_dump(mode="json")
|
|
_assert_projection(
|
|
learner_payload, expected_statuses=expected_statuses, control=control
|
|
)
|
|
_assert_projection(
|
|
teacher_payload, expected_statuses=expected_statuses, control=control
|
|
)
|
|
learner_statuses = [
|
|
item["status"] for item in learner_payload["assessment"]["sessions"]
|
|
]
|
|
teacher_statuses = [
|
|
item["status"] for item in teacher_payload["assessment"]["sessions"]
|
|
]
|
|
if learner_statuses != teacher_statuses:
|
|
raise SmokeError("learner and teacher projections disagree")
|
|
if learner_payload["revision_id"] != teacher_payload["revision_id"]:
|
|
raise SmokeError("role-safe reads did not project the same latest revision")
|
|
return {
|
|
"case_id": case_id,
|
|
"session_ids": session_ids,
|
|
"submitted_measurement_ids": submitted_measurement_ids,
|
|
"revision_id": learner_payload["revision_id"],
|
|
"revision_no": learner_payload["revision_no"],
|
|
"source_fingerprint": learner_payload["source_fingerprint"],
|
|
"session_statuses": learner_statuses,
|
|
"learner_teacher_projection_equal": True,
|
|
"next_question_count": len(learner_payload.get("next_questions") or []),
|
|
"alert_promotion_count": len(ALERT_STATUSES.intersection(learner_statuses)),
|
|
}
|
|
|
|
|
|
async def _postgres_proof(case_ids: list[str]) -> dict[str, Any]:
|
|
async with db.acquire(role="admin") as conn:
|
|
measurement = await conn.fetchrow(
|
|
"""
|
|
SELECT count(*)::int AS row_count,
|
|
count(DISTINCT me.measurement_id)::int AS distinct_measurements,
|
|
count(DISTINCT me.session_id)::int AS session_count,
|
|
array_agg(DISTINCT me.source_kind::text ORDER BY me.source_kind::text)
|
|
AS source_kinds,
|
|
array_agg(DISTINCT me.perspective::text ORDER BY me.perspective::text)
|
|
AS perspectives,
|
|
array_agg(DISTINCT me.instrument_id ORDER BY me.instrument_id)
|
|
AS instrument_ids,
|
|
array_agg(DISTINCT me.instrument_version ORDER BY me.instrument_version)
|
|
AS instrument_versions,
|
|
bool_and((me.metadata ->> 'clinical_claim_allowed')::boolean = false)
|
|
AS all_nonclinical,
|
|
bool_and(me.metadata ? 'submission_id' AND me.metadata ? 'submission_hash')
|
|
AS all_have_submission_provenance
|
|
FROM app.measurement_event me
|
|
JOIN app.sessions s ON s.id = me.session_id
|
|
WHERE s.case_id = ANY($1::uuid[])
|
|
AND me.construct = 'session_outcome'
|
|
""",
|
|
[UUID(item) for item in case_ids],
|
|
)
|
|
revisions = await conn.fetch(
|
|
"""
|
|
SELECT case_id, count(*)::int AS revision_count,
|
|
min(revision_no)::int AS first_revision,
|
|
max(revision_no)::int AS last_revision,
|
|
count(DISTINCT source_fingerprint)::int AS fingerprint_count,
|
|
count(*) FILTER (WHERE supersedes_revision_id IS NOT NULL)::int
|
|
AS superseding_revision_count,
|
|
array_agg(DISTINCT computed_role ORDER BY computed_role)
|
|
AS computed_roles
|
|
FROM app.outcome_trajectory_revision
|
|
WHERE case_id = ANY($1::uuid[])
|
|
GROUP BY case_id
|
|
ORDER BY case_id
|
|
""",
|
|
[UUID(item) for item in case_ids],
|
|
)
|
|
latest_observations = await conn.fetchrow(
|
|
"""
|
|
WITH latest AS (
|
|
SELECT DISTINCT ON (case_id) revision_id, case_id
|
|
FROM app.outcome_trajectory_revision
|
|
WHERE case_id = ANY($1::uuid[])
|
|
ORDER BY case_id, revision_no DESC
|
|
)
|
|
SELECT count(*)::int AS row_count,
|
|
count(DISTINCT oto.measurement_id)::int AS distinct_measurements,
|
|
count(DISTINCT oto.session_id)::int AS session_count,
|
|
count(DISTINCT oto.axis)::int AS axis_count,
|
|
bool_and(oto.status = 'observed') AS all_observed,
|
|
bool_and(oto.source_kind = 'learner_reported') AS source_preserved,
|
|
bool_and(oto.perspective = 'learner_self_report')
|
|
AS perspective_preserved
|
|
FROM latest
|
|
JOIN app.outcome_trajectory_observation oto
|
|
ON oto.revision_id = latest.revision_id
|
|
""",
|
|
[UUID(item) for item in case_ids],
|
|
)
|
|
all_linked = await conn.fetchval(
|
|
"""
|
|
SELECT count(DISTINCT oto.measurement_id)::int
|
|
FROM app.outcome_trajectory_observation oto
|
|
JOIN app.outcome_trajectory_revision otr
|
|
ON otr.revision_id = oto.revision_id
|
|
WHERE otr.case_id = ANY($1::uuid[])
|
|
AND oto.measurement_id IS NOT NULL
|
|
""",
|
|
[UUID(item) for item in case_ids],
|
|
)
|
|
|
|
measurement_payload = dict(measurement or {})
|
|
latest_payload = dict(latest_observations or {})
|
|
if measurement_payload.get("row_count") != 30:
|
|
raise SmokeError(f"measurement ledger count differs: {measurement_payload}")
|
|
if measurement_payload.get("session_count") != 10:
|
|
raise SmokeError("measurement ledger did not cover ten real sessions")
|
|
if measurement_payload.get("source_kinds") != ["learner_reported"]:
|
|
raise SmokeError("measurement producer source kind drifted")
|
|
if measurement_payload.get("perspectives") != ["learner_self_report"]:
|
|
raise SmokeError("measurement producer perspective drifted")
|
|
if measurement_payload.get("instrument_ids") != [INSTRUMENT_ID]:
|
|
raise SmokeError("measurement producer instrument id drifted")
|
|
if measurement_payload.get("instrument_versions") != [INSTRUMENT_VERSION]:
|
|
raise SmokeError("measurement producer instrument version drifted")
|
|
if not measurement_payload.get("all_nonclinical"):
|
|
raise SmokeError("a producer row allowed clinical claims")
|
|
if not measurement_payload.get("all_have_submission_provenance"):
|
|
raise SmokeError("a producer row omitted submission provenance")
|
|
revision_payloads = [dict(row) for row in revisions]
|
|
if len(revision_payloads) != 2:
|
|
raise SmokeError("trajectory revision ledger omitted a case")
|
|
for row in revision_payloads:
|
|
if row["revision_count"] != 5 or row["first_revision"] != 1:
|
|
raise SmokeError(f"revision chain is incomplete: {row}")
|
|
if row["last_revision"] != 5 or row["superseding_revision_count"] != 4:
|
|
raise SmokeError(f"revision supersession chain is invalid: {row}")
|
|
if row["fingerprint_count"] != 5 or row["computed_roles"] != ["learner"]:
|
|
raise SmokeError(f"revision provenance drifted: {row}")
|
|
if latest_payload.get("row_count") != 30:
|
|
raise SmokeError("latest revisions do not contain 30 immutable snapshots")
|
|
if latest_payload.get("distinct_measurements") != 30 or all_linked != 30:
|
|
raise SmokeError("measurement-to-observation provenance is incomplete")
|
|
if latest_payload.get("session_count") != 10 or latest_payload.get("axis_count") != 3:
|
|
raise SmokeError("latest observation snapshots lost session/axis coverage")
|
|
if not all(
|
|
latest_payload.get(key)
|
|
for key in ("all_observed", "source_preserved", "perspective_preserved")
|
|
):
|
|
raise SmokeError("observation snapshot provenance drifted")
|
|
|
|
return {
|
|
"measurement_event_count": measurement_payload["row_count"],
|
|
"distinct_measurement_count": measurement_payload["distinct_measurements"],
|
|
"session_count": measurement_payload["session_count"],
|
|
"source_kinds": measurement_payload["source_kinds"],
|
|
"perspectives": measurement_payload["perspectives"],
|
|
"instrument_ids": measurement_payload["instrument_ids"],
|
|
"instrument_versions": measurement_payload["instrument_versions"],
|
|
"all_nonclinical": measurement_payload["all_nonclinical"],
|
|
"all_have_submission_provenance": measurement_payload[
|
|
"all_have_submission_provenance"
|
|
],
|
|
"revision_chains": revision_payloads,
|
|
"latest_observation_snapshot_count": latest_payload["row_count"],
|
|
"all_measurements_linked_to_observation_ledger": all_linked == 30,
|
|
"producer_store_evaluator_boundary_proved": True,
|
|
}
|
|
|
|
|
|
async def run() -> dict[str, Any]:
|
|
await db.init_pool()
|
|
try:
|
|
await persona_repository.materialize_seed_personas()
|
|
worsening_learner_id = uuid4()
|
|
control_learner_id = uuid4()
|
|
teacher_id = uuid4()
|
|
await _seed_users(
|
|
(worsening_learner_id, "learner"),
|
|
(control_learner_id, "learner"),
|
|
(teacher_id, "instructor"),
|
|
)
|
|
teacher = _principal(teacher_id, Role.TEACHER)
|
|
worsening = await _run_case(
|
|
learner=_principal(worsening_learner_id, Role.LEARNER),
|
|
teacher=teacher,
|
|
profile=WORSENING_PROFILE,
|
|
expected_statuses=EXPECTED_WORSENING,
|
|
control=False,
|
|
)
|
|
control = await _run_case(
|
|
learner=_principal(control_learner_id, Role.LEARNER),
|
|
teacher=teacher,
|
|
profile=NOISY_CONTROL_PROFILE,
|
|
expected_statuses=EXPECTED_CONTROL,
|
|
control=True,
|
|
)
|
|
postgres = await _postgres_proof([worsening["case_id"], control["case_id"]])
|
|
return {
|
|
"ok": True,
|
|
"fixture_classification": "synthetic_educational",
|
|
"clinical_claim_allowed": False,
|
|
"notice_ko": "교육용 합성 fixture 검증이며 임상적 효과·예후를 주장하지 않는다.",
|
|
"fixture_policy": (
|
|
"retained unique dev:e2e rows in expendable development DB; "
|
|
"append-only ledger"
|
|
),
|
|
"entrypoint": (
|
|
"outcome_trajectories.create_outcome_observations -> "
|
|
"outcome_trajectory_store measurement producer -> evaluator"
|
|
),
|
|
"direct_measurement_or_trajectory_insert_used": False,
|
|
"source_values_in_evidence_json": False,
|
|
"worsening_case": worsening,
|
|
"noisy_one_off_control": control,
|
|
"postgres": postgres,
|
|
}
|
|
finally:
|
|
await db.close_pool()
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--out", default="")
|
|
args = parser.parse_args()
|
|
result = asyncio.run(run())
|
|
output = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
|
if args.out:
|
|
path = Path(args.out)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(output + "\n", encoding="utf-8")
|
|
print(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|