16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
584 lines
21 KiB
Python
584 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove the automatic G3 learner-turn -> durable rupture chain on PostgreSQL.
|
|
|
|
This smoke calls the real ``sessions.submit_turn`` and ``sessions.end_session``
|
|
route functions. Only the external client engine and fast evaluator are replaced
|
|
with deterministic in-process seams; no rupture observation/reconciliation write
|
|
endpoint or store append function is called by the smoke.
|
|
|
|
Run only against an expendable development database. Unique fixture rows are
|
|
retained because the rupture ledger is intentionally append-only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
from unittest.mock import AsyncMock, patch
|
|
from uuid import UUID, uuid4
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
API_ROOT = REPO_ROOT / "apps" / "api"
|
|
API_ENV = API_ROOT / ".env"
|
|
|
|
|
|
def _load_api_env() -> None:
|
|
if not API_ENV.exists():
|
|
return
|
|
for raw_line in API_ENV.read_text(encoding="utf-8").splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
|
|
|
|
|
_load_api_env()
|
|
if str(API_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(API_ROOT))
|
|
|
|
from app import db, persona_repository, session_persistence # noqa: E402
|
|
from app.contracts.engine_gateway import GenerateResponse # noqa: E402
|
|
from app.deps import Principal, Role # noqa: E402
|
|
from app.routes import rupture_repairs, sessions # noqa: E402
|
|
from app.services import memory, rupture_runtime, state_machine # noqa: E402
|
|
from app.store import InProcSession, store # noqa: E402
|
|
|
|
|
|
COHORT = "g3-runtime-auto-smoke"
|
|
TerminalStatus = Literal["missed", "partial", "resolved"]
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _principal(user_id: UUID, role: Role) -> Principal:
|
|
return Principal(
|
|
user_id=str(user_id),
|
|
role=role,
|
|
cohort_ids=[COHORT],
|
|
consent_at=1.0,
|
|
profile_completed_at=1.0,
|
|
)
|
|
|
|
|
|
async def _seed_users(learner_id: UUID, teacher_id: UUID) -> None:
|
|
async with db.acquire(role="admin") as conn:
|
|
for user_id, role, label in (
|
|
(learner_id, "learner", "Learner"),
|
|
(teacher_id, "instructor", "Teacher"),
|
|
):
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.app_user (
|
|
user_id, external_id, email, display_name, role, cohort,
|
|
consent_at, profile_completed_at, terms_agreed_at,
|
|
privacy_agreed_at
|
|
) VALUES ($1,$2,$3,$4,$5,$6,now(),now(),now(),now())
|
|
""",
|
|
user_id,
|
|
f"dev:e2e:g3-runtime:{user_id}",
|
|
f"{user_id}@g3-runtime-smoke.invalid",
|
|
f"G3 Runtime {label}",
|
|
role,
|
|
COHORT,
|
|
)
|
|
|
|
|
|
async def _create_session(learner: Principal) -> InProcSession:
|
|
await persona_repository.materialize_seed_personas()
|
|
catalog = await persona_repository.get_approved_persona("P1")
|
|
if catalog is None:
|
|
raise SmokeError("approved P1 persona is unavailable")
|
|
state = state_machine.init_state(params=catalog.card.openness_params())
|
|
session = await session_persistence.create_session(
|
|
learner_id=learner.user_id,
|
|
card=catalog.card,
|
|
theory_mode="humanistic",
|
|
state=state,
|
|
persona_id=catalog.persona_id,
|
|
persona_version=catalog.version,
|
|
goal_stages=["라포", "탐색"],
|
|
)
|
|
if session is None:
|
|
raise SmokeError("durable session creation fell back or failed")
|
|
store.put(session)
|
|
sessions._RECALL_CACHE[session.session_id] = memory.RecallContext()
|
|
return session
|
|
|
|
|
|
def _tag(code: str) -> dict[str, str]:
|
|
return {
|
|
"code": code,
|
|
"label_ko": code,
|
|
"category": "g3-runtime-smoke",
|
|
"rationale": "deterministic structured smoke evidence",
|
|
}
|
|
|
|
|
|
def _state_tag(code: str) -> dict[str, str]:
|
|
return {
|
|
"code": code,
|
|
"label_ko": code,
|
|
"rationale": "deterministic structured smoke evidence",
|
|
}
|
|
|
|
|
|
def _rupture_evaluation(*, turn_seq: int, stage: str) -> dict[str, Any]:
|
|
return {
|
|
"loop": "fast",
|
|
"turn_seq": turn_seq,
|
|
"stage": stage,
|
|
"techniques": [],
|
|
"client_state_read": [_state_tag("defensive")],
|
|
"appropriateness": "warn",
|
|
"appropriateness_note": "공감 의도와 실제 반응이 어긋남",
|
|
"rapport_signal": -0.4,
|
|
"theory_mode": "humanistic",
|
|
"intent_deviation": {
|
|
"dimension": "empathy",
|
|
"expected": "client affect attunement",
|
|
"actual": "premature interpretation",
|
|
"severity": "major",
|
|
},
|
|
}
|
|
|
|
|
|
def _repair_evaluation(
|
|
*, turn_seq: int, stage: str, target: TerminalStatus
|
|
) -> dict[str, Any]:
|
|
if target == "partial":
|
|
techniques = [_tag("clarification")]
|
|
client_states = [_state_tag("thought_organizing")]
|
|
elif target == "resolved":
|
|
techniques = [_tag("empathy"), _tag("clarification")]
|
|
client_states = [_state_tag("defense_loosening")]
|
|
else:
|
|
raise SmokeError("missed sessions must end without a repair turn")
|
|
return {
|
|
"loop": "fast",
|
|
"turn_seq": turn_seq,
|
|
"stage": stage,
|
|
"techniques": techniques,
|
|
"client_state_read": client_states,
|
|
"appropriateness": "pos",
|
|
"appropriateness_note": "영향을 확인하고 후속 반응을 탐색함",
|
|
"rapport_signal": 0.5,
|
|
"theory_mode": "humanistic",
|
|
}
|
|
|
|
|
|
async def _await_scan(
|
|
session_id: str,
|
|
*,
|
|
expected_status: Literal["recorded", "reconciled"],
|
|
expected_trigger: str,
|
|
) -> rupture_runtime.RuptureRuntimeResult:
|
|
matching = [
|
|
task
|
|
for task in tuple(rupture_runtime._RUNTIME_TASKS)
|
|
if session_id in task.get_name()
|
|
]
|
|
if not matching:
|
|
raise SmokeError(
|
|
f"automatic runtime callback did not schedule a scan for {session_id}"
|
|
)
|
|
await asyncio.gather(*matching)
|
|
result = rupture_runtime.last_runtime_result(session_id)
|
|
if result is None:
|
|
raise SmokeError(f"automatic runtime scan omitted its result for {session_id}")
|
|
if result.status != expected_status or result.trigger != expected_trigger:
|
|
raise SmokeError(
|
|
"unexpected runtime result: "
|
|
f"status={result.status} trigger={result.trigger} "
|
|
f"expected={expected_status}/{expected_trigger} error={result.error_code}"
|
|
)
|
|
return result
|
|
|
|
|
|
async def _run_session_chain(
|
|
*,
|
|
learner: Principal,
|
|
target: TerminalStatus,
|
|
evaluations: dict[str, TerminalStatus],
|
|
) -> tuple[InProcSession, list[dict[str, Any]]]:
|
|
session = await _create_session(learner)
|
|
evaluations[session.session_id] = target
|
|
runtime_results: list[dict[str, Any]] = []
|
|
|
|
await sessions.submit_turn(
|
|
session.session_id,
|
|
sessions.TurnRequest(text="그 정도 일은 누구나 겪으니 크게 볼 필요는 없어요."),
|
|
learner,
|
|
)
|
|
detected = await _await_scan(
|
|
session.session_id,
|
|
expected_status="recorded",
|
|
expected_trigger="turn_persisted",
|
|
)
|
|
runtime_results.append(asdict(detected))
|
|
|
|
if target == "missed":
|
|
await sessions.end_session(session.session_id, learner)
|
|
reconciled = await _await_scan(
|
|
session.session_id,
|
|
expected_status="reconciled",
|
|
expected_trigger="session_ended",
|
|
)
|
|
else:
|
|
await sessions.submit_turn(
|
|
session.session_id,
|
|
sessions.TurnRequest(
|
|
text="내가 서둘러 해석해서 답답했을 수 있겠네요. "
|
|
"지금 느낌을 조금 더 확인해도 괜찮을까요?"
|
|
),
|
|
learner,
|
|
)
|
|
reconciled = await _await_scan(
|
|
session.session_id,
|
|
expected_status="reconciled",
|
|
expected_trigger="turn_persisted",
|
|
)
|
|
runtime_results.append(asdict(reconciled))
|
|
return session, runtime_results
|
|
|
|
|
|
def _assert_no_total_score(value: Any, path: str = "response") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden = {"total", "total_score", "overall_score"} & set(value)
|
|
if forbidden:
|
|
raise SmokeError(f"{path} exposed total-score keys: {forbidden}")
|
|
for key, child in value.items():
|
|
_assert_no_total_score(child, f"{path}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, child in enumerate(value):
|
|
_assert_no_total_score(child, f"{path}[{index}]")
|
|
|
|
|
|
async def _read_role_proof(
|
|
*,
|
|
session_id: str,
|
|
learner: Principal,
|
|
teacher: Principal,
|
|
expected_status: TerminalStatus,
|
|
) -> dict[str, Any]:
|
|
learner_model = await rupture_repairs.get_rupture_repairs(
|
|
UUID(session_id), learner
|
|
)
|
|
teacher_model = await rupture_repairs.get_rupture_repairs(
|
|
UUID(session_id), teacher
|
|
)
|
|
learner_payload = learner_model.model_dump(mode="json")
|
|
teacher_payload = teacher_model.model_dump(mode="json")
|
|
for payload, view in (
|
|
(learner_payload, "counselor"),
|
|
(teacher_payload, "supervisor"),
|
|
):
|
|
if payload["requested_view"] != view:
|
|
raise SmokeError(f"role-safe read used {payload['requested_view']} not {view}")
|
|
if payload["clinical_claim_allowed"] is not False:
|
|
raise SmokeError("rupture read escaped the non-clinical boundary")
|
|
if len(payload["episodes"]) != 1:
|
|
raise SmokeError("automatic runtime smoke expected exactly one episode")
|
|
if payload["episodes"][0]["current_status"] != expected_status:
|
|
raise SmokeError(
|
|
f"read status {payload['episodes'][0]['current_status']} "
|
|
f"did not match {expected_status}"
|
|
)
|
|
_assert_no_total_score(payload)
|
|
if (
|
|
learner_payload["episodes"][0]["episode_id"]
|
|
!= teacher_payload["episodes"][0]["episode_id"]
|
|
):
|
|
raise SmokeError("learner and teacher projections disagree on episode identity")
|
|
return {
|
|
"episode_id": learner_payload["episodes"][0]["episode_id"],
|
|
"learner_view": learner_payload["requested_view"],
|
|
"teacher_view": teacher_payload["requested_view"],
|
|
"current_status": learner_payload["episodes"][0]["current_status"],
|
|
"status_source": learner_payload["episodes"][0]["status_source"],
|
|
"observation_count": len(learner_payload["episodes"][0]["observations"]),
|
|
"reconciliation_count": len(
|
|
learner_payload["episodes"][0]["reconciliation_revisions"]
|
|
),
|
|
}
|
|
|
|
|
|
async def _postgres_proof(
|
|
session_ids: dict[TerminalStatus, str],
|
|
) -> dict[str, Any]:
|
|
async with db.acquire(role="admin") as conn:
|
|
episode_rows = await conn.fetch(
|
|
"""
|
|
SELECT episode_id, session_id, episode_key, learner_id
|
|
FROM app.rupture_episode
|
|
WHERE session_id = ANY($1::uuid[])
|
|
ORDER BY session_id, created_at
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
observation_rows = await conn.fetch(
|
|
"""
|
|
SELECT observation_id, episode_id, session_id, sequence_no, event_kind,
|
|
to_state, source_kind, perspective, ai_view, evidence_turn_ids,
|
|
model_run_id
|
|
FROM app.rupture_observation_event
|
|
WHERE session_id = ANY($1::uuid[])
|
|
ORDER BY session_id, episode_id, sequence_no
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
reconciliation_rows = await conn.fetch(
|
|
"""
|
|
SELECT episode_id, session_id, revision_no, deep_status, disposition,
|
|
evidence_turn_ids, model_run_id
|
|
FROM app.rupture_reconciliation_revision
|
|
WHERE session_id = ANY($1::uuid[])
|
|
ORDER BY session_id, episode_id, revision_no
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
runtime_model_rows = await conn.fetch(
|
|
"""
|
|
SELECT session_id, model_run_id, model, status, metadata
|
|
FROM audit.model_run
|
|
WHERE session_id = ANY($1::uuid[])
|
|
AND model = 'rupture-runtime-deterministic'
|
|
ORDER BY session_id, created_at, model_run_id
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
invalid_evidence = await conn.fetchval(
|
|
"""
|
|
SELECT count(*)
|
|
FROM app.rupture_observation_event observation
|
|
CROSS JOIN LATERAL unnest(observation.evidence_turn_ids) evidence(turn_id)
|
|
WHERE observation.session_id = ANY($1::uuid[])
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM app.turns turn_row
|
|
WHERE turn_row.id = evidence.turn_id
|
|
AND turn_row.session_id = observation.session_id
|
|
)
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
counselor_evaluation_count = await conn.fetchval(
|
|
"""
|
|
SELECT count(DISTINCT turn_row.id)
|
|
FROM app.turns turn_row
|
|
JOIN app.feedback_scores score ON score.turn_id = turn_row.id
|
|
WHERE turn_row.session_id = ANY($1::uuid[])
|
|
AND turn_row.speaker = 'counselor'
|
|
AND score.dimension = 'appropriateness'
|
|
""",
|
|
list(session_ids.values()),
|
|
)
|
|
|
|
if len(episode_rows) != 3:
|
|
raise SmokeError(f"Postgres stored {len(episode_rows)} episodes, expected 3")
|
|
if len(reconciliation_rows) != 3:
|
|
raise SmokeError(
|
|
f"Postgres stored {len(reconciliation_rows)} reconciliations, expected 3"
|
|
)
|
|
if int(invalid_evidence or 0) != 0:
|
|
raise SmokeError("runtime ledger references evidence turns outside its session")
|
|
if int(counselor_evaluation_count or 0) != 5:
|
|
raise SmokeError("all five learner turns did not retain structured evaluations")
|
|
|
|
status_by_session = {
|
|
str(row["session_id"]): str(row["deep_status"])
|
|
for row in reconciliation_rows
|
|
}
|
|
expected_by_session = {
|
|
session_id: status for status, session_id in session_ids.items()
|
|
}
|
|
if status_by_session != expected_by_session:
|
|
raise SmokeError(
|
|
f"Postgres reconciliation states differ: {status_by_session}"
|
|
)
|
|
|
|
events_by_session: dict[str, list[str]] = {}
|
|
for row in observation_rows:
|
|
session_id = str(row["session_id"])
|
|
events_by_session.setdefault(session_id, []).append(str(row["event_kind"]))
|
|
if row["source_kind"] != "model_inferred":
|
|
raise SmokeError("automatic runtime emitted a non-model-inferred observation")
|
|
if row["perspective"] != "independent_observer" or row["ai_view"] != "evaluator":
|
|
raise SmokeError("automatic runtime emitted invalid evaluator provenance")
|
|
if row["model_run_id"] is None:
|
|
raise SmokeError("automatic runtime observation omitted model provenance")
|
|
|
|
expected_events = {
|
|
session_ids["missed"]: ["rupture.detected", "rupture.missed"],
|
|
session_ids["partial"]: [
|
|
"rupture.detected",
|
|
"rupture.recognized",
|
|
"repair.attempted",
|
|
"repair.partial",
|
|
],
|
|
session_ids["resolved"]: [
|
|
"rupture.detected",
|
|
"rupture.recognized",
|
|
"repair.attempted",
|
|
"repair.resolved",
|
|
],
|
|
}
|
|
if events_by_session != expected_events:
|
|
raise SmokeError(f"unexpected append-only event chains: {events_by_session}")
|
|
|
|
model_runs_by_session: dict[str, int] = {}
|
|
for row in runtime_model_rows:
|
|
session_id = str(row["session_id"])
|
|
model_runs_by_session[session_id] = model_runs_by_session.get(session_id, 0) + 1
|
|
if row["status"] != "ready":
|
|
raise SmokeError("runtime detector model provenance is not ready")
|
|
if set(model_runs_by_session.values()) != {2}:
|
|
raise SmokeError(f"expected fast+deep provenance per session: {model_runs_by_session}")
|
|
|
|
return {
|
|
"episode_count": len(episode_rows),
|
|
"observation_count": len(observation_rows),
|
|
"reconciliation_count": len(reconciliation_rows),
|
|
"structured_learner_turn_count": int(counselor_evaluation_count or 0),
|
|
"invalid_evidence_turn_reference_count": int(invalid_evidence or 0),
|
|
"event_kinds_by_session": events_by_session,
|
|
"deep_status_by_session": status_by_session,
|
|
"runtime_model_runs_by_session": model_runs_by_session,
|
|
"runtime_episode_keys": [str(row["episode_key"]) for row in episode_rows],
|
|
}
|
|
|
|
|
|
async def run() -> dict[str, Any]:
|
|
learner_id = uuid4()
|
|
teacher_id = uuid4()
|
|
learner = _principal(learner_id, Role.LEARNER)
|
|
teacher = _principal(teacher_id, Role.TEACHER)
|
|
evaluations: dict[str, TerminalStatus] = {}
|
|
generated_reply_counts: dict[str, int] = {}
|
|
|
|
await db.init_pool()
|
|
try:
|
|
await _seed_users(learner_id, teacher_id)
|
|
|
|
async def fake_generate(request: Any) -> GenerateResponse:
|
|
session_id = str(request.session_id or "")
|
|
reply_no = generated_reply_counts.get(session_id, 0) + 1
|
|
generated_reply_counts[session_id] = reply_no
|
|
replies = (
|
|
"그렇게 가볍게 말씀하시면 더 이야기하고 싶지 않아져요.",
|
|
"아까는 마음이 닫혔는데, 지금은 조금 정리해서 말해볼 수 있을 것 같아요.",
|
|
)
|
|
return GenerateResponse(
|
|
text=replies[min(reply_no - 1, len(replies) - 1)],
|
|
model="g3-runtime-smoke-client",
|
|
provider="codex_cli",
|
|
tokens_in=17,
|
|
tokens_out=13,
|
|
cost_usd=0.0,
|
|
)
|
|
|
|
async def fake_evaluate(ctx: Any, _reply: str) -> dict[str, Any]:
|
|
if ctx.state_after is None:
|
|
raise SmokeError("turn context omitted state_after")
|
|
target = evaluations[ctx.session_id]
|
|
if ctx.state_after.turn_seq == 1:
|
|
return _rupture_evaluation(
|
|
turn_seq=ctx.state_after.turn_seq,
|
|
stage=ctx.state_after.stage.value,
|
|
)
|
|
return _repair_evaluation(
|
|
turn_seq=ctx.state_after.turn_seq,
|
|
stage=ctx.state_after.stage.value,
|
|
target=target,
|
|
)
|
|
|
|
async def no_op_async(*_args: Any, **_kwargs: Any) -> None:
|
|
return None
|
|
|
|
runtime_by_status: dict[TerminalStatus, list[dict[str, Any]]] = {}
|
|
session_ids: dict[TerminalStatus, str] = {}
|
|
read_models: dict[TerminalStatus, dict[str, Any]] = {}
|
|
with (
|
|
patch.object(
|
|
sessions.engine_client,
|
|
"generate",
|
|
new=AsyncMock(side_effect=fake_generate),
|
|
),
|
|
patch.object(
|
|
sessions.evaluator,
|
|
"make_eval_hook",
|
|
return_value=fake_evaluate,
|
|
),
|
|
patch.object(sessions, "_schedule_session_evaluation", return_value=None),
|
|
patch.object(
|
|
sessions,
|
|
"_write_episodic_embeddings",
|
|
new=AsyncMock(side_effect=no_op_async),
|
|
),
|
|
patch.object(
|
|
sessions.engine_client,
|
|
"close_session",
|
|
new=AsyncMock(return_value=True),
|
|
),
|
|
):
|
|
for target in ("missed", "partial", "resolved"):
|
|
session, runtime_results = await _run_session_chain(
|
|
learner=learner,
|
|
target=target,
|
|
evaluations=evaluations,
|
|
)
|
|
session_ids[target] = session.session_id
|
|
runtime_by_status[target] = runtime_results
|
|
|
|
for target, session_id in session_ids.items():
|
|
read_models[target] = await _read_role_proof(
|
|
session_id=session_id,
|
|
learner=learner,
|
|
teacher=teacher,
|
|
expected_status=target,
|
|
)
|
|
|
|
await asyncio.sleep(0)
|
|
|
|
postgres = await _postgres_proof(session_ids)
|
|
return {
|
|
"ok": True,
|
|
"detector_version": rupture_runtime.RUNTIME_DETECTOR_VERSION,
|
|
"fixture_policy": "retained unique dev:e2e rows in expendable development DB",
|
|
"entrypoint": "sessions.submit_turn/finalize_completed_turn hook",
|
|
"manual_rupture_write_endpoint_used": False,
|
|
"learner_id": str(learner_id),
|
|
"teacher_id": str(teacher_id),
|
|
"session_ids": session_ids,
|
|
"runtime_callback_results": runtime_by_status,
|
|
"read_models": read_models,
|
|
"postgres": postgres,
|
|
}
|
|
finally:
|
|
await db.close_pool()
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--out", default="")
|
|
args = parser.parse_args()
|
|
result = asyncio.run(run())
|
|
output = json.dumps(result, ensure_ascii=False, indent=2)
|
|
if args.out:
|
|
path = Path(args.out)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(output + "\n", encoding="utf-8")
|
|
print(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|