"""Prove the scheduled G6 repo benchmark comparison on a real dev PostgreSQL. The bootstrap creates metadata-only synthetic measurement anchors through the owner connection inside the local dev DB container. The actual comparison is then written twice through the runtime app role to prove first append, idempotent replay, hydrated subgroup drift, and zero duplicate rows. """ from __future__ import annotations import argparse import asyncio import json import subprocess import sys from pathlib import Path from typing import Any from uuid import UUID, uuid5 import asyncpg REPO_ROOT = Path(__file__).resolve().parents[1] API_ROOT = REPO_ROOT / "apps" / "api" if str(API_ROOT) not in sys.path: sys.path.insert(0, str(API_ROOT)) from app.services import ( # noqa: E402 supervision_research_store, supervision_research_version_evaluator as evaluator, ) FIXTURE_NAMESPACE = UUID("46838644-191a-42c6-9737-dc1b9fbe6e63") FIXTURE_COHORT = "g6-repo-benchmark-v1" FIXTURE_LEARNER_ID = uuid5(FIXTURE_NAMESPACE, "learner") FIXTURE_SESSION_ID = uuid5(FIXTURE_NAMESPACE, "session") FIXTURE_INSTRUMENT_ID = "g6-repo-version-comparison" FIXTURE_INSTRUMENT_VERSION = "1.0.0" class SmokeError(RuntimeError): pass def _parse_env(path: Path) -> dict[str, str]: values: dict[str, str] = {} for raw in path.read_text(encoding="utf-8-sig").splitlines(): line = raw.strip() if not line or line.startswith("#") or "=" not in line: continue key, value = line.split("=", 1) values[key.strip()] = value.strip().strip('"').strip("'") return values def _sql_literal(value: str) -> str: return "'" + value.replace("'", "''") + "'" def _bootstrap_sql() -> str: benchmark = evaluator.load_validated_repository_benchmark() statements = [ "\\set ON_ERROR_STOP on", "INSERT INTO app.app_user " "(user_id, external_id, email, display_name, role, cohort) VALUES (" f"'{FIXTURE_LEARNER_ID}'," "'dev:g6-repo-benchmark-v1'," "'dev.g6.repo.benchmark.v1@example.invalid'," "'G6 Repo Benchmark V1','learner'," f"'{FIXTURE_COHORT}') ON CONFLICT (user_id) DO NOTHING;", "INSERT INTO app.sessions (id, learner_id, theory_mode) VALUES (" f"'{FIXTURE_SESSION_ID}','{FIXTURE_LEARNER_ID}','integrative') " "ON CONFLICT (id) DO NOTHING;", "INSERT INTO app.measurement_instrument (" "instrument_id,instrument_version,name_ko,instrument_kind,construct," "validation_basis,scoring_schema,metadata) VALUES (" f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}'," "'G6 저장소 승인 합성 평가기 비교','runtime_metric','counselor_skill'," "'repo-approved synthetic benchmark only'," '\'{"min":0,"max":1}\',' '\'{"data_classification":"synthetic_educational",' '"clinical_claim_allowed":false}\') ' "ON CONFLICT (instrument_id,instrument_version) DO NOTHING;", ] for batch in ( benchmark.pack.baseline_batch, benchmark.pack.candidate_batch, ): for observation in batch.observations: measurement_id = uuid5( FIXTURE_NAMESPACE, f"measurement:{observation.evidence_event_id}" ) metadata = evaluator.benchmark_anchor_metadata( benchmark, batch, observation ) metadata_json = json.dumps( metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) value = ( 1.0 if observation.predicted_label == observation.gold_label else 0.0 ) statements.append( "INSERT INTO app.measurement_event (" "measurement_id,session_id,construct,dimension,perspective,source_kind," "instrument_id,instrument_version,value,scale_min,scale_max,status," "visible_to,metadata) VALUES (" f"'{measurement_id}','{FIXTURE_SESSION_ID}','counselor_skill'," f"{_sql_literal('g6.' + observation.evidence_event_id)}," "'runtime_observation','observed_runtime'," f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}'," f"{value},0,1,'ready','{{evaluator,supervisor,research}}'," f"{_sql_literal(metadata_json)}::jsonb) " "ON CONFLICT (measurement_id) DO NOTHING;" ) return "\n".join(statements) + "\n" def _bootstrap_fixture(args: argparse.Namespace) -> None: result = subprocess.run( [ "docker", "exec", "-i", args.container, "psql", "-v", "ON_ERROR_STOP=1", "-U", args.owner_user, "-d", args.database, ], input=_bootstrap_sql().encode("utf-8"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) if result.returncode: detail = result.stderr.decode("utf-8", errors="replace")[-2000:] raise SmokeError(f"dev DB synthetic anchor bootstrap failed: {detail}") async def _set_research_context(conn: asyncpg.Connection) -> None: await conn.execute("SELECT set_config('app.ai_context','1',true)") await conn.execute("SELECT set_config('app.current_ai_view','research',true)") async def _counts(conn: asyncpg.Connection) -> dict[str, int]: return { "batches": int( await conn.fetchval( """ SELECT count(*) FROM app.supervision_evaluation_batch WHERE batch_id = ANY($1::text[]) """, ["oas-g6-batch-baseline", "oas-g6-batch-candidate"], ) ), "reports": int( await conn.fetchval( "SELECT count(*) FROM app.supervision_drift_report WHERE cohort_id=$1", FIXTURE_COHORT, ) ), "subgroup_metrics": int( await conn.fetchval( """ SELECT count(*) FROM app.supervision_drift_subgroup_metric m JOIN app.supervision_drift_report r ON r.drift_report_id=m.drift_report_id WHERE r.cohort_id=$1 """, FIXTURE_COHORT, ) ), } async def _produce_once( conn: asyncpg.Connection, ) -> tuple[dict[str, Any], dict[str, int]]: async with conn.transaction(): await _set_research_context(conn) result = await evaluator.produce_repository_version_comparison(conn) counts = await _counts(conn) return result, counts async def _run(args: argparse.Namespace) -> dict[str, Any]: env = _parse_env(Path(args.env_file)) database_url = env.get("DATABASE_URL", "") if not database_url: raise SmokeError("DATABASE_URL is missing from the selected env file") _bootstrap_fixture(args) conn = await asyncpg.connect(database_url) try: async with conn.transaction(): await _set_research_context(conn) before = await _counts(conn) if not args.allow_existing and any(before.values()): raise SmokeError(f"expected first append but rows already exist: {before}") first, after_first = await _produce_once(conn) replay, after_replay = await _produce_once(conn) async with conn.transaction(): await _set_research_context(conn) research_view = await supervision_research_store.read_research_view(conn) finally: await conn.close() if not args.allow_existing and first.get("idempotent_replay") is not False: raise SmokeError("first comparison did not append a new report") if replay.get("idempotent_replay") is not True: raise SmokeError("second comparison was not an idempotent replay") if ( first.get("status") != "drift_flagged" or replay.get("status") != "drift_flagged" ): raise SmokeError("repository comparison lost expected drift status") if after_first != {"batches": 2, "reports": 1, "subgroup_metrics": 2}: raise SmokeError(f"unexpected first append cardinality: {after_first}") if after_replay != after_first: raise SmokeError("idempotent replay created duplicate rows") drift_report_id = str(first["drift_report_id"]) hydrated = next( ( item for item in research_view["drift_reports"] if str(item["drift_report_id"]) == drift_report_id ), None, ) if hydrated is None: raise SmokeError("research read model did not hydrate the drift report") if hydrated.get("status") != "drift_flagged": raise SmokeError("hydrated drift report status changed") subgroup_metrics = hydrated.get("subgroup_metrics") or [] if {item["subgroup"] for item in subgroup_metrics} != { "synthetic-a", "synthetic-b", }: raise SmokeError("hydrated drift report omitted synthetic subgroup metrics") required_provenance = { "baseline_model", "candidate_model", "baseline_prompt_version", "candidate_prompt_version", "instrument_id", "baseline_instrument_version", "candidate_instrument_version", } if any(not hydrated.get(key) for key in required_provenance): raise SmokeError("hydrated drift report omitted evaluator provenance") return { "ok": True, "database": "actual dev PostgreSQL via runtime app role", "fixture_cohort": FIXTURE_COHORT, "benchmark_schema_version": first["benchmark_schema_version"], "benchmark_version": first["benchmark_version"], "benchmark_content_sha256": first["benchmark_content_sha256"], "first_append": { "idempotent_replay": first["idempotent_replay"], "drift_report_id": drift_report_id, "status": first["status"], "counts": after_first, }, "replay": { "idempotent_replay": replay["idempotent_replay"], "counts": after_replay, "duplicate_rows": sum( after_replay[key] - after_first[key] for key in after_first ), }, "hydrated": { "status": hydrated["status"], "baseline_model": hydrated["baseline_model"], "candidate_model": hydrated["candidate_model"], "baseline_prompt_version": hydrated["baseline_prompt_version"], "candidate_prompt_version": hydrated["candidate_prompt_version"], "instrument_id": hydrated["instrument_id"], "baseline_instrument_version": hydrated["baseline_instrument_version"], "candidate_instrument_version": hydrated["candidate_instrument_version"], "subgroup_metrics": subgroup_metrics, }, "data_classification": "synthetic_educational", "raw_transcript_included": False, "clinical_claim_allowed": False, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--env-file", default="apps/api/.env") parser.add_argument("--container", default="vignette-dev-db") parser.add_argument("--owner-user", default="vignette_owner") parser.add_argument("--database", default="vignette") parser.add_argument("--allow-existing", action="store_true") parser.add_argument("--out", default="") args = parser.parse_args() result = asyncio.run(_run(args)) text = json.dumps(result, ensure_ascii=False, indent=2, default=str) if args.out: output = Path(args.out) output.parent.mkdir(parents=True, exist_ok=True) output.write_text(text + "\n", encoding="utf-8") print(text) if __name__ == "__main__": main()