vignette/scripts/smoke-supervision-version-comparison-producer.py

"""Prove the scheduled G6 repo benchmark comparison on a real dev PostgreSQL.

The bootstrap creates metadata-only synthetic measurement anchors through the
owner connection inside the local dev DB container.  The actual comparison is
then written twice through the runtime app role to prove first append,
idempotent replay, hydrated subgroup drift, and zero duplicate rows.
"""

from __future__ import annotations

import argparse
import asyncio
import json
import subprocess
import sys
from pathlib import Path
from typing import Any
from uuid import UUID, uuid5

import asyncpg


REPO_ROOT = Path(__file__).resolve().parents[1]
API_ROOT = REPO_ROOT / "apps" / "api"
if str(API_ROOT) not in sys.path:
    sys.path.insert(0, str(API_ROOT))

from app.services import (  # noqa: E402
    supervision_research_store,
    supervision_research_version_evaluator as evaluator,
)


FIXTURE_NAMESPACE = UUID("46838644-191a-42c6-9737-dc1b9fbe6e63")
FIXTURE_COHORT = "g6-repo-benchmark-v1"
FIXTURE_LEARNER_ID = uuid5(FIXTURE_NAMESPACE, "learner")
FIXTURE_SESSION_ID = uuid5(FIXTURE_NAMESPACE, "session")
FIXTURE_INSTRUMENT_ID = "g6-repo-version-comparison"
FIXTURE_INSTRUMENT_VERSION = "1.0.0"


class SmokeError(RuntimeError):
    pass


def _parse_env(path: Path) -> dict[str, str]:
    values: dict[str, str] = {}
    for raw in path.read_text(encoding="utf-8-sig").splitlines():
        line = raw.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, value = line.split("=", 1)
        values[key.strip()] = value.strip().strip('"').strip("'")
    return values


def _sql_literal(value: str) -> str:
    return "'" + value.replace("'", "''") + "'"


def _bootstrap_sql() -> str:
    benchmark = evaluator.load_validated_repository_benchmark()
    statements = [
        "\\set ON_ERROR_STOP on",
        "INSERT INTO app.app_user "
        "(user_id, external_id, email, display_name, role, cohort) VALUES ("
        f"'{FIXTURE_LEARNER_ID}',"
        "'dev:g6-repo-benchmark-v1',"
        "'dev.g6.repo.benchmark.v1@example.invalid',"
        "'G6 Repo Benchmark V1','learner',"
        f"'{FIXTURE_COHORT}') ON CONFLICT (user_id) DO NOTHING;",
        "INSERT INTO app.sessions (id, learner_id, theory_mode) VALUES ("
        f"'{FIXTURE_SESSION_ID}','{FIXTURE_LEARNER_ID}','integrative') "
        "ON CONFLICT (id) DO NOTHING;",
        "INSERT INTO app.measurement_instrument ("
        "instrument_id,instrument_version,name_ko,instrument_kind,construct,"
        "validation_basis,scoring_schema,metadata) VALUES ("
        f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}',"
        "'G6 저장소 승인 합성 평가기 비교','runtime_metric','counselor_skill',"
        "'repo-approved synthetic benchmark only',"
        '\'{"min":0,"max":1}\','
        '\'{"data_classification":"synthetic_educational",'
        '"clinical_claim_allowed":false}\') '
        "ON CONFLICT (instrument_id,instrument_version) DO NOTHING;",
    ]
    for batch in (
        benchmark.pack.baseline_batch,
        benchmark.pack.candidate_batch,
    ):
        for observation in batch.observations:
            measurement_id = uuid5(
                FIXTURE_NAMESPACE, f"measurement:{observation.evidence_event_id}"
            )
            metadata = evaluator.benchmark_anchor_metadata(
                benchmark, batch, observation
            )
            metadata_json = json.dumps(
                metadata,
                ensure_ascii=False,
                sort_keys=True,
                separators=(",", ":"),
            )
            value = (
                1.0 if observation.predicted_label == observation.gold_label else 0.0
            )
            statements.append(
                "INSERT INTO app.measurement_event ("
                "measurement_id,session_id,construct,dimension,perspective,source_kind,"
                "instrument_id,instrument_version,value,scale_min,scale_max,status,"
                "visible_to,metadata) VALUES ("
                f"'{measurement_id}','{FIXTURE_SESSION_ID}','counselor_skill',"
                f"{_sql_literal('g6.' + observation.evidence_event_id)},"
                "'runtime_observation','observed_runtime',"
                f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}',"
                f"{value},0,1,'ready','{{evaluator,supervisor,research}}',"
                f"{_sql_literal(metadata_json)}::jsonb) "
                "ON CONFLICT (measurement_id) DO NOTHING;"
            )
    return "\n".join(statements) + "\n"


def _bootstrap_fixture(args: argparse.Namespace) -> None:
    result = subprocess.run(
        [
            "docker",
            "exec",
            "-i",
            args.container,
            "psql",
            "-v",
            "ON_ERROR_STOP=1",
            "-U",
            args.owner_user,
            "-d",
            args.database,
        ],
        input=_bootstrap_sql().encode("utf-8"),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        check=False,
    )
    if result.returncode:
        detail = result.stderr.decode("utf-8", errors="replace")[-2000:]
        raise SmokeError(f"dev DB synthetic anchor bootstrap failed: {detail}")


async def _set_research_context(conn: asyncpg.Connection) -> None:
    await conn.execute("SELECT set_config('app.ai_context','1',true)")
    await conn.execute("SELECT set_config('app.current_ai_view','research',true)")


async def _counts(conn: asyncpg.Connection) -> dict[str, int]:
    return {
        "batches": int(
            await conn.fetchval(
                """
                SELECT count(*) FROM app.supervision_evaluation_batch
                WHERE batch_id = ANY($1::text[])
                """,
                ["oas-g6-batch-baseline", "oas-g6-batch-candidate"],
            )
        ),
        "reports": int(
            await conn.fetchval(
                "SELECT count(*) FROM app.supervision_drift_report WHERE cohort_id=$1",
                FIXTURE_COHORT,
            )
        ),
        "subgroup_metrics": int(
            await conn.fetchval(
                """
                SELECT count(*)
                FROM app.supervision_drift_subgroup_metric m
                JOIN app.supervision_drift_report r
                  ON r.drift_report_id=m.drift_report_id
                WHERE r.cohort_id=$1
                """,
                FIXTURE_COHORT,
            )
        ),
    }


async def _produce_once(
    conn: asyncpg.Connection,
) -> tuple[dict[str, Any], dict[str, int]]:
    async with conn.transaction():
        await _set_research_context(conn)
        result = await evaluator.produce_repository_version_comparison(conn)
        counts = await _counts(conn)
    return result, counts


async def _run(args: argparse.Namespace) -> dict[str, Any]:
    env = _parse_env(Path(args.env_file))
    database_url = env.get("DATABASE_URL", "")
    if not database_url:
        raise SmokeError("DATABASE_URL is missing from the selected env file")
    _bootstrap_fixture(args)
    conn = await asyncpg.connect(database_url)
    try:
        async with conn.transaction():
            await _set_research_context(conn)
            before = await _counts(conn)
        if not args.allow_existing and any(before.values()):
            raise SmokeError(f"expected first append but rows already exist: {before}")

        first, after_first = await _produce_once(conn)
        replay, after_replay = await _produce_once(conn)
        async with conn.transaction():
            await _set_research_context(conn)
            research_view = await supervision_research_store.read_research_view(conn)
    finally:
        await conn.close()

    if not args.allow_existing and first.get("idempotent_replay") is not False:
        raise SmokeError("first comparison did not append a new report")
    if replay.get("idempotent_replay") is not True:
        raise SmokeError("second comparison was not an idempotent replay")
    if (
        first.get("status") != "drift_flagged"
        or replay.get("status") != "drift_flagged"
    ):
        raise SmokeError("repository comparison lost expected drift status")
    if after_first != {"batches": 2, "reports": 1, "subgroup_metrics": 2}:
        raise SmokeError(f"unexpected first append cardinality: {after_first}")
    if after_replay != after_first:
        raise SmokeError("idempotent replay created duplicate rows")

    drift_report_id = str(first["drift_report_id"])
    hydrated = next(
        (
            item
            for item in research_view["drift_reports"]
            if str(item["drift_report_id"]) == drift_report_id
        ),
        None,
    )
    if hydrated is None:
        raise SmokeError("research read model did not hydrate the drift report")
    if hydrated.get("status") != "drift_flagged":
        raise SmokeError("hydrated drift report status changed")
    subgroup_metrics = hydrated.get("subgroup_metrics") or []
    if {item["subgroup"] for item in subgroup_metrics} != {
        "synthetic-a",
        "synthetic-b",
    }:
        raise SmokeError("hydrated drift report omitted synthetic subgroup metrics")
    required_provenance = {
        "baseline_model",
        "candidate_model",
        "baseline_prompt_version",
        "candidate_prompt_version",
        "instrument_id",
        "baseline_instrument_version",
        "candidate_instrument_version",
    }
    if any(not hydrated.get(key) for key in required_provenance):
        raise SmokeError("hydrated drift report omitted evaluator provenance")

    return {
        "ok": True,
        "database": "actual dev PostgreSQL via runtime app role",
        "fixture_cohort": FIXTURE_COHORT,
        "benchmark_schema_version": first["benchmark_schema_version"],
        "benchmark_version": first["benchmark_version"],
        "benchmark_content_sha256": first["benchmark_content_sha256"],
        "first_append": {
            "idempotent_replay": first["idempotent_replay"],
            "drift_report_id": drift_report_id,
            "status": first["status"],
            "counts": after_first,
        },
        "replay": {
            "idempotent_replay": replay["idempotent_replay"],
            "counts": after_replay,
            "duplicate_rows": sum(
                after_replay[key] - after_first[key] for key in after_first
            ),
        },
        "hydrated": {
            "status": hydrated["status"],
            "baseline_model": hydrated["baseline_model"],
            "candidate_model": hydrated["candidate_model"],
            "baseline_prompt_version": hydrated["baseline_prompt_version"],
            "candidate_prompt_version": hydrated["candidate_prompt_version"],
            "instrument_id": hydrated["instrument_id"],
            "baseline_instrument_version": hydrated["baseline_instrument_version"],
            "candidate_instrument_version": hydrated["candidate_instrument_version"],
            "subgroup_metrics": subgroup_metrics,
        },
        "data_classification": "synthetic_educational",
        "raw_transcript_included": False,
        "clinical_claim_allowed": False,
    }


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--env-file", default="apps/api/.env")
    parser.add_argument("--container", default="vignette-dev-db")
    parser.add_argument("--owner-user", default="vignette_owner")
    parser.add_argument("--database", default="vignette")
    parser.add_argument("--allow-existing", action="store_true")
    parser.add_argument("--out", default="")
    args = parser.parse_args()
    result = asyncio.run(_run(args))
    text = json.dumps(result, ensure_ascii=False, indent=2, default=str)
    if args.out:
        output = Path(args.out)
        output.parent.mkdir(parents=True, exist_ok=True)
        output.write_text(text + "\n", encoding="utf-8")
    print(text)


if __name__ == "__main__":
    main()