vignette/scripts/smoke-continuous-improvement-agentic-producer.py

"""Run the real G8 scheduled worker against the configured dev engine and DB."""

from __future__ import annotations

import argparse
import asyncio
import hashlib
import json
import os
from pathlib import Path
import sys
from typing import Any


REPO_ROOT = Path(__file__).resolve().parents[1]
API_ROOT = REPO_ROOT / "apps" / "api"
if str(API_ROOT) not in sys.path:
    sys.path.insert(0, str(API_ROOT))
os.chdir(API_ROOT)

from app import db  # noqa: E402
from app.engine_client import engine_client  # noqa: E402
from app.services import continuous_improvement_agentic as agentic  # noqa: E402
from app.services import continuous_improvement_producer as producer  # noqa: E402


class SmokeFailure(RuntimeError):
    pass


class CountingEngine:
    def __init__(self) -> None:
        self.call_count = 0
        self.stages: list[str] = []
        self.responses: list[dict[str, str]] = []

    async def generate(self, request: Any) -> Any:
        self.call_count += 1
        self.stages.append(str(request.metadata.get("agentic_stage", "unknown")))
        response = await engine_client.generate(
            request,
            timeout=producer.settings.continuous_improvement_producer_engine_timeout_seconds,
        )
        self.responses.append(
            {"provider": str(response.provider), "model": str(response.model)}
        )
        return response


def _sha256_text(value: str) -> str:
    return hashlib.sha256(value.encode("utf-8")).hexdigest()


async def _read_evidence_row(job_id: Any) -> dict[str, Any]:
    async with db.acquire(ai_view="research", ai_context=True) as conn:
        row = await conn.fetchrow(
            """
            SELECT j.job_id, j.job_key, j.status, j.attempt_count,
                   j.data_classification, j.source_fingerprint,
                   j.last_error_code, j.last_error_message,
                   j.result_submission_id, j.result_qualification_id,
                   p.pipeline_id, p.generation_model, p.prompt_version,
                   p.prompt_sha256, p.payload_sha256, p.state AS pipeline_state,
                   p.clinical_claim_allowed,
                   q.gate_state, q.catalog_entry_id,
                   b.variant_count, b.variant_pass_rate, b.qualified,
                   (SELECT count(*) FROM app.ci_red_team_review r
                    WHERE r.pipeline_id = p.pipeline_id) AS reviewer_count,
                   (SELECT count(*) FROM app.ci_catalog_entry c
                    WHERE c.qualification_id = q.qualification_id) AS catalog_count,
                   (SELECT count(*) FROM audit.ci_human_approval_event a
                    WHERE a.target_kind = 'content_qualification'
                      AND a.target_id = q.qualification_id) AS approval_count,
                   (SELECT jsonb_agg(jsonb_build_object(
                      'source_id', s.source_id,
                      'version', s.source_version,
                      'content_sha256', s.content_sha256,
                      'provenance_uri', s.provenance_uri,
                      'usage_status', s.usage_status
                    ) ORDER BY s.source_id)
                    FROM app.ci_source_artifact s
                    WHERE s.source_record_id = ANY(p.source_record_ids)) AS sources
            FROM app.ci_agentic_job j
            LEFT JOIN app.ci_content_pipeline p
              ON p.submission_id = j.result_submission_id
            LEFT JOIN app.ci_content_qualification q
              ON q.qualification_id = j.result_qualification_id
            LEFT JOIN app.ci_content_benchmark b
              ON b.pipeline_id = p.pipeline_id
            WHERE j.job_id = $1
            """,
            job_id,
        )
    if row is None:
        raise SmokeFailure("scheduled agentic job is not visible")
    return dict(row)


async def _run(out_path: Path) -> dict[str, Any]:
    await db.init_pool()
    await engine_client.startup()
    try:
        readiness = await engine_client.health_detail()
        if not readiness.get("ok"):
            raise SmokeFailure(f"engine is not ready: {readiness.get('detail')}")

        spec = producer.load_repo_approved_job()
        job_id = await producer.ensure_repo_approved_job()
        claimed = await producer.claim_next_agentic_job(job_id=job_id)
        if claimed is None:
            raise SmokeFailure("repo-approved job was not available for a fresh model run")

        first_engine = CountingEngine()
        first = await producer.execute_claimed_agentic_job(
            claimed,
            engine=first_engine,
        )
        row = await _read_evidence_row(job_id)

        evidence: dict[str, Any] = {
            "evidence_version": "g8-agentic-producer-live-v1",
            "environment": "dev",
            "engine_ready": {
                "ok": bool(readiness.get("ok")),
                "detail": str(readiness.get("detail", "")),
            },
            "repo_source": {
                "job_key": spec.job_key,
                "data_classification": spec.data_classification,
                "source_ids": [item.artifact.source_id for item in spec.source_packs],
                "usage_statuses": [
                    item.artifact.usage_status for item in spec.source_packs
                ],
                "content_hashes_verified": all(
                    _sha256_text(item.content) == item.artifact.content_sha256
                    for item in spec.source_packs
                ),
                "raw_transcript_included": False,
                "pii_included": False,
            },
            "first_run": {
                "status": first.status,
                "engine_call_count": first_engine.call_count,
                "stages": sorted(first_engine.stages),
                "provider_models": first_engine.responses,
                "error_code": first.error_code,
            },
            "durable_db": row,
            "safety_boundary": {
                "human_approval_required": row.get("gate_state")
                == "pending_human_approval",
                "catalog_promoted": int(row.get("catalog_count") or 0) > 0,
                "human_approval_event_count": int(row.get("approval_count") or 0),
                "clinical_claim_allowed": bool(row.get("clinical_claim_allowed")),
            },
        }

        if first.status == "completed":
            ids = producer._job_ids(job_id)
            replay_engine = CountingEngine()
            async with db.acquire(ai_view="research", ai_context=True) as conn:
                replay = await agentic.run_agentic_content_pipeline(
                    conn=conn,
                    engine=replay_engine,
                    submission_id=ids["submission"],
                    pipeline_id=ids["pipeline"],
                    benchmark_record_id=ids["benchmark_record"],
                    qualification_id=ids["qualification"],
                    source_packs=spec.source_packs,
                    content_kind=spec.content_kind,
                    difficulty_level=spec.difficulty_level,
                    variant_count=spec.variant_count,
                    prompt_version=spec.prompt_version,
                    trigger_kind=spec.trigger_kind,
                )
            evidence["idempotent_replay"] = {
                "reported": replay.idempotent_replay,
                "engine_call_count": replay_engine.call_count,
                "same_submission_id": str(replay.submission_id)
                == str(row.get("result_submission_id")),
                "same_qualification_id": str(replay.qualification_id)
                == str(row.get("result_qualification_id")),
            }
        else:
            evidence["storage_rejected"] = {
                "pipeline_absent": row.get("pipeline_id") is None,
                "qualification_absent": row.get("result_qualification_id") is None,
                "catalog_count": int(row.get("catalog_count") or 0),
            }

        out_path.parent.mkdir(parents=True, exist_ok=True)
        out_path.write_text(
            json.dumps(evidence, ensure_ascii=False, indent=2, default=str) + "\n",
            encoding="utf-8",
        )
        return evidence
    finally:
        await engine_client.shutdown()
        await db.close_pool()


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--out",
        type=Path,
        required=True,
        help="JSON evidence path",
    )
    args = parser.parse_args()
    requested_out = args.out
    out_path = (
        requested_out
        if requested_out.is_absolute()
        else REPO_ROOT / requested_out
    ).resolve()
    evidence = asyncio.run(_run(out_path))
    print(json.dumps(evidence, ensure_ascii=False, indent=2, default=str))
    first_status = evidence["first_run"]["status"]
    return 0 if first_status == "completed" else 2


if __name__ == "__main__":
    raise SystemExit(main())