"""G8 operational drift -> incident -> adversarial queue trigger. This boundary reads only synthetic, metadata-only G6 benchmark drift rows from the research RLS view. It deterministically records an incident DAG and enqueues a ``scheduled_incident`` job into the existing G8 lease/retry queue. It never reads session/learner/transcript data and never performs approval or catalog promotion. """ from __future__ import annotations import hashlib import json import logging import re from typing import Any, Literal from uuid import UUID, uuid5 from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from .. import db from ..contracts.continuous_improvement import OperationalIncident from ..contracts.supervision_research import SubgroupVersionMetric from . import continuous_improvement_agentic from . import continuous_improvement_producer from . import continuous_improvement_store from . import supervision_research logger = logging.getLogger(__name__) _TRIGGER_NAMESPACE = UUID("00b71f78-77b1-5908-a25d-2bd532440159") _MAX_TRIGGER_BATCH = 50 _SUBGROUP_ALERT = re.compile(r"^synthetic_subgroup_regression:synthetic-[a-z0-9-]+$") _AFFECTED_CONTRACT = "supervision.research.version-benchmark" _PROMPT_VERSION = "g8-drift-adversarial-v1" _HEX_TO_LETTER_TOKEN = str.maketrans("0123456789abcdef", "abcdefghijklmnop") _FINGERPRINT_ALPHABET = "abcdef" class DriftBenchmarkSignal(BaseModel): """Strict projection of a triggerable G6 research benchmark ledger row.""" model_config = ConfigDict(extra="forbid", frozen=True) drift_report_id: UUID content_hash: str = Field(pattern=r"^[a-f0-9]{64}$") matched_count: int = Field(ge=supervision_research.MIN_DRIFT_MATCHES) status: Literal["drift_flagged"] baseline_accuracy: float = Field(ge=0.0, le=1.0) candidate_accuracy: float = Field(ge=0.0, le=1.0) accuracy_delta: float = Field(ge=-1.0, le=1.0) alerts: tuple[str, ...] = Field(min_length=1, max_length=20) subgroup_metrics: tuple[SubgroupVersionMetric, ...] = Field(max_length=20) data_classification: Literal["synthetic_educational"] clinical_claim_allowed: Literal[False] @model_validator(mode="after") def require_canonical_weak_benchmark_signal(self) -> "DriftBenchmarkSignal": if len(self.alerts) != len(set(self.alerts)): raise ValueError("drift alerts must be unique") if any( alert != "overall_accuracy_regression" and _SUBGROUP_ALERT.fullmatch(alert) is None for alert in self.alerts ): raise ValueError("drift signal contains an unsupported alert code") expected_delta = self.candidate_accuracy - self.baseline_accuracy if abs(expected_delta - self.accuracy_delta) > 1e-9: raise ValueError("drift signal accuracy delta is inconsistent") subgroups = [metric.subgroup for metric in self.subgroup_metrics] if len(subgroups) != len(set(subgroups)): raise ValueError("drift subgroup metrics must be unique") expected_alerts: list[str] = [] if self.accuracy_delta < -supervision_research.ACCURACY_DROP_THRESHOLD: expected_alerts.append("overall_accuracy_regression") for metric in self.subgroup_metrics: if _SUBGROUP_ALERT.fullmatch( f"synthetic_subgroup_regression:{metric.subgroup}" ) is None: raise ValueError("drift signal contains an unsupported subgroup") baseline_accuracy = metric.baseline_accuracy candidate_accuracy = metric.candidate_accuracy accuracy_delta = metric.accuracy_delta values = (baseline_accuracy, candidate_accuracy, accuracy_delta) if metric.matched_count < supervision_research.MIN_SUBGROUP_MATCHES: if any(value is not None for value in values): raise ValueError("underpowered subgroup metric must remain scoreless") continue if ( baseline_accuracy is None or candidate_accuracy is None or accuracy_delta is None ): raise ValueError("powered subgroup metric requires complete scores") expected_subgroup_delta = candidate_accuracy - baseline_accuracy if abs(expected_subgroup_delta - accuracy_delta) > 1e-9: raise ValueError("drift subgroup accuracy delta is inconsistent") if accuracy_delta < -supervision_research.SUBGROUP_DROP_THRESHOLD: expected_alerts.append( f"synthetic_subgroup_regression:{metric.subgroup}" ) if self.alerts != tuple(expected_alerts): raise ValueError("drift alerts do not match the canonical G6 thresholds") return self class DriftAdversarialTrigger(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) submission_id: UUID incident_record_id: UUID incident: OperationalIncident job_spec: continuous_improvement_producer.ScheduledAgenticJobSpec class DriftTriggerCycleResult(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) scanned: int = Field(ge=0) invalid_signals: int = Field(ge=0) incidents_created: int = Field(ge=0) incident_replays: int = Field(ge=0) jobs_enqueued: int = Field(ge=0) def _canonical_json(value: Any) -> str: return json.dumps( value, ensure_ascii=False, separators=(",", ":"), sort_keys=True, default=str, ) def _signal_fingerprint(signal: DriftBenchmarkSignal) -> str: # OperationalIncident allows only 64 hexadecimal characters. A raw hex # digest can look like a phone/identifier to the mandatory PII scanner, so # retain ~165 bits of digest entropy as a deterministic base-6 a-f token. # This remains inside the schema while avoiding numeric pseudo-identifiers. value = int.from_bytes( hashlib.sha256( _canonical_json(signal.model_dump(mode="json")).encode("utf-8") ).digest(), byteorder="big", ) encoded: list[str] = [] for _ in range(64): value, remainder = divmod(value, len(_FINGERPRINT_ALPHABET)) encoded.append(_FINGERPRINT_ALPHABET[remainder]) return "".join(reversed(encoded)) def _signal_token(signal: DriftBenchmarkSignal) -> str: # Reversible UUID-nibble encoding with letters a-p. Numeric UUID strings # are intentionally excluded from agent-visible source content. return signal.drift_report_id.hex.translate(_HEX_TO_LETTER_TOKEN) async def load_triggerable_drift_signals( conn: Any, *, limit: int, ) -> tuple[tuple[DriftBenchmarkSignal, ...], int]: """Load a metadata-only research projection; malformed rows are rejected.""" if not 1 <= limit <= _MAX_TRIGGER_BATCH: raise ValueError(f"drift trigger limit must be between 1 and {_MAX_TRIGGER_BATCH}") rows = await conn.fetch( """ SELECT report.drift_report_id, report.content_hash, report.matched_count, report.status, report.baseline_accuracy, report.candidate_accuracy, report.accuracy_delta, report.alerts, COALESCE( ( SELECT jsonb_agg( jsonb_build_object( 'subgroup', metric.subgroup, 'matched_count', metric.matched_count, 'baseline_accuracy', metric.baseline_accuracy, 'candidate_accuracy', metric.candidate_accuracy, 'accuracy_delta', metric.accuracy_delta ) ORDER BY metric.subgroup ) FROM app.supervision_drift_subgroup_metric AS metric WHERE metric.drift_report_id = report.drift_report_id ), '[]'::jsonb ) AS subgroup_metrics, report.data_classification, report.clinical_claim_allowed FROM app.supervision_drift_report AS report LEFT JOIN app.ci_agentic_job AS queued ON queued.job_key = 'oas-g8-job-g6-drift-' || replace(report.drift_report_id::text, '-', '') WHERE report.status = 'drift_flagged' AND report.data_classification = 'synthetic_educational' AND report.clinical_claim_allowed = FALSE AND report.matched_count >= $2 AND cardinality(report.alerts) >= 1 AND queued.job_id IS NULL ORDER BY report.created_at, report.drift_report_id LIMIT $1 """, limit, supervision_research.MIN_DRIFT_MATCHES, ) accepted: list[DriftBenchmarkSignal] = [] rejected = 0 for row in rows: try: accepted.append(DriftBenchmarkSignal.model_validate(dict(row))) except (TypeError, ValidationError): rejected += 1 record_id = None try: record_id = row["drift_report_id"] except (KeyError, TypeError): pass logger.warning( "G8 drift trigger rejected invalid metadata signal: drift_report_id=%s", record_id, ) return tuple(accepted), rejected def build_drift_adversarial_trigger( signal: DriftBenchmarkSignal, ) -> DriftAdversarialTrigger: """Build deterministic incident/DAG/job identifiers from an immutable row.""" # Revalidate callers that constructed a model through non-standard means. signal = DriftBenchmarkSignal.model_validate(signal.model_dump(mode="json")) token = _signal_token(signal) fingerprint = _signal_fingerprint(signal) incident = OperationalIncident( incident_id=f"oas-g8-incident-g6-drift-{token}", error_fingerprint=fingerprint, affected_contract=_AFFECTED_CONTRACT, evidence_refs=( f"db://app/supervision-drift-report/{token}", ), pii_included=False, ) source_pack = continuous_improvement_agentic.source_pack_from_operational_incident( incident ) job_spec = continuous_improvement_producer.ScheduledAgenticJobSpec( # The queue key stays a DB-only opaque UUID token so the loader can # exclude already-enqueued rows without crossing into source content. job_key=f"oas-g8-job-g6-drift-{signal.drift_report_id.hex}", data_classification=continuous_improvement_producer.DATA_CLASSIFICATION, content_kind="benchmark", difficulty_level=5, variant_count=5, prompt_version=_PROMPT_VERSION, trigger_kind="scheduled_incident", source_packs=(source_pack,), ) return DriftAdversarialTrigger( submission_id=uuid5( _TRIGGER_NAMESPACE, f"incident-submission:{signal.drift_report_id}:{signal.content_hash}", ), incident_record_id=uuid5( _TRIGGER_NAMESPACE, f"incident-record:{signal.drift_report_id}:{signal.content_hash}", ), incident=incident, job_spec=job_spec, ) async def enqueue_drift_adversarial_jobs_once( *, limit: int | None = None, ) -> DriftTriggerCycleResult: """Atomically persist incident DAGs and enqueue jobs under research RLS.""" trigger_limit = ( continuous_improvement_producer.settings.continuous_improvement_producer_batch_size if limit is None else limit ) async with db.acquire(ai_view="research", ai_context=True) as conn: signals, invalid_signals = await load_triggerable_drift_signals( conn, limit=trigger_limit, ) incidents_created = 0 incident_replays = 0 jobs_enqueued = 0 for signal in signals: request = build_drift_adversarial_trigger(signal) incident_result = await continuous_improvement_store.submit_incident_dag( conn=conn, submission_id=request.submission_id, incident_record_id=request.incident_record_id, incident=request.incident, ) if bool(incident_result["idempotent_replay"]): incident_replays += 1 else: incidents_created += 1 await continuous_improvement_producer.enqueue_agentic_job( conn, request.job_spec, ) jobs_enqueued += 1 return DriftTriggerCycleResult( scanned=len(signals) + invalid_signals, invalid_signals=invalid_signals, incidents_created=incidents_created, incident_replays=incident_replays, jobs_enqueued=jobs_enqueued, ) __all__ = [ "DriftAdversarialTrigger", "DriftBenchmarkSignal", "DriftTriggerCycleResult", "build_drift_adversarial_trigger", "enqueue_drift_adversarial_jobs_once", "load_triggerable_drift_signals", ]