16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
334 lines
13 KiB
Python
334 lines
13 KiB
Python
"""G8 operational drift -> incident -> adversarial queue trigger.
|
|
|
|
This boundary reads only synthetic, metadata-only G6 benchmark drift rows from
|
|
the research RLS view. It deterministically records an incident DAG and
|
|
enqueues a ``scheduled_incident`` job into the existing G8 lease/retry queue.
|
|
It never reads session/learner/transcript data and never performs approval or
|
|
catalog promotion.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
from typing import Any, Literal
|
|
from uuid import UUID, uuid5
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
|
|
|
|
from .. import db
|
|
from ..contracts.continuous_improvement import OperationalIncident
|
|
from ..contracts.supervision_research import SubgroupVersionMetric
|
|
from . import continuous_improvement_agentic
|
|
from . import continuous_improvement_producer
|
|
from . import continuous_improvement_store
|
|
from . import supervision_research
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_TRIGGER_NAMESPACE = UUID("00b71f78-77b1-5908-a25d-2bd532440159")
|
|
_MAX_TRIGGER_BATCH = 50
|
|
_SUBGROUP_ALERT = re.compile(r"^synthetic_subgroup_regression:synthetic-[a-z0-9-]+$")
|
|
_AFFECTED_CONTRACT = "supervision.research.version-benchmark"
|
|
_PROMPT_VERSION = "g8-drift-adversarial-v1"
|
|
_HEX_TO_LETTER_TOKEN = str.maketrans("0123456789abcdef", "abcdefghijklmnop")
|
|
_FINGERPRINT_ALPHABET = "abcdef"
|
|
|
|
|
|
class DriftBenchmarkSignal(BaseModel):
|
|
"""Strict projection of a triggerable G6 research benchmark ledger row."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
|
|
drift_report_id: UUID
|
|
content_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
|
matched_count: int = Field(ge=supervision_research.MIN_DRIFT_MATCHES)
|
|
status: Literal["drift_flagged"]
|
|
baseline_accuracy: float = Field(ge=0.0, le=1.0)
|
|
candidate_accuracy: float = Field(ge=0.0, le=1.0)
|
|
accuracy_delta: float = Field(ge=-1.0, le=1.0)
|
|
alerts: tuple[str, ...] = Field(min_length=1, max_length=20)
|
|
subgroup_metrics: tuple[SubgroupVersionMetric, ...] = Field(max_length=20)
|
|
data_classification: Literal["synthetic_educational"]
|
|
clinical_claim_allowed: Literal[False]
|
|
|
|
@model_validator(mode="after")
|
|
def require_canonical_weak_benchmark_signal(self) -> "DriftBenchmarkSignal":
|
|
if len(self.alerts) != len(set(self.alerts)):
|
|
raise ValueError("drift alerts must be unique")
|
|
if any(
|
|
alert != "overall_accuracy_regression"
|
|
and _SUBGROUP_ALERT.fullmatch(alert) is None
|
|
for alert in self.alerts
|
|
):
|
|
raise ValueError("drift signal contains an unsupported alert code")
|
|
expected_delta = self.candidate_accuracy - self.baseline_accuracy
|
|
if abs(expected_delta - self.accuracy_delta) > 1e-9:
|
|
raise ValueError("drift signal accuracy delta is inconsistent")
|
|
subgroups = [metric.subgroup for metric in self.subgroup_metrics]
|
|
if len(subgroups) != len(set(subgroups)):
|
|
raise ValueError("drift subgroup metrics must be unique")
|
|
|
|
expected_alerts: list[str] = []
|
|
if self.accuracy_delta < -supervision_research.ACCURACY_DROP_THRESHOLD:
|
|
expected_alerts.append("overall_accuracy_regression")
|
|
for metric in self.subgroup_metrics:
|
|
if _SUBGROUP_ALERT.fullmatch(
|
|
f"synthetic_subgroup_regression:{metric.subgroup}"
|
|
) is None:
|
|
raise ValueError("drift signal contains an unsupported subgroup")
|
|
baseline_accuracy = metric.baseline_accuracy
|
|
candidate_accuracy = metric.candidate_accuracy
|
|
accuracy_delta = metric.accuracy_delta
|
|
values = (baseline_accuracy, candidate_accuracy, accuracy_delta)
|
|
if metric.matched_count < supervision_research.MIN_SUBGROUP_MATCHES:
|
|
if any(value is not None for value in values):
|
|
raise ValueError("underpowered subgroup metric must remain scoreless")
|
|
continue
|
|
if (
|
|
baseline_accuracy is None
|
|
or candidate_accuracy is None
|
|
or accuracy_delta is None
|
|
):
|
|
raise ValueError("powered subgroup metric requires complete scores")
|
|
expected_subgroup_delta = candidate_accuracy - baseline_accuracy
|
|
if abs(expected_subgroup_delta - accuracy_delta) > 1e-9:
|
|
raise ValueError("drift subgroup accuracy delta is inconsistent")
|
|
if accuracy_delta < -supervision_research.SUBGROUP_DROP_THRESHOLD:
|
|
expected_alerts.append(
|
|
f"synthetic_subgroup_regression:{metric.subgroup}"
|
|
)
|
|
if self.alerts != tuple(expected_alerts):
|
|
raise ValueError("drift alerts do not match the canonical G6 thresholds")
|
|
return self
|
|
|
|
|
|
class DriftAdversarialTrigger(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
|
|
submission_id: UUID
|
|
incident_record_id: UUID
|
|
incident: OperationalIncident
|
|
job_spec: continuous_improvement_producer.ScheduledAgenticJobSpec
|
|
|
|
|
|
class DriftTriggerCycleResult(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
|
|
scanned: int = Field(ge=0)
|
|
invalid_signals: int = Field(ge=0)
|
|
incidents_created: int = Field(ge=0)
|
|
incident_replays: int = Field(ge=0)
|
|
jobs_enqueued: int = Field(ge=0)
|
|
|
|
|
|
def _canonical_json(value: Any) -> str:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
|
|
|
|
def _signal_fingerprint(signal: DriftBenchmarkSignal) -> str:
|
|
# OperationalIncident allows only 64 hexadecimal characters. A raw hex
|
|
# digest can look like a phone/identifier to the mandatory PII scanner, so
|
|
# retain ~165 bits of digest entropy as a deterministic base-6 a-f token.
|
|
# This remains inside the schema while avoiding numeric pseudo-identifiers.
|
|
value = int.from_bytes(
|
|
hashlib.sha256(
|
|
_canonical_json(signal.model_dump(mode="json")).encode("utf-8")
|
|
).digest(),
|
|
byteorder="big",
|
|
)
|
|
encoded: list[str] = []
|
|
for _ in range(64):
|
|
value, remainder = divmod(value, len(_FINGERPRINT_ALPHABET))
|
|
encoded.append(_FINGERPRINT_ALPHABET[remainder])
|
|
return "".join(reversed(encoded))
|
|
|
|
|
|
def _signal_token(signal: DriftBenchmarkSignal) -> str:
|
|
# Reversible UUID-nibble encoding with letters a-p. Numeric UUID strings
|
|
# are intentionally excluded from agent-visible source content.
|
|
return signal.drift_report_id.hex.translate(_HEX_TO_LETTER_TOKEN)
|
|
|
|
|
|
async def load_triggerable_drift_signals(
|
|
conn: Any,
|
|
*,
|
|
limit: int,
|
|
) -> tuple[tuple[DriftBenchmarkSignal, ...], int]:
|
|
"""Load a metadata-only research projection; malformed rows are rejected."""
|
|
|
|
if not 1 <= limit <= _MAX_TRIGGER_BATCH:
|
|
raise ValueError(f"drift trigger limit must be between 1 and {_MAX_TRIGGER_BATCH}")
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT
|
|
report.drift_report_id,
|
|
report.content_hash,
|
|
report.matched_count,
|
|
report.status,
|
|
report.baseline_accuracy,
|
|
report.candidate_accuracy,
|
|
report.accuracy_delta,
|
|
report.alerts,
|
|
COALESCE(
|
|
(
|
|
SELECT jsonb_agg(
|
|
jsonb_build_object(
|
|
'subgroup', metric.subgroup,
|
|
'matched_count', metric.matched_count,
|
|
'baseline_accuracy', metric.baseline_accuracy,
|
|
'candidate_accuracy', metric.candidate_accuracy,
|
|
'accuracy_delta', metric.accuracy_delta
|
|
) ORDER BY metric.subgroup
|
|
)
|
|
FROM app.supervision_drift_subgroup_metric AS metric
|
|
WHERE metric.drift_report_id = report.drift_report_id
|
|
),
|
|
'[]'::jsonb
|
|
) AS subgroup_metrics,
|
|
report.data_classification,
|
|
report.clinical_claim_allowed
|
|
FROM app.supervision_drift_report AS report
|
|
LEFT JOIN app.ci_agentic_job AS queued
|
|
ON queued.job_key = 'oas-g8-job-g6-drift-'
|
|
|| replace(report.drift_report_id::text, '-', '')
|
|
WHERE report.status = 'drift_flagged'
|
|
AND report.data_classification = 'synthetic_educational'
|
|
AND report.clinical_claim_allowed = FALSE
|
|
AND report.matched_count >= $2
|
|
AND cardinality(report.alerts) >= 1
|
|
AND queued.job_id IS NULL
|
|
ORDER BY report.created_at, report.drift_report_id
|
|
LIMIT $1
|
|
""",
|
|
limit,
|
|
supervision_research.MIN_DRIFT_MATCHES,
|
|
)
|
|
accepted: list[DriftBenchmarkSignal] = []
|
|
rejected = 0
|
|
for row in rows:
|
|
try:
|
|
accepted.append(DriftBenchmarkSignal.model_validate(dict(row)))
|
|
except (TypeError, ValidationError):
|
|
rejected += 1
|
|
record_id = None
|
|
try:
|
|
record_id = row["drift_report_id"]
|
|
except (KeyError, TypeError):
|
|
pass
|
|
logger.warning(
|
|
"G8 drift trigger rejected invalid metadata signal: drift_report_id=%s",
|
|
record_id,
|
|
)
|
|
return tuple(accepted), rejected
|
|
|
|
|
|
def build_drift_adversarial_trigger(
|
|
signal: DriftBenchmarkSignal,
|
|
) -> DriftAdversarialTrigger:
|
|
"""Build deterministic incident/DAG/job identifiers from an immutable row."""
|
|
|
|
# Revalidate callers that constructed a model through non-standard means.
|
|
signal = DriftBenchmarkSignal.model_validate(signal.model_dump(mode="json"))
|
|
token = _signal_token(signal)
|
|
fingerprint = _signal_fingerprint(signal)
|
|
incident = OperationalIncident(
|
|
incident_id=f"oas-g8-incident-g6-drift-{token}",
|
|
error_fingerprint=fingerprint,
|
|
affected_contract=_AFFECTED_CONTRACT,
|
|
evidence_refs=(
|
|
f"db://app/supervision-drift-report/{token}",
|
|
),
|
|
pii_included=False,
|
|
)
|
|
source_pack = continuous_improvement_agentic.source_pack_from_operational_incident(
|
|
incident
|
|
)
|
|
job_spec = continuous_improvement_producer.ScheduledAgenticJobSpec(
|
|
# The queue key stays a DB-only opaque UUID token so the loader can
|
|
# exclude already-enqueued rows without crossing into source content.
|
|
job_key=f"oas-g8-job-g6-drift-{signal.drift_report_id.hex}",
|
|
data_classification=continuous_improvement_producer.DATA_CLASSIFICATION,
|
|
content_kind="benchmark",
|
|
difficulty_level=5,
|
|
variant_count=5,
|
|
prompt_version=_PROMPT_VERSION,
|
|
trigger_kind="scheduled_incident",
|
|
source_packs=(source_pack,),
|
|
)
|
|
return DriftAdversarialTrigger(
|
|
submission_id=uuid5(
|
|
_TRIGGER_NAMESPACE,
|
|
f"incident-submission:{signal.drift_report_id}:{signal.content_hash}",
|
|
),
|
|
incident_record_id=uuid5(
|
|
_TRIGGER_NAMESPACE,
|
|
f"incident-record:{signal.drift_report_id}:{signal.content_hash}",
|
|
),
|
|
incident=incident,
|
|
job_spec=job_spec,
|
|
)
|
|
|
|
|
|
async def enqueue_drift_adversarial_jobs_once(
|
|
*,
|
|
limit: int | None = None,
|
|
) -> DriftTriggerCycleResult:
|
|
"""Atomically persist incident DAGs and enqueue jobs under research RLS."""
|
|
|
|
trigger_limit = (
|
|
continuous_improvement_producer.settings.continuous_improvement_producer_batch_size
|
|
if limit is None
|
|
else limit
|
|
)
|
|
async with db.acquire(ai_view="research", ai_context=True) as conn:
|
|
signals, invalid_signals = await load_triggerable_drift_signals(
|
|
conn,
|
|
limit=trigger_limit,
|
|
)
|
|
incidents_created = 0
|
|
incident_replays = 0
|
|
jobs_enqueued = 0
|
|
for signal in signals:
|
|
request = build_drift_adversarial_trigger(signal)
|
|
incident_result = await continuous_improvement_store.submit_incident_dag(
|
|
conn=conn,
|
|
submission_id=request.submission_id,
|
|
incident_record_id=request.incident_record_id,
|
|
incident=request.incident,
|
|
)
|
|
if bool(incident_result["idempotent_replay"]):
|
|
incident_replays += 1
|
|
else:
|
|
incidents_created += 1
|
|
await continuous_improvement_producer.enqueue_agentic_job(
|
|
conn,
|
|
request.job_spec,
|
|
)
|
|
jobs_enqueued += 1
|
|
return DriftTriggerCycleResult(
|
|
scanned=len(signals) + invalid_signals,
|
|
invalid_signals=invalid_signals,
|
|
incidents_created=incidents_created,
|
|
incident_replays=incident_replays,
|
|
jobs_enqueued=jobs_enqueued,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"DriftAdversarialTrigger",
|
|
"DriftBenchmarkSignal",
|
|
"DriftTriggerCycleResult",
|
|
"build_drift_adversarial_trigger",
|
|
"enqueue_drift_adversarial_jobs_once",
|
|
"load_triggerable_drift_signals",
|
|
]
|