vignette/apps/api/app/services/continuous_improvement.py

"""G8 콘텐츠 승격, 모델 교체 gate, 운영 오류 회귀 DAG 코어."""

from __future__ import annotations

from typing import Iterable

from ..contracts.continuous_improvement import (
    AgenticReleaseManifest,
    ApprovedCatalogEntry,
    ContentBenchmarkQualification,
    ContentSourceArtifact,
    GeneratedContentDraft,
    IncidentRegressionDag,
    IndependentRedTeamReview,
    ModelCalibrationSnapshot,
    ModelChangeGateResult,
    OperationalIncident,
    RegressionBacklogNode,
)


REQUIRED_REVIEW_DIMENSIONS = {
    "safety",
    "identity",
    "answer_leakage",
    "cultural_bias",
    "difficulty",
    "pii",
    "grounding",
}


def promote_content_to_catalog(
    *,
    draft: GeneratedContentDraft,
    sources: Iterable[ContentSourceArtifact],
    reviews: Iterable[IndependentRedTeamReview],
    benchmark: ContentBenchmarkQualification,
) -> ApprovedCatalogEntry:
    source_by_id = {item.source_id: item for item in sources}
    if set(draft.source_refs) - set(source_by_id):
        raise ValueError("content draft references an unknown source")
    if any(source_by_id[item].usage_status != "approved" for item in draft.source_refs):
        raise ValueError("content promotion requires approved source usage")
    if draft.visible_answer_overlap_tokens:
        raise ValueError("content promotion blocked by visible answer leakage")
    if draft.pii_findings:
        raise ValueError("content promotion blocked by PII findings")
    if draft.unsupported_clinical_claims:
        raise ValueError("content promotion blocked by unsupported clinical claims")

    review_items = tuple(reviews)
    if len(review_items) < 2:
        raise ValueError("content promotion requires two independent red-team reviews")
    if len({item.reviewer_agent_id for item in review_items}) != len(review_items):
        raise ValueError("red-team reviewers must be independent")
    if any(item.draft_id != draft.draft_id for item in review_items):
        raise ValueError("red-team review references another draft")
    if any(
        item.reviewed_payload_sha256 != draft.payload_sha256 for item in review_items
    ):
        raise ValueError("red-team review payload hash differs from draft")
    covered = {dimension for item in review_items for dimension in item.dimensions}
    if not REQUIRED_REVIEW_DIMENSIONS.issubset(covered):
        raise ValueError("red-team reviews do not cover every required dimension")
    unresolved = [
        finding
        for item in review_items
        for finding in item.findings
        if finding.state == "open"
        or (
            finding.state == "accepted_risk" and finding.severity in {"blocker", "high"}
        )
    ]
    if unresolved:
        raise ValueError("content promotion blocked by unresolved red-team findings")
    if benchmark.draft_id != draft.draft_id or not benchmark.qualified:
        raise ValueError("content promotion requires a qualified benchmark")

    return ApprovedCatalogEntry(
        catalog_entry_id=draft.draft_id.replace("oas-g8-draft-", "oas-g8-catalog-"),
        draft_id=draft.draft_id,
        payload_sha256=draft.payload_sha256,
        source_refs=draft.source_refs,
        review_ids=tuple(item.review_id for item in review_items),
        benchmark_id=benchmark.benchmark_id,
    )


def decide_model_change(
    *,
    baseline: ModelCalibrationSnapshot,
    candidate: ModelCalibrationSnapshot,
) -> ModelChangeGateResult:
    reasons: list[str] = []
    if candidate.critical_miss_count > baseline.critical_miss_count:
        reasons.append("critical_miss_regression")
    if candidate.leakage_count or candidate.pii_count:
        reasons.append("candidate_privacy_or_leakage_failure")
    if candidate.task_accuracy < baseline.task_accuracy - 0.02:
        reasons.append("task_accuracy_regression")
    if candidate.calibration_error > baseline.calibration_error + 0.02:
        reasons.append("calibration_error_regression")
    if candidate.subgroup_max_gap > baseline.subgroup_max_gap + 0.05:
        reasons.append("subgroup_gap_regression")

    if candidate.leakage_count or candidate.pii_count or candidate.critical_miss_count:
        decision = "rollback"
    elif reasons:
        decision = "quarantine"
    else:
        decision = "promote"
        reasons.append("candidate_passed_all_calibration_gates")
    return ModelChangeGateResult(
        baseline_snapshot_id=baseline.snapshot_id,
        candidate_snapshot_id=candidate.snapshot_id,
        decision=decision,
        reasons=tuple(reasons),
        rollback_target_snapshot_id=(
            baseline.snapshot_id if decision == "rollback" else None
        ),
    )


def build_incident_regression_dag(
    incident: OperationalIncident,
) -> IncidentRegressionDag:
    prefix = incident.incident_id.replace("oas-g8-incident-", "")
    reproduction_id = f"oas-g8-node-{prefix}-reproduction"
    implementation_id = f"oas-g8-node-{prefix}-implementation"
    e2e_id = f"oas-g8-node-{prefix}-e2e"
    runtime_id = f"oas-g8-node-{prefix}-runtime"
    return IncidentRegressionDag(
        incident_id=incident.incident_id,
        nodes=(
            RegressionBacklogNode(
                node_id=reproduction_id,
                node_type="reproduction_test",
                depends_on=(),
                evidence_ref=incident.evidence_refs[0],
                status="pending",
            ),
            RegressionBacklogNode(
                node_id=implementation_id,
                node_type="implementation",
                depends_on=(reproduction_id,),
                status="pending",
            ),
            RegressionBacklogNode(
                node_id=e2e_id,
                node_type="e2e",
                depends_on=(implementation_id,),
                status="pending",
            ),
            RegressionBacklogNode(
                node_id=runtime_id,
                node_type="runtime_proof",
                depends_on=(e2e_id,),
                status="pending",
            ),
        ),
    )


def release_allowed(manifest: AgenticReleaseManifest) -> bool:
    return manifest.releasable


__all__ = [
    "REQUIRED_REVIEW_DIMENSIONS",
    "build_incident_regression_dag",
    "decide_model_change",
    "promote_content_to_catalog",
    "release_allowed",
]