G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
175
apps/api/app/services/continuous_improvement.py
Normal file
175
apps/api/app/services/continuous_improvement.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""G8 콘텐츠 승격, 모델 교체 gate, 운영 오류 회귀 DAG 코어."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable
|
||||
|
||||
from ..contracts.continuous_improvement import (
|
||||
AgenticReleaseManifest,
|
||||
ApprovedCatalogEntry,
|
||||
ContentBenchmarkQualification,
|
||||
ContentSourceArtifact,
|
||||
GeneratedContentDraft,
|
||||
IncidentRegressionDag,
|
||||
IndependentRedTeamReview,
|
||||
ModelCalibrationSnapshot,
|
||||
ModelChangeGateResult,
|
||||
OperationalIncident,
|
||||
RegressionBacklogNode,
|
||||
)
|
||||
|
||||
|
||||
REQUIRED_REVIEW_DIMENSIONS = {
|
||||
"safety",
|
||||
"identity",
|
||||
"answer_leakage",
|
||||
"cultural_bias",
|
||||
"difficulty",
|
||||
"pii",
|
||||
"grounding",
|
||||
}
|
||||
|
||||
|
||||
def promote_content_to_catalog(
|
||||
*,
|
||||
draft: GeneratedContentDraft,
|
||||
sources: Iterable[ContentSourceArtifact],
|
||||
reviews: Iterable[IndependentRedTeamReview],
|
||||
benchmark: ContentBenchmarkQualification,
|
||||
) -> ApprovedCatalogEntry:
|
||||
source_by_id = {item.source_id: item for item in sources}
|
||||
if set(draft.source_refs) - set(source_by_id):
|
||||
raise ValueError("content draft references an unknown source")
|
||||
if any(source_by_id[item].usage_status != "approved" for item in draft.source_refs):
|
||||
raise ValueError("content promotion requires approved source usage")
|
||||
if draft.visible_answer_overlap_tokens:
|
||||
raise ValueError("content promotion blocked by visible answer leakage")
|
||||
if draft.pii_findings:
|
||||
raise ValueError("content promotion blocked by PII findings")
|
||||
if draft.unsupported_clinical_claims:
|
||||
raise ValueError("content promotion blocked by unsupported clinical claims")
|
||||
|
||||
review_items = tuple(reviews)
|
||||
if len(review_items) < 2:
|
||||
raise ValueError("content promotion requires two independent red-team reviews")
|
||||
if len({item.reviewer_agent_id for item in review_items}) != len(review_items):
|
||||
raise ValueError("red-team reviewers must be independent")
|
||||
if any(item.draft_id != draft.draft_id for item in review_items):
|
||||
raise ValueError("red-team review references another draft")
|
||||
if any(
|
||||
item.reviewed_payload_sha256 != draft.payload_sha256 for item in review_items
|
||||
):
|
||||
raise ValueError("red-team review payload hash differs from draft")
|
||||
covered = {dimension for item in review_items for dimension in item.dimensions}
|
||||
if not REQUIRED_REVIEW_DIMENSIONS.issubset(covered):
|
||||
raise ValueError("red-team reviews do not cover every required dimension")
|
||||
unresolved = [
|
||||
finding
|
||||
for item in review_items
|
||||
for finding in item.findings
|
||||
if finding.state == "open"
|
||||
or (
|
||||
finding.state == "accepted_risk" and finding.severity in {"blocker", "high"}
|
||||
)
|
||||
]
|
||||
if unresolved:
|
||||
raise ValueError("content promotion blocked by unresolved red-team findings")
|
||||
if benchmark.draft_id != draft.draft_id or not benchmark.qualified:
|
||||
raise ValueError("content promotion requires a qualified benchmark")
|
||||
|
||||
return ApprovedCatalogEntry(
|
||||
catalog_entry_id=draft.draft_id.replace("oas-g8-draft-", "oas-g8-catalog-"),
|
||||
draft_id=draft.draft_id,
|
||||
payload_sha256=draft.payload_sha256,
|
||||
source_refs=draft.source_refs,
|
||||
review_ids=tuple(item.review_id for item in review_items),
|
||||
benchmark_id=benchmark.benchmark_id,
|
||||
)
|
||||
|
||||
|
||||
def decide_model_change(
|
||||
*,
|
||||
baseline: ModelCalibrationSnapshot,
|
||||
candidate: ModelCalibrationSnapshot,
|
||||
) -> ModelChangeGateResult:
|
||||
reasons: list[str] = []
|
||||
if candidate.critical_miss_count > baseline.critical_miss_count:
|
||||
reasons.append("critical_miss_regression")
|
||||
if candidate.leakage_count or candidate.pii_count:
|
||||
reasons.append("candidate_privacy_or_leakage_failure")
|
||||
if candidate.task_accuracy < baseline.task_accuracy - 0.02:
|
||||
reasons.append("task_accuracy_regression")
|
||||
if candidate.calibration_error > baseline.calibration_error + 0.02:
|
||||
reasons.append("calibration_error_regression")
|
||||
if candidate.subgroup_max_gap > baseline.subgroup_max_gap + 0.05:
|
||||
reasons.append("subgroup_gap_regression")
|
||||
|
||||
if candidate.leakage_count or candidate.pii_count or candidate.critical_miss_count:
|
||||
decision = "rollback"
|
||||
elif reasons:
|
||||
decision = "quarantine"
|
||||
else:
|
||||
decision = "promote"
|
||||
reasons.append("candidate_passed_all_calibration_gates")
|
||||
return ModelChangeGateResult(
|
||||
baseline_snapshot_id=baseline.snapshot_id,
|
||||
candidate_snapshot_id=candidate.snapshot_id,
|
||||
decision=decision,
|
||||
reasons=tuple(reasons),
|
||||
rollback_target_snapshot_id=(
|
||||
baseline.snapshot_id if decision == "rollback" else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_incident_regression_dag(
|
||||
incident: OperationalIncident,
|
||||
) -> IncidentRegressionDag:
|
||||
prefix = incident.incident_id.replace("oas-g8-incident-", "")
|
||||
reproduction_id = f"oas-g8-node-{prefix}-reproduction"
|
||||
implementation_id = f"oas-g8-node-{prefix}-implementation"
|
||||
e2e_id = f"oas-g8-node-{prefix}-e2e"
|
||||
runtime_id = f"oas-g8-node-{prefix}-runtime"
|
||||
return IncidentRegressionDag(
|
||||
incident_id=incident.incident_id,
|
||||
nodes=(
|
||||
RegressionBacklogNode(
|
||||
node_id=reproduction_id,
|
||||
node_type="reproduction_test",
|
||||
depends_on=(),
|
||||
evidence_ref=incident.evidence_refs[0],
|
||||
status="pending",
|
||||
),
|
||||
RegressionBacklogNode(
|
||||
node_id=implementation_id,
|
||||
node_type="implementation",
|
||||
depends_on=(reproduction_id,),
|
||||
status="pending",
|
||||
),
|
||||
RegressionBacklogNode(
|
||||
node_id=e2e_id,
|
||||
node_type="e2e",
|
||||
depends_on=(implementation_id,),
|
||||
status="pending",
|
||||
),
|
||||
RegressionBacklogNode(
|
||||
node_id=runtime_id,
|
||||
node_type="runtime_proof",
|
||||
depends_on=(e2e_id,),
|
||||
status="pending",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def release_allowed(manifest: AgenticReleaseManifest) -> bool:
|
||||
return manifest.releasable
|
||||
|
||||
|
||||
__all__ = [
|
||||
"REQUIRED_REVIEW_DIMENSIONS",
|
||||
"build_incident_regression_dag",
|
||||
"decide_model_change",
|
||||
"promote_content_to_catalog",
|
||||
"release_allowed",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue