G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
564
apps/api/app/contracts/calibration_transfer.py
Normal file
564
apps/api/app/contracts/calibration_transfer.py
Normal file
|
|
@ -0,0 +1,564 @@
|
|||
"""G5 Calibration Mirror & Transfer의 버전 고정 순수 도메인 계약.
|
||||
|
||||
외부 평가 공개 전에 잠근 자기예측과 독립 관찰을 역량별로만 대조한다.
|
||||
전이는 익숙한 문장 재현과 분리하고, 합성 subgroup 차이는 표본 근거가
|
||||
충분할 때만 drift 신호로 남긴다. 단일 총점이나 임상 주장은 허용하지 않는다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
CalibrationBias = Literal[
|
||||
"overconfident", "underconfident", "aligned", "insufficient_evidence"
|
||||
]
|
||||
ImprovementStatus = Literal["improved", "not_improved", "insufficient_evidence"]
|
||||
PerformanceStatus = Literal["passed", "failed", "insufficient_evidence"]
|
||||
RelationshipStyle = Literal[
|
||||
"collaborative", "withdrawn", "confrontational", "ambivalent"
|
||||
]
|
||||
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
||||
ActualTransferStatus = Literal["verified", "not_verified", "insufficient_evidence"]
|
||||
|
||||
|
||||
class SelfPredictionRevision(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
prediction_id: str = Field(pattern=r"^oas-g5-prediction-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
revision_no: int = Field(ge=1)
|
||||
supersedes_prediction_id: str | None = None
|
||||
recorded_sequence: int = Field(ge=1)
|
||||
revision_reason: str = Field(min_length=1, max_length=300)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_revision_link(self) -> "SelfPredictionRevision":
|
||||
if self.revision_no == 1 and self.supersedes_prediction_id is not None:
|
||||
raise ValueError("first self-prediction revision cannot supersede another")
|
||||
if self.revision_no > 1 and self.supersedes_prediction_id is None:
|
||||
raise ValueError(
|
||||
"later self-prediction revision must supersede its predecessor"
|
||||
)
|
||||
if self.supersedes_prediction_id == self.prediction_id:
|
||||
raise ValueError("self-prediction revision cannot supersede itself")
|
||||
return self
|
||||
|
||||
|
||||
class LockedSelfPredictionHistory(BaseModel):
|
||||
"""외부 관찰 공개 시퀀스 이전에만 수정 가능한 append-only 자기예측."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
lock_id: str = Field(pattern=r"^oas-g5-lock-[a-z0-9-]+$")
|
||||
revisions: tuple[SelfPredictionRevision, ...] = Field(min_length=1)
|
||||
locked_sequence: int = Field(ge=1)
|
||||
external_reveal_sequence: int | None = Field(default=None, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_append_only_pre_reveal_history(self) -> "LockedSelfPredictionHistory":
|
||||
first = self.revisions[0]
|
||||
expected_revisions = list(range(1, len(self.revisions) + 1))
|
||||
if [item.revision_no for item in self.revisions] != expected_revisions:
|
||||
raise ValueError("self-prediction revisions must be contiguous and ordered")
|
||||
sequences = [item.recorded_sequence for item in self.revisions]
|
||||
if sequences != sorted(set(sequences)):
|
||||
raise ValueError(
|
||||
"self-prediction revision sequences must be unique and ordered"
|
||||
)
|
||||
for index, item in enumerate(self.revisions):
|
||||
if (
|
||||
item.competency_id != first.competency_id
|
||||
or item.practice_block_id != first.practice_block_id
|
||||
or item.scenario_variant_id != first.scenario_variant_id
|
||||
or item.phrase_family_id != first.phrase_family_id
|
||||
):
|
||||
raise ValueError(
|
||||
"one prediction history must keep one assessment target"
|
||||
)
|
||||
if (
|
||||
index
|
||||
and item.supersedes_prediction_id
|
||||
!= self.revisions[index - 1].prediction_id
|
||||
):
|
||||
raise ValueError(
|
||||
"prediction revision must supersede the immediate predecessor"
|
||||
)
|
||||
if self.locked_sequence < sequences[-1]:
|
||||
raise ValueError("prediction lock cannot precede its latest revision")
|
||||
if self.external_reveal_sequence is not None:
|
||||
if self.external_reveal_sequence <= self.locked_sequence:
|
||||
raise ValueError(
|
||||
"external evaluation must be revealed after prediction lock"
|
||||
)
|
||||
if any(
|
||||
item.recorded_sequence >= self.external_reveal_sequence
|
||||
for item in self.revisions
|
||||
):
|
||||
raise ValueError(
|
||||
"self-prediction cannot be revised after external reveal"
|
||||
)
|
||||
return self
|
||||
|
||||
@property
|
||||
def locked_prediction(self) -> SelfPredictionRevision:
|
||||
return self.revisions[-1]
|
||||
|
||||
|
||||
class IndependentPerformanceObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
observation_id: str = Field(pattern=r"^oas-g5-observation-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
status: PerformanceStatus
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective
|
||||
model_run_id: UUID | None = None
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
revealed_sequence: int = Field(ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_independent_observation_truth(
|
||||
self,
|
||||
) -> "IndependentPerformanceObservation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError(
|
||||
"performance observation mixes source and perspective layers"
|
||||
)
|
||||
if (
|
||||
self.source_kind in {"model_inferred", "agent_reported"}
|
||||
and self.model_run_id is None
|
||||
):
|
||||
raise ValueError("model/agent observation requires model_run_id")
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_refs:
|
||||
raise ValueError(
|
||||
"insufficient performance evidence must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_refs:
|
||||
raise ValueError("ready performance observation requires evidence")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed performance observation requires counterevidence")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationBlockInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
block_sequence: int = Field(ge=1)
|
||||
prediction_history: LockedSelfPredictionHistory
|
||||
observation: IndependentPerformanceObservation
|
||||
|
||||
@model_validator(mode="after")
|
||||
def align_prediction_and_observation(self) -> "CalibrationBlockInput":
|
||||
prediction = self.prediction_history.locked_prediction
|
||||
observed = self.observation
|
||||
for field in (
|
||||
"competency_id",
|
||||
"practice_block_id",
|
||||
"scenario_variant_id",
|
||||
"phrase_family_id",
|
||||
):
|
||||
if getattr(prediction, field) != getattr(observed, field):
|
||||
raise ValueError(f"calibration block mismatches {field}")
|
||||
if (
|
||||
self.prediction_history.external_reveal_sequence
|
||||
!= observed.revealed_sequence
|
||||
):
|
||||
raise ValueError("prediction history must record the exact external reveal")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationPair(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
practice_block_id: str
|
||||
prediction_id: str
|
||||
observation_id: str
|
||||
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
||||
observed_success: bool
|
||||
signed_error: float = Field(ge=-1.0, le=1.0)
|
||||
absolute_error: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class ConfidenceInterval(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
method: Literal["normal_95_bounded", "wilson_95"]
|
||||
lower: float = Field(ge=0.0, le=1.0)
|
||||
upper: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_order(self) -> "ConfidenceInterval":
|
||||
if self.lower > self.upper:
|
||||
raise ValueError("confidence interval lower bound exceeds upper bound")
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyCalibrationAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
pair_count: int = Field(ge=0)
|
||||
mean_absolute_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
mean_signed_error: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
error_interval: ConfidenceInterval | None = None
|
||||
bias: CalibrationBias
|
||||
baseline_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
recent_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
improvement: ImprovementStatus
|
||||
pairs: tuple[CalibrationPair, ...]
|
||||
excluded_block_ids: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_insufficient_calibration_scoreless(
|
||||
self,
|
||||
) -> "CompetencyCalibrationAssessment":
|
||||
if self.pair_count != len(self.pairs):
|
||||
raise ValueError("calibration pair_count must match pair evidence")
|
||||
if self.bias == "insufficient_evidence":
|
||||
if any(
|
||||
item is not None
|
||||
for item in (
|
||||
self.mean_absolute_error,
|
||||
self.mean_signed_error,
|
||||
self.error_interval,
|
||||
)
|
||||
):
|
||||
raise ValueError("insufficient calibration must remain scoreless")
|
||||
elif self.mean_absolute_error is None or self.mean_signed_error is None:
|
||||
raise ValueError("classified calibration requires error estimates")
|
||||
return self
|
||||
|
||||
|
||||
class MetacognitivePrescription(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
bias: CalibrationBias
|
||||
practice_mode: Literal[
|
||||
"counterevidence_forecast",
|
||||
"evidence_recall",
|
||||
"uncertainty_range",
|
||||
"collect_more_evidence",
|
||||
]
|
||||
instruction_ko: str = Field(min_length=10, max_length=500)
|
||||
completion_evidence: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class TransferVariation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
context_variant: str = Field(min_length=1, max_length=120)
|
||||
relationship_style: RelationshipStyle
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
expression_variant: str = Field(min_length=1, max_length=120)
|
||||
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
||||
scenario_family_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class TransferTrial(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
trial_id: str = Field(pattern=r"^oas-g5-transfer-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
||||
variation: TransferVariation
|
||||
status: PerformanceStatus
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_transfer_evidence(self) -> "TransferTrial":
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_refs:
|
||||
raise ValueError(
|
||||
"insufficient transfer trial must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_refs:
|
||||
raise ValueError("ready transfer trial requires evidence")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed transfer trial requires counterevidence")
|
||||
return self
|
||||
|
||||
|
||||
class NormalizedEvaluatorLabels(BaseModel):
|
||||
"""원문 없이 실제 회기 판정에 사용한 정규화 evaluator 라벨."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
technique_codes: tuple[str, ...] = ()
|
||||
client_state_codes: tuple[str, ...] = ()
|
||||
appropriateness: tuple[Literal["pos", "neutral", "warn"], ...] = ()
|
||||
intent_deviation_dimensions: tuple[str, ...] = ()
|
||||
evaluator_error_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ActualTransferExecution(BaseModel):
|
||||
"""서버 원장에서 재구성한 한 번의 실제 transfer 연습 실행."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
execution_event_id: UUID
|
||||
original_transfer_trial_record_id: UUID
|
||||
practice_session_id: UUID
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
||||
variation: TransferVariation
|
||||
status: PerformanceStatus
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_turn_ids: tuple[UUID, ...] = ()
|
||||
normalized_evaluator_labels: NormalizedEvaluatorLabels
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
model_run_id: UUID
|
||||
source_kind: Literal["model_inferred"] = "model_inferred"
|
||||
perspective: Literal["independent_observer"] = "independent_observer"
|
||||
instrument_id: Literal["unseen-transfer-g5"] = "unseen-transfer-g5"
|
||||
instrument_version: Literal["1.0.0"] = "1.0.0"
|
||||
observer_version: Literal["calibration-actual-transfer-observer-v1"] = (
|
||||
"calibration-actual-transfer-observer-v1"
|
||||
)
|
||||
training_phrase_collision: bool = False
|
||||
created_at: datetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_actual_execution_truth(self) -> "ActualTransferExecution":
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_turn_ids:
|
||||
raise ValueError(
|
||||
"insufficient actual execution must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_turn_ids:
|
||||
raise ValueError("observed actual execution requires durable turn UUIDs")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed actual execution requires counterevidence")
|
||||
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
||||
raise ValueError("actual execution turn evidence must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TransferSuiteInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
suite_id: str = Field(pattern=r"^oas-g5-suite-[a-z0-9-]+$")
|
||||
training_phrase_family_ids: tuple[str, ...] = Field(min_length=1)
|
||||
trials: tuple[TransferTrial, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_trials(self) -> "TransferSuiteInput":
|
||||
ids = [item.trial_id for item in self.trials]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("transfer trial ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TransferAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
trial_count: int = Field(ge=0)
|
||||
observed_trial_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
success_interval: ConfidenceInterval | None = None
|
||||
coverage: dict[str, int]
|
||||
eligible: bool
|
||||
transfer_verified: bool
|
||||
blockers: tuple[str, ...]
|
||||
evidence_refs: tuple[str, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_transfer_gate(self) -> "TransferAssessment":
|
||||
if self.transfer_verified and (
|
||||
not self.eligible
|
||||
or self.success_rate is None
|
||||
or self.success_rate < 0.85
|
||||
or not self.evidence_refs
|
||||
):
|
||||
raise ValueError(
|
||||
"transfer verification requires eligible evidence at target rate"
|
||||
)
|
||||
if not self.observed_trial_count and (
|
||||
self.success_rate is not None or self.success_interval is not None
|
||||
):
|
||||
raise ValueError("unobserved transfer must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class ActualTransferAssessment(BaseModel):
|
||||
"""실제 완료 회기만으로 계산한 역량별 transfer 상태."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
evidence_source: Literal["actual_practice_execution"] = "actual_practice_execution"
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
execution_count: int = Field(ge=0)
|
||||
independent_execution_count: int = Field(ge=0)
|
||||
observed_execution_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
success_interval: ConfidenceInterval | None = None
|
||||
coverage: dict[str, int]
|
||||
phrase_family_collision_count: int = Field(ge=0)
|
||||
eligible: bool
|
||||
actual_transfer_status: ActualTransferStatus
|
||||
blockers: tuple[str, ...]
|
||||
source_execution_event_ids: tuple[UUID, ...]
|
||||
evidence_turn_ids: tuple[UUID, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_actual_transfer_gate(self) -> "ActualTransferAssessment":
|
||||
if self.independent_execution_count > self.execution_count:
|
||||
raise ValueError("independent actual execution count exceeds total")
|
||||
if self.observed_execution_count > self.independent_execution_count:
|
||||
raise ValueError("observed actual execution count exceeds independent count")
|
||||
if self.actual_transfer_status == "verified" and (
|
||||
not self.eligible
|
||||
or self.success_rate is None
|
||||
or self.success_rate < 0.85
|
||||
or not self.evidence_turn_ids
|
||||
):
|
||||
raise ValueError(
|
||||
"actual transfer verification requires eligible durable evidence"
|
||||
)
|
||||
if self.actual_transfer_status == "not_verified" and not self.eligible:
|
||||
raise ValueError("classified actual transfer requires eligible evidence")
|
||||
if self.actual_transfer_status == "insufficient_evidence" and self.eligible:
|
||||
raise ValueError("eligible actual transfer cannot remain insufficient")
|
||||
if not self.observed_execution_count and (
|
||||
self.success_rate is not None or self.success_interval is not None
|
||||
):
|
||||
raise ValueError("unobserved actual transfer must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class SyntheticSubgroupResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
subgroup: str
|
||||
observed_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
interval: ConfidenceInterval | None = None
|
||||
|
||||
|
||||
class SubgroupDriftReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
status: DriftStatus
|
||||
max_rate_gap: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
compared_subgroups: tuple[str, ...]
|
||||
subgroup_results: tuple[SyntheticSubgroupResult, ...]
|
||||
threshold: float = Field(default=0.2, ge=0.0, le=1.0)
|
||||
notice_ko: str = Field(min_length=10, max_length=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_underpowered_drift_scoreless(self) -> "SubgroupDriftReport":
|
||||
if self.status == "insufficient_evidence" and self.max_rate_gap is not None:
|
||||
raise ValueError("underpowered subgroup report cannot claim a rate gap")
|
||||
if self.status != "insufficient_evidence" and self.max_rate_gap is None:
|
||||
raise ValueError("classified subgroup report requires an observed gap")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
calibration_improved: dict[str, bool] = Field(default_factory=dict)
|
||||
transfer_verified: dict[str, bool] = Field(default_factory=dict)
|
||||
drift_status: dict[str, DriftStatus] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g5-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
tags: tuple[str, ...] = Field(min_length=1)
|
||||
calibration_blocks: tuple[CalibrationBlockInput, ...] = ()
|
||||
transfer_suite: TransferSuiteInput | None = None
|
||||
expected: CalibrationTransferBenchmarkExpectation
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.calibration-transfer-benchmark.v1"]
|
||||
version: Literal["1.0.0"]
|
||||
data_classification: Literal["synthetic_educational"]
|
||||
clinical_claim_allowed: Literal[False]
|
||||
cases: tuple[CalibrationTransferBenchmarkCase, ...] = Field(min_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_adversarial_coverage(self) -> "CalibrationTransferBenchmarkPack":
|
||||
ids = [item.case_id for item in self.cases]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("calibration benchmark case ids must be unique")
|
||||
tags = {tag for item in self.cases for tag in item.tags}
|
||||
required = {
|
||||
"post_reveal_contamination",
|
||||
"memorized_phrase_transfer",
|
||||
"calibration_improvement",
|
||||
"synthetic_subgroup_drift",
|
||||
}
|
||||
if not required.issubset(tags):
|
||||
raise ValueError(
|
||||
"calibration benchmark lacks required adversarial coverage"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActualTransferAssessment",
|
||||
"ActualTransferExecution",
|
||||
"ActualTransferStatus",
|
||||
"CalibrationBias",
|
||||
"CalibrationBlockInput",
|
||||
"CalibrationPair",
|
||||
"CalibrationTransferBenchmarkCase",
|
||||
"CalibrationTransferBenchmarkExpectation",
|
||||
"CalibrationTransferBenchmarkPack",
|
||||
"CompetencyCalibrationAssessment",
|
||||
"ConfidenceInterval",
|
||||
"DriftStatus",
|
||||
"ImprovementStatus",
|
||||
"IndependentPerformanceObservation",
|
||||
"LockedSelfPredictionHistory",
|
||||
"MetacognitivePrescription",
|
||||
"NormalizedEvaluatorLabels",
|
||||
"PerformanceStatus",
|
||||
"RelationshipStyle",
|
||||
"SelfPredictionRevision",
|
||||
"SubgroupDriftReport",
|
||||
"SyntheticSubgroupResult",
|
||||
"TransferAssessment",
|
||||
"TransferSuiteInput",
|
||||
"TransferTrial",
|
||||
"TransferVariation",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue