16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
564 lines
23 KiB
Python
564 lines
23 KiB
Python
"""G5 Calibration Mirror & Transfer의 버전 고정 순수 도메인 계약.
|
|
|
|
외부 평가 공개 전에 잠근 자기예측과 독립 관찰을 역량별로만 대조한다.
|
|
전이는 익숙한 문장 재현과 분리하고, 합성 subgroup 차이는 표본 근거가
|
|
충분할 때만 drift 신호로 남긴다. 단일 총점이나 임상 주장은 허용하지 않는다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from typing import Literal
|
|
from uuid import UUID
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
|
from .measurement import (
|
|
MeasurementPerspective,
|
|
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
|
SourceKind,
|
|
)
|
|
|
|
|
|
CalibrationBias = Literal[
|
|
"overconfident", "underconfident", "aligned", "insufficient_evidence"
|
|
]
|
|
ImprovementStatus = Literal["improved", "not_improved", "insufficient_evidence"]
|
|
PerformanceStatus = Literal["passed", "failed", "insufficient_evidence"]
|
|
RelationshipStyle = Literal[
|
|
"collaborative", "withdrawn", "confrontational", "ambivalent"
|
|
]
|
|
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
|
ActualTransferStatus = Literal["verified", "not_verified", "insufficient_evidence"]
|
|
|
|
|
|
class SelfPredictionRevision(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
prediction_id: str = Field(pattern=r"^oas-g5-prediction-[a-z0-9-]+$")
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
phrase_family_id: str = Field(min_length=1, max_length=180)
|
|
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
|
confidence: float = Field(ge=0.0, le=1.0)
|
|
revision_no: int = Field(ge=1)
|
|
supersedes_prediction_id: str | None = None
|
|
recorded_sequence: int = Field(ge=1)
|
|
revision_reason: str = Field(min_length=1, max_length=300)
|
|
|
|
@model_validator(mode="after")
|
|
def require_revision_link(self) -> "SelfPredictionRevision":
|
|
if self.revision_no == 1 and self.supersedes_prediction_id is not None:
|
|
raise ValueError("first self-prediction revision cannot supersede another")
|
|
if self.revision_no > 1 and self.supersedes_prediction_id is None:
|
|
raise ValueError(
|
|
"later self-prediction revision must supersede its predecessor"
|
|
)
|
|
if self.supersedes_prediction_id == self.prediction_id:
|
|
raise ValueError("self-prediction revision cannot supersede itself")
|
|
return self
|
|
|
|
|
|
class LockedSelfPredictionHistory(BaseModel):
|
|
"""외부 관찰 공개 시퀀스 이전에만 수정 가능한 append-only 자기예측."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
lock_id: str = Field(pattern=r"^oas-g5-lock-[a-z0-9-]+$")
|
|
revisions: tuple[SelfPredictionRevision, ...] = Field(min_length=1)
|
|
locked_sequence: int = Field(ge=1)
|
|
external_reveal_sequence: int | None = Field(default=None, ge=1)
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_append_only_pre_reveal_history(self) -> "LockedSelfPredictionHistory":
|
|
first = self.revisions[0]
|
|
expected_revisions = list(range(1, len(self.revisions) + 1))
|
|
if [item.revision_no for item in self.revisions] != expected_revisions:
|
|
raise ValueError("self-prediction revisions must be contiguous and ordered")
|
|
sequences = [item.recorded_sequence for item in self.revisions]
|
|
if sequences != sorted(set(sequences)):
|
|
raise ValueError(
|
|
"self-prediction revision sequences must be unique and ordered"
|
|
)
|
|
for index, item in enumerate(self.revisions):
|
|
if (
|
|
item.competency_id != first.competency_id
|
|
or item.practice_block_id != first.practice_block_id
|
|
or item.scenario_variant_id != first.scenario_variant_id
|
|
or item.phrase_family_id != first.phrase_family_id
|
|
):
|
|
raise ValueError(
|
|
"one prediction history must keep one assessment target"
|
|
)
|
|
if (
|
|
index
|
|
and item.supersedes_prediction_id
|
|
!= self.revisions[index - 1].prediction_id
|
|
):
|
|
raise ValueError(
|
|
"prediction revision must supersede the immediate predecessor"
|
|
)
|
|
if self.locked_sequence < sequences[-1]:
|
|
raise ValueError("prediction lock cannot precede its latest revision")
|
|
if self.external_reveal_sequence is not None:
|
|
if self.external_reveal_sequence <= self.locked_sequence:
|
|
raise ValueError(
|
|
"external evaluation must be revealed after prediction lock"
|
|
)
|
|
if any(
|
|
item.recorded_sequence >= self.external_reveal_sequence
|
|
for item in self.revisions
|
|
):
|
|
raise ValueError(
|
|
"self-prediction cannot be revised after external reveal"
|
|
)
|
|
return self
|
|
|
|
@property
|
|
def locked_prediction(self) -> SelfPredictionRevision:
|
|
return self.revisions[-1]
|
|
|
|
|
|
class IndependentPerformanceObservation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
observation_id: str = Field(pattern=r"^oas-g5-observation-[a-z0-9-]+$")
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
phrase_family_id: str = Field(min_length=1, max_length=180)
|
|
status: PerformanceStatus
|
|
source_kind: SourceKind
|
|
perspective: MeasurementPerspective
|
|
model_run_id: UUID | None = None
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence_refs: tuple[str, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
revealed_sequence: int = Field(ge=1)
|
|
|
|
@model_validator(mode="after")
|
|
def preserve_independent_observation_truth(
|
|
self,
|
|
) -> "IndependentPerformanceObservation":
|
|
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
|
raise ValueError(
|
|
"performance observation mixes source and perspective layers"
|
|
)
|
|
if (
|
|
self.source_kind in {"model_inferred", "agent_reported"}
|
|
and self.model_run_id is None
|
|
):
|
|
raise ValueError("model/agent observation requires model_run_id")
|
|
if self.status == "insufficient_evidence":
|
|
if self.uncertainty != 1.0 or self.evidence_refs:
|
|
raise ValueError(
|
|
"insufficient performance evidence must remain evidence-free"
|
|
)
|
|
elif not self.evidence_refs:
|
|
raise ValueError("ready performance observation requires evidence")
|
|
if self.status == "failed" and not self.counterevidence:
|
|
raise ValueError("failed performance observation requires counterevidence")
|
|
return self
|
|
|
|
|
|
class CalibrationBlockInput(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
block_sequence: int = Field(ge=1)
|
|
prediction_history: LockedSelfPredictionHistory
|
|
observation: IndependentPerformanceObservation
|
|
|
|
@model_validator(mode="after")
|
|
def align_prediction_and_observation(self) -> "CalibrationBlockInput":
|
|
prediction = self.prediction_history.locked_prediction
|
|
observed = self.observation
|
|
for field in (
|
|
"competency_id",
|
|
"practice_block_id",
|
|
"scenario_variant_id",
|
|
"phrase_family_id",
|
|
):
|
|
if getattr(prediction, field) != getattr(observed, field):
|
|
raise ValueError(f"calibration block mismatches {field}")
|
|
if (
|
|
self.prediction_history.external_reveal_sequence
|
|
!= observed.revealed_sequence
|
|
):
|
|
raise ValueError("prediction history must record the exact external reveal")
|
|
return self
|
|
|
|
|
|
class CalibrationPair(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
practice_block_id: str
|
|
prediction_id: str
|
|
observation_id: str
|
|
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
|
observed_success: bool
|
|
signed_error: float = Field(ge=-1.0, le=1.0)
|
|
absolute_error: float = Field(ge=0.0, le=1.0)
|
|
confidence: float = Field(ge=0.0, le=1.0)
|
|
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
|
|
|
|
|
class ConfidenceInterval(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
method: Literal["normal_95_bounded", "wilson_95"]
|
|
lower: float = Field(ge=0.0, le=1.0)
|
|
upper: float = Field(ge=0.0, le=1.0)
|
|
|
|
@model_validator(mode="after")
|
|
def require_order(self) -> "ConfidenceInterval":
|
|
if self.lower > self.upper:
|
|
raise ValueError("confidence interval lower bound exceeds upper bound")
|
|
return self
|
|
|
|
|
|
class CompetencyCalibrationAssessment(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str
|
|
pair_count: int = Field(ge=0)
|
|
mean_absolute_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
mean_signed_error: float | None = Field(default=None, ge=-1.0, le=1.0)
|
|
error_interval: ConfidenceInterval | None = None
|
|
bias: CalibrationBias
|
|
baseline_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
recent_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
improvement: ImprovementStatus
|
|
pairs: tuple[CalibrationPair, ...]
|
|
excluded_block_ids: tuple[str, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
|
|
@model_validator(mode="after")
|
|
def keep_insufficient_calibration_scoreless(
|
|
self,
|
|
) -> "CompetencyCalibrationAssessment":
|
|
if self.pair_count != len(self.pairs):
|
|
raise ValueError("calibration pair_count must match pair evidence")
|
|
if self.bias == "insufficient_evidence":
|
|
if any(
|
|
item is not None
|
|
for item in (
|
|
self.mean_absolute_error,
|
|
self.mean_signed_error,
|
|
self.error_interval,
|
|
)
|
|
):
|
|
raise ValueError("insufficient calibration must remain scoreless")
|
|
elif self.mean_absolute_error is None or self.mean_signed_error is None:
|
|
raise ValueError("classified calibration requires error estimates")
|
|
return self
|
|
|
|
|
|
class MetacognitivePrescription(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str
|
|
bias: CalibrationBias
|
|
practice_mode: Literal[
|
|
"counterevidence_forecast",
|
|
"evidence_recall",
|
|
"uncertainty_range",
|
|
"collect_more_evidence",
|
|
]
|
|
instruction_ko: str = Field(min_length=10, max_length=500)
|
|
completion_evidence: tuple[str, ...] = Field(min_length=1)
|
|
|
|
|
|
class TransferVariation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
context_variant: str = Field(min_length=1, max_length=120)
|
|
relationship_style: RelationshipStyle
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
expression_variant: str = Field(min_length=1, max_length=120)
|
|
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
|
scenario_family_id: str = Field(min_length=1, max_length=180)
|
|
phrase_family_id: str = Field(min_length=1, max_length=180)
|
|
|
|
|
|
class TransferTrial(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
trial_id: str = Field(pattern=r"^oas-g5-transfer-[a-z0-9-]+$")
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
|
variation: TransferVariation
|
|
status: PerformanceStatus
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence_refs: tuple[str, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
|
|
@model_validator(mode="after")
|
|
def preserve_transfer_evidence(self) -> "TransferTrial":
|
|
if self.status == "insufficient_evidence":
|
|
if self.uncertainty != 1.0 or self.evidence_refs:
|
|
raise ValueError(
|
|
"insufficient transfer trial must remain evidence-free"
|
|
)
|
|
elif not self.evidence_refs:
|
|
raise ValueError("ready transfer trial requires evidence")
|
|
if self.status == "failed" and not self.counterevidence:
|
|
raise ValueError("failed transfer trial requires counterevidence")
|
|
return self
|
|
|
|
|
|
class NormalizedEvaluatorLabels(BaseModel):
|
|
"""원문 없이 실제 회기 판정에 사용한 정규화 evaluator 라벨."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
technique_codes: tuple[str, ...] = ()
|
|
client_state_codes: tuple[str, ...] = ()
|
|
appropriateness: tuple[Literal["pos", "neutral", "warn"], ...] = ()
|
|
intent_deviation_dimensions: tuple[str, ...] = ()
|
|
evaluator_error_count: int = Field(ge=0)
|
|
|
|
|
|
class ActualTransferExecution(BaseModel):
|
|
"""서버 원장에서 재구성한 한 번의 실제 transfer 연습 실행."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
execution_event_id: UUID
|
|
original_transfer_trial_record_id: UUID
|
|
practice_session_id: UUID
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
|
variation: TransferVariation
|
|
status: PerformanceStatus
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence_turn_ids: tuple[UUID, ...] = ()
|
|
normalized_evaluator_labels: NormalizedEvaluatorLabels
|
|
counterevidence: tuple[str, ...] = ()
|
|
model_run_id: UUID
|
|
source_kind: Literal["model_inferred"] = "model_inferred"
|
|
perspective: Literal["independent_observer"] = "independent_observer"
|
|
instrument_id: Literal["unseen-transfer-g5"] = "unseen-transfer-g5"
|
|
instrument_version: Literal["1.0.0"] = "1.0.0"
|
|
observer_version: Literal["calibration-actual-transfer-observer-v1"] = (
|
|
"calibration-actual-transfer-observer-v1"
|
|
)
|
|
training_phrase_collision: bool = False
|
|
created_at: datetime
|
|
|
|
@model_validator(mode="after")
|
|
def preserve_actual_execution_truth(self) -> "ActualTransferExecution":
|
|
if self.status == "insufficient_evidence":
|
|
if self.uncertainty != 1.0 or self.evidence_turn_ids:
|
|
raise ValueError(
|
|
"insufficient actual execution must remain evidence-free"
|
|
)
|
|
elif not self.evidence_turn_ids:
|
|
raise ValueError("observed actual execution requires durable turn UUIDs")
|
|
if self.status == "failed" and not self.counterevidence:
|
|
raise ValueError("failed actual execution requires counterevidence")
|
|
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
|
raise ValueError("actual execution turn evidence must be unique")
|
|
return self
|
|
|
|
|
|
class TransferSuiteInput(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
suite_id: str = Field(pattern=r"^oas-g5-suite-[a-z0-9-]+$")
|
|
training_phrase_family_ids: tuple[str, ...] = Field(min_length=1)
|
|
trials: tuple[TransferTrial, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def require_unique_trials(self) -> "TransferSuiteInput":
|
|
ids = [item.trial_id for item in self.trials]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError("transfer trial ids must be unique")
|
|
return self
|
|
|
|
|
|
class TransferAssessment(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str
|
|
trial_count: int = Field(ge=0)
|
|
observed_trial_count: int = Field(ge=0)
|
|
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
success_interval: ConfidenceInterval | None = None
|
|
coverage: dict[str, int]
|
|
eligible: bool
|
|
transfer_verified: bool
|
|
blockers: tuple[str, ...]
|
|
evidence_refs: tuple[str, ...]
|
|
counterevidence: tuple[str, ...]
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_transfer_gate(self) -> "TransferAssessment":
|
|
if self.transfer_verified and (
|
|
not self.eligible
|
|
or self.success_rate is None
|
|
or self.success_rate < 0.85
|
|
or not self.evidence_refs
|
|
):
|
|
raise ValueError(
|
|
"transfer verification requires eligible evidence at target rate"
|
|
)
|
|
if not self.observed_trial_count and (
|
|
self.success_rate is not None or self.success_interval is not None
|
|
):
|
|
raise ValueError("unobserved transfer must remain scoreless")
|
|
return self
|
|
|
|
|
|
class ActualTransferAssessment(BaseModel):
|
|
"""실제 완료 회기만으로 계산한 역량별 transfer 상태."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
evidence_source: Literal["actual_practice_execution"] = "actual_practice_execution"
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
execution_count: int = Field(ge=0)
|
|
independent_execution_count: int = Field(ge=0)
|
|
observed_execution_count: int = Field(ge=0)
|
|
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
success_interval: ConfidenceInterval | None = None
|
|
coverage: dict[str, int]
|
|
phrase_family_collision_count: int = Field(ge=0)
|
|
eligible: bool
|
|
actual_transfer_status: ActualTransferStatus
|
|
blockers: tuple[str, ...]
|
|
source_execution_event_ids: tuple[UUID, ...]
|
|
evidence_turn_ids: tuple[UUID, ...]
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_actual_transfer_gate(self) -> "ActualTransferAssessment":
|
|
if self.independent_execution_count > self.execution_count:
|
|
raise ValueError("independent actual execution count exceeds total")
|
|
if self.observed_execution_count > self.independent_execution_count:
|
|
raise ValueError("observed actual execution count exceeds independent count")
|
|
if self.actual_transfer_status == "verified" and (
|
|
not self.eligible
|
|
or self.success_rate is None
|
|
or self.success_rate < 0.85
|
|
or not self.evidence_turn_ids
|
|
):
|
|
raise ValueError(
|
|
"actual transfer verification requires eligible durable evidence"
|
|
)
|
|
if self.actual_transfer_status == "not_verified" and not self.eligible:
|
|
raise ValueError("classified actual transfer requires eligible evidence")
|
|
if self.actual_transfer_status == "insufficient_evidence" and self.eligible:
|
|
raise ValueError("eligible actual transfer cannot remain insufficient")
|
|
if not self.observed_execution_count and (
|
|
self.success_rate is not None or self.success_interval is not None
|
|
):
|
|
raise ValueError("unobserved actual transfer must remain scoreless")
|
|
return self
|
|
|
|
|
|
class SyntheticSubgroupResult(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
subgroup: str
|
|
observed_count: int = Field(ge=0)
|
|
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
interval: ConfidenceInterval | None = None
|
|
|
|
|
|
class SubgroupDriftReport(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str
|
|
status: DriftStatus
|
|
max_rate_gap: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
compared_subgroups: tuple[str, ...]
|
|
subgroup_results: tuple[SyntheticSubgroupResult, ...]
|
|
threshold: float = Field(default=0.2, ge=0.0, le=1.0)
|
|
notice_ko: str = Field(min_length=10, max_length=500)
|
|
|
|
@model_validator(mode="after")
|
|
def keep_underpowered_drift_scoreless(self) -> "SubgroupDriftReport":
|
|
if self.status == "insufficient_evidence" and self.max_rate_gap is not None:
|
|
raise ValueError("underpowered subgroup report cannot claim a rate gap")
|
|
if self.status != "insufficient_evidence" and self.max_rate_gap is None:
|
|
raise ValueError("classified subgroup report requires an observed gap")
|
|
return self
|
|
|
|
|
|
class CalibrationTransferBenchmarkExpectation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
calibration_improved: dict[str, bool] = Field(default_factory=dict)
|
|
transfer_verified: dict[str, bool] = Field(default_factory=dict)
|
|
drift_status: dict[str, DriftStatus] = Field(default_factory=dict)
|
|
|
|
|
|
class CalibrationTransferBenchmarkCase(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
case_id: str = Field(pattern=r"^oas-g5-bench-[0-9]{3}$")
|
|
title_ko: str = Field(min_length=1, max_length=200)
|
|
tags: tuple[str, ...] = Field(min_length=1)
|
|
calibration_blocks: tuple[CalibrationBlockInput, ...] = ()
|
|
transfer_suite: TransferSuiteInput | None = None
|
|
expected: CalibrationTransferBenchmarkExpectation
|
|
|
|
|
|
class CalibrationTransferBenchmarkPack(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.calibration-transfer-benchmark.v1"]
|
|
version: Literal["1.0.0"]
|
|
data_classification: Literal["synthetic_educational"]
|
|
clinical_claim_allowed: Literal[False]
|
|
cases: tuple[CalibrationTransferBenchmarkCase, ...] = Field(min_length=4)
|
|
|
|
@model_validator(mode="after")
|
|
def require_adversarial_coverage(self) -> "CalibrationTransferBenchmarkPack":
|
|
ids = [item.case_id for item in self.cases]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError("calibration benchmark case ids must be unique")
|
|
tags = {tag for item in self.cases for tag in item.tags}
|
|
required = {
|
|
"post_reveal_contamination",
|
|
"memorized_phrase_transfer",
|
|
"calibration_improvement",
|
|
"synthetic_subgroup_drift",
|
|
}
|
|
if not required.issubset(tags):
|
|
raise ValueError(
|
|
"calibration benchmark lacks required adversarial coverage"
|
|
)
|
|
return self
|
|
|
|
|
|
__all__ = [
|
|
"ActualTransferAssessment",
|
|
"ActualTransferExecution",
|
|
"ActualTransferStatus",
|
|
"CalibrationBias",
|
|
"CalibrationBlockInput",
|
|
"CalibrationPair",
|
|
"CalibrationTransferBenchmarkCase",
|
|
"CalibrationTransferBenchmarkExpectation",
|
|
"CalibrationTransferBenchmarkPack",
|
|
"CompetencyCalibrationAssessment",
|
|
"ConfidenceInterval",
|
|
"DriftStatus",
|
|
"ImprovementStatus",
|
|
"IndependentPerformanceObservation",
|
|
"LockedSelfPredictionHistory",
|
|
"MetacognitivePrescription",
|
|
"NormalizedEvaluatorLabels",
|
|
"PerformanceStatus",
|
|
"RelationshipStyle",
|
|
"SelfPredictionRevision",
|
|
"SubgroupDriftReport",
|
|
"SyntheticSubgroupResult",
|
|
"TransferAssessment",
|
|
"TransferSuiteInput",
|
|
"TransferTrial",
|
|
"TransferVariation",
|
|
]
|