"""G5 Calibration Mirror & Transfer의 버전 고정 순수 도메인 계약. 외부 평가 공개 전에 잠근 자기예측과 독립 관찰을 역량별로만 대조한다. 전이는 익숙한 문장 재현과 분리하고, 합성 subgroup 차이는 표본 근거가 충분할 때만 drift 신호로 남긴다. 단일 총점이나 임상 주장은 허용하지 않는다. """ from __future__ import annotations from datetime import datetime from typing import Literal from uuid import UUID from pydantic import BaseModel, ConfigDict, Field, model_validator from .measurement import ( MeasurementPerspective, SOURCE_PERSPECTIVE_COMPATIBILITY, SourceKind, ) CalibrationBias = Literal[ "overconfident", "underconfident", "aligned", "insufficient_evidence" ] ImprovementStatus = Literal["improved", "not_improved", "insufficient_evidence"] PerformanceStatus = Literal["passed", "failed", "insufficient_evidence"] RelationshipStyle = Literal[ "collaborative", "withdrawn", "confrontational", "ambivalent" ] DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"] ActualTransferStatus = Literal["verified", "not_verified", "insufficient_evidence"] class SelfPredictionRevision(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) prediction_id: str = Field(pattern=r"^oas-g5-prediction-[a-z0-9-]+$") competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$") practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$") scenario_variant_id: str = Field(min_length=1, max_length=180) phrase_family_id: str = Field(min_length=1, max_length=180) predicted_success_probability: float = Field(ge=0.0, le=1.0) confidence: float = Field(ge=0.0, le=1.0) revision_no: int = Field(ge=1) supersedes_prediction_id: str | None = None recorded_sequence: int = Field(ge=1) revision_reason: str = Field(min_length=1, max_length=300) @model_validator(mode="after") def require_revision_link(self) -> "SelfPredictionRevision": if self.revision_no == 1 and self.supersedes_prediction_id is not None: raise ValueError("first self-prediction revision cannot supersede another") if self.revision_no > 1 and self.supersedes_prediction_id is None: raise ValueError( "later self-prediction revision must supersede its predecessor" ) if self.supersedes_prediction_id == self.prediction_id: raise ValueError("self-prediction revision cannot supersede itself") return self class LockedSelfPredictionHistory(BaseModel): """외부 관찰 공개 시퀀스 이전에만 수정 가능한 append-only 자기예측.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) lock_id: str = Field(pattern=r"^oas-g5-lock-[a-z0-9-]+$") revisions: tuple[SelfPredictionRevision, ...] = Field(min_length=1) locked_sequence: int = Field(ge=1) external_reveal_sequence: int | None = Field(default=None, ge=1) @model_validator(mode="after") def enforce_append_only_pre_reveal_history(self) -> "LockedSelfPredictionHistory": first = self.revisions[0] expected_revisions = list(range(1, len(self.revisions) + 1)) if [item.revision_no for item in self.revisions] != expected_revisions: raise ValueError("self-prediction revisions must be contiguous and ordered") sequences = [item.recorded_sequence for item in self.revisions] if sequences != sorted(set(sequences)): raise ValueError( "self-prediction revision sequences must be unique and ordered" ) for index, item in enumerate(self.revisions): if ( item.competency_id != first.competency_id or item.practice_block_id != first.practice_block_id or item.scenario_variant_id != first.scenario_variant_id or item.phrase_family_id != first.phrase_family_id ): raise ValueError( "one prediction history must keep one assessment target" ) if ( index and item.supersedes_prediction_id != self.revisions[index - 1].prediction_id ): raise ValueError( "prediction revision must supersede the immediate predecessor" ) if self.locked_sequence < sequences[-1]: raise ValueError("prediction lock cannot precede its latest revision") if self.external_reveal_sequence is not None: if self.external_reveal_sequence <= self.locked_sequence: raise ValueError( "external evaluation must be revealed after prediction lock" ) if any( item.recorded_sequence >= self.external_reveal_sequence for item in self.revisions ): raise ValueError( "self-prediction cannot be revised after external reveal" ) return self @property def locked_prediction(self) -> SelfPredictionRevision: return self.revisions[-1] class IndependentPerformanceObservation(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) observation_id: str = Field(pattern=r"^oas-g5-observation-[a-z0-9-]+$") competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$") practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$") scenario_variant_id: str = Field(min_length=1, max_length=180) phrase_family_id: str = Field(min_length=1, max_length=180) status: PerformanceStatus source_kind: SourceKind perspective: MeasurementPerspective model_run_id: UUID | None = None uncertainty: float = Field(ge=0.0, le=1.0) evidence_refs: tuple[str, ...] = () counterevidence: tuple[str, ...] = () revealed_sequence: int = Field(ge=1) @model_validator(mode="after") def preserve_independent_observation_truth( self, ) -> "IndependentPerformanceObservation": if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]: raise ValueError( "performance observation mixes source and perspective layers" ) if ( self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None ): raise ValueError("model/agent observation requires model_run_id") if self.status == "insufficient_evidence": if self.uncertainty != 1.0 or self.evidence_refs: raise ValueError( "insufficient performance evidence must remain evidence-free" ) elif not self.evidence_refs: raise ValueError("ready performance observation requires evidence") if self.status == "failed" and not self.counterevidence: raise ValueError("failed performance observation requires counterevidence") return self class CalibrationBlockInput(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) block_sequence: int = Field(ge=1) prediction_history: LockedSelfPredictionHistory observation: IndependentPerformanceObservation @model_validator(mode="after") def align_prediction_and_observation(self) -> "CalibrationBlockInput": prediction = self.prediction_history.locked_prediction observed = self.observation for field in ( "competency_id", "practice_block_id", "scenario_variant_id", "phrase_family_id", ): if getattr(prediction, field) != getattr(observed, field): raise ValueError(f"calibration block mismatches {field}") if ( self.prediction_history.external_reveal_sequence != observed.revealed_sequence ): raise ValueError("prediction history must record the exact external reveal") return self class CalibrationPair(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) practice_block_id: str prediction_id: str observation_id: str predicted_success_probability: float = Field(ge=0.0, le=1.0) observed_success: bool signed_error: float = Field(ge=-1.0, le=1.0) absolute_error: float = Field(ge=0.0, le=1.0) confidence: float = Field(ge=0.0, le=1.0) evidence_refs: tuple[str, ...] = Field(min_length=1) class ConfidenceInterval(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) method: Literal["normal_95_bounded", "wilson_95"] lower: float = Field(ge=0.0, le=1.0) upper: float = Field(ge=0.0, le=1.0) @model_validator(mode="after") def require_order(self) -> "ConfidenceInterval": if self.lower > self.upper: raise ValueError("confidence interval lower bound exceeds upper bound") return self class CompetencyCalibrationAssessment(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) competency_id: str pair_count: int = Field(ge=0) mean_absolute_error: float | None = Field(default=None, ge=0.0, le=1.0) mean_signed_error: float | None = Field(default=None, ge=-1.0, le=1.0) error_interval: ConfidenceInterval | None = None bias: CalibrationBias baseline_error: float | None = Field(default=None, ge=0.0, le=1.0) recent_error: float | None = Field(default=None, ge=0.0, le=1.0) improvement: ImprovementStatus pairs: tuple[CalibrationPair, ...] excluded_block_ids: tuple[str, ...] = () counterevidence: tuple[str, ...] = () @model_validator(mode="after") def keep_insufficient_calibration_scoreless( self, ) -> "CompetencyCalibrationAssessment": if self.pair_count != len(self.pairs): raise ValueError("calibration pair_count must match pair evidence") if self.bias == "insufficient_evidence": if any( item is not None for item in ( self.mean_absolute_error, self.mean_signed_error, self.error_interval, ) ): raise ValueError("insufficient calibration must remain scoreless") elif self.mean_absolute_error is None or self.mean_signed_error is None: raise ValueError("classified calibration requires error estimates") return self class MetacognitivePrescription(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) competency_id: str bias: CalibrationBias practice_mode: Literal[ "counterevidence_forecast", "evidence_recall", "uncertainty_range", "collect_more_evidence", ] instruction_ko: str = Field(min_length=10, max_length=500) completion_evidence: tuple[str, ...] = Field(min_length=1) class TransferVariation(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) context_variant: str = Field(min_length=1, max_length=120) relationship_style: RelationshipStyle difficulty_level: int = Field(ge=1, le=5) expression_variant: str = Field(min_length=1, max_length=120) synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$") scenario_family_id: str = Field(min_length=1, max_length=180) phrase_family_id: str = Field(min_length=1, max_length=180) class TransferTrial(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) trial_id: str = Field(pattern=r"^oas-g5-transfer-[a-z0-9-]+$") competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$") scenario_variant_id: str = Field(min_length=1, max_length=180) scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer" variation: TransferVariation status: PerformanceStatus uncertainty: float = Field(ge=0.0, le=1.0) evidence_refs: tuple[str, ...] = () counterevidence: tuple[str, ...] = () @model_validator(mode="after") def preserve_transfer_evidence(self) -> "TransferTrial": if self.status == "insufficient_evidence": if self.uncertainty != 1.0 or self.evidence_refs: raise ValueError( "insufficient transfer trial must remain evidence-free" ) elif not self.evidence_refs: raise ValueError("ready transfer trial requires evidence") if self.status == "failed" and not self.counterevidence: raise ValueError("failed transfer trial requires counterevidence") return self class NormalizedEvaluatorLabels(BaseModel): """원문 없이 실제 회기 판정에 사용한 정규화 evaluator 라벨.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) technique_codes: tuple[str, ...] = () client_state_codes: tuple[str, ...] = () appropriateness: tuple[Literal["pos", "neutral", "warn"], ...] = () intent_deviation_dimensions: tuple[str, ...] = () evaluator_error_count: int = Field(ge=0) class ActualTransferExecution(BaseModel): """서버 원장에서 재구성한 한 번의 실제 transfer 연습 실행.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) execution_event_id: UUID original_transfer_trial_record_id: UUID practice_session_id: UUID competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$") scenario_variant_id: str = Field(min_length=1, max_length=180) scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer" variation: TransferVariation status: PerformanceStatus uncertainty: float = Field(ge=0.0, le=1.0) evidence_turn_ids: tuple[UUID, ...] = () normalized_evaluator_labels: NormalizedEvaluatorLabels counterevidence: tuple[str, ...] = () model_run_id: UUID source_kind: Literal["model_inferred"] = "model_inferred" perspective: Literal["independent_observer"] = "independent_observer" instrument_id: Literal["unseen-transfer-g5"] = "unseen-transfer-g5" instrument_version: Literal["1.0.0"] = "1.0.0" observer_version: Literal["calibration-actual-transfer-observer-v1"] = ( "calibration-actual-transfer-observer-v1" ) training_phrase_collision: bool = False created_at: datetime @model_validator(mode="after") def preserve_actual_execution_truth(self) -> "ActualTransferExecution": if self.status == "insufficient_evidence": if self.uncertainty != 1.0 or self.evidence_turn_ids: raise ValueError( "insufficient actual execution must remain evidence-free" ) elif not self.evidence_turn_ids: raise ValueError("observed actual execution requires durable turn UUIDs") if self.status == "failed" and not self.counterevidence: raise ValueError("failed actual execution requires counterevidence") if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids): raise ValueError("actual execution turn evidence must be unique") return self class TransferSuiteInput(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) suite_id: str = Field(pattern=r"^oas-g5-suite-[a-z0-9-]+$") training_phrase_family_ids: tuple[str, ...] = Field(min_length=1) trials: tuple[TransferTrial, ...] = Field(min_length=1) @model_validator(mode="after") def require_unique_trials(self) -> "TransferSuiteInput": ids = [item.trial_id for item in self.trials] if len(set(ids)) != len(ids): raise ValueError("transfer trial ids must be unique") return self class TransferAssessment(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) competency_id: str trial_count: int = Field(ge=0) observed_trial_count: int = Field(ge=0) success_rate: float | None = Field(default=None, ge=0.0, le=1.0) success_interval: ConfidenceInterval | None = None coverage: dict[str, int] eligible: bool transfer_verified: bool blockers: tuple[str, ...] evidence_refs: tuple[str, ...] counterevidence: tuple[str, ...] @model_validator(mode="after") def enforce_transfer_gate(self) -> "TransferAssessment": if self.transfer_verified and ( not self.eligible or self.success_rate is None or self.success_rate < 0.85 or not self.evidence_refs ): raise ValueError( "transfer verification requires eligible evidence at target rate" ) if not self.observed_trial_count and ( self.success_rate is not None or self.success_interval is not None ): raise ValueError("unobserved transfer must remain scoreless") return self class ActualTransferAssessment(BaseModel): """실제 완료 회기만으로 계산한 역량별 transfer 상태.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) evidence_source: Literal["actual_practice_execution"] = "actual_practice_execution" competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$") execution_count: int = Field(ge=0) independent_execution_count: int = Field(ge=0) observed_execution_count: int = Field(ge=0) success_rate: float | None = Field(default=None, ge=0.0, le=1.0) success_interval: ConfidenceInterval | None = None coverage: dict[str, int] phrase_family_collision_count: int = Field(ge=0) eligible: bool actual_transfer_status: ActualTransferStatus blockers: tuple[str, ...] source_execution_event_ids: tuple[UUID, ...] evidence_turn_ids: tuple[UUID, ...] @model_validator(mode="after") def enforce_actual_transfer_gate(self) -> "ActualTransferAssessment": if self.independent_execution_count > self.execution_count: raise ValueError("independent actual execution count exceeds total") if self.observed_execution_count > self.independent_execution_count: raise ValueError("observed actual execution count exceeds independent count") if self.actual_transfer_status == "verified" and ( not self.eligible or self.success_rate is None or self.success_rate < 0.85 or not self.evidence_turn_ids ): raise ValueError( "actual transfer verification requires eligible durable evidence" ) if self.actual_transfer_status == "not_verified" and not self.eligible: raise ValueError("classified actual transfer requires eligible evidence") if self.actual_transfer_status == "insufficient_evidence" and self.eligible: raise ValueError("eligible actual transfer cannot remain insufficient") if not self.observed_execution_count and ( self.success_rate is not None or self.success_interval is not None ): raise ValueError("unobserved actual transfer must remain scoreless") return self class SyntheticSubgroupResult(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) subgroup: str observed_count: int = Field(ge=0) success_rate: float | None = Field(default=None, ge=0.0, le=1.0) interval: ConfidenceInterval | None = None class SubgroupDriftReport(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) competency_id: str status: DriftStatus max_rate_gap: float | None = Field(default=None, ge=0.0, le=1.0) compared_subgroups: tuple[str, ...] subgroup_results: tuple[SyntheticSubgroupResult, ...] threshold: float = Field(default=0.2, ge=0.0, le=1.0) notice_ko: str = Field(min_length=10, max_length=500) @model_validator(mode="after") def keep_underpowered_drift_scoreless(self) -> "SubgroupDriftReport": if self.status == "insufficient_evidence" and self.max_rate_gap is not None: raise ValueError("underpowered subgroup report cannot claim a rate gap") if self.status != "insufficient_evidence" and self.max_rate_gap is None: raise ValueError("classified subgroup report requires an observed gap") return self class CalibrationTransferBenchmarkExpectation(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) calibration_improved: dict[str, bool] = Field(default_factory=dict) transfer_verified: dict[str, bool] = Field(default_factory=dict) drift_status: dict[str, DriftStatus] = Field(default_factory=dict) class CalibrationTransferBenchmarkCase(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) case_id: str = Field(pattern=r"^oas-g5-bench-[0-9]{3}$") title_ko: str = Field(min_length=1, max_length=200) tags: tuple[str, ...] = Field(min_length=1) calibration_blocks: tuple[CalibrationBlockInput, ...] = () transfer_suite: TransferSuiteInput | None = None expected: CalibrationTransferBenchmarkExpectation class CalibrationTransferBenchmarkPack(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) schema_version: Literal["vignette.calibration-transfer-benchmark.v1"] version: Literal["1.0.0"] data_classification: Literal["synthetic_educational"] clinical_claim_allowed: Literal[False] cases: tuple[CalibrationTransferBenchmarkCase, ...] = Field(min_length=4) @model_validator(mode="after") def require_adversarial_coverage(self) -> "CalibrationTransferBenchmarkPack": ids = [item.case_id for item in self.cases] if len(set(ids)) != len(ids): raise ValueError("calibration benchmark case ids must be unique") tags = {tag for item in self.cases for tag in item.tags} required = { "post_reveal_contamination", "memorized_phrase_transfer", "calibration_improvement", "synthetic_subgroup_drift", } if not required.issubset(tags): raise ValueError( "calibration benchmark lacks required adversarial coverage" ) return self __all__ = [ "ActualTransferAssessment", "ActualTransferExecution", "ActualTransferStatus", "CalibrationBias", "CalibrationBlockInput", "CalibrationPair", "CalibrationTransferBenchmarkCase", "CalibrationTransferBenchmarkExpectation", "CalibrationTransferBenchmarkPack", "CompetencyCalibrationAssessment", "ConfidenceInterval", "DriftStatus", "ImprovementStatus", "IndependentPerformanceObservation", "LockedSelfPredictionHistory", "MetacognitivePrescription", "NormalizedEvaluatorLabels", "PerformanceStatus", "RelationshipStyle", "SelfPredictionRevision", "SubgroupDriftReport", "SyntheticSubgroupResult", "TransferAssessment", "TransferSuiteInput", "TransferTrial", "TransferVariation", ]