"""Outcome & Alliance OS 측정 원장의 Python 계약. 점수의 숫자 자체보다 출처, 관점, 도구, 모델 실행, 근거를 먼저 고정한다. 이 모듈은 DB/API/프론트 계약이 맞춰야 하는 의미론적 SSOT다. """ from __future__ import annotations from datetime import datetime, timezone from typing import Any, Literal from uuid import UUID, uuid4 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator SOURCE_KINDS = ( "simulated_state", "model_inferred", "agent_reported", "learner_reported", "human_rated", "observed_runtime", ) SourceKind = Literal[ "simulated_state", "model_inferred", "agent_reported", "learner_reported", "human_rated", "observed_runtime", ] MEASUREMENT_CONSTRUCTS = ( "working_alliance", "session_outcome", "rupture_repair", "counselor_skill", "self_calibration", "transfer", "simulation_progress", ) MeasurementConstruct = Literal[ "working_alliance", "session_outcome", "rupture_repair", "counselor_skill", "self_calibration", "transfer", "simulation_progress", ] MEASUREMENT_PERSPECTIVES = ( "client_agent_report", "learner_self_report", "independent_observer", "supervisor_human", "client_simulation", "runtime_observation", ) MeasurementPerspective = Literal[ "client_agent_report", "learner_self_report", "independent_observer", "supervisor_human", "client_simulation", "runtime_observation", ] MEASUREMENT_STATUSES = ("ready", "degraded", "error", "rejected") MeasurementStatus = Literal["ready", "degraded", "error", "rejected"] INSTRUMENT_KINDS = ( "validated_measure", "training_metric", "simulation_signal", "runtime_metric", ) InstrumentKind = Literal[ "validated_measure", "training_metric", "simulation_signal", "runtime_metric", ] AI_VIEWS = ("client", "counselor", "evaluator", "supervisor", "research") AIView = Literal["client", "counselor", "evaluator", "supervisor", "research"] MODEL_RUN_STATUSES = ("ready", "degraded", "error") ModelRunStatus = Literal["ready", "degraded", "error"] ALLIANCE_DIMENSIONS = ("goal", "task", "bond") AllianceDimension = Literal["goal", "task", "bond"] ALLIANCE_CHECKPOINTS = ("pre", "mid", "post") AllianceCheckpoint = Literal["pre", "mid", "post"] SOURCE_PERSPECTIVE_COMPATIBILITY: dict[str, frozenset[str]] = { "simulated_state": frozenset({"client_simulation"}), "model_inferred": frozenset({"independent_observer"}), "agent_reported": frozenset({"client_agent_report"}), "learner_reported": frozenset({"learner_self_report"}), "human_rated": frozenset({"supervisor_human"}), "observed_runtime": frozenset({"runtime_observation"}), } def _utc_now() -> datetime: return datetime.now(timezone.utc) class MeasurementInstrument(BaseModel): """척도·훈련지표·시뮬레이션 신호의 버전 고정 레지스트리.""" model_config = ConfigDict( extra="forbid", frozen=True, populate_by_name=True, protected_namespaces=(), ) instrument_id: str = Field(min_length=1, max_length=120) instrument_version: str = Field(min_length=1, max_length=40) name_ko: str = Field(min_length=1, max_length=200) instrument_kind: InstrumentKind construct_key: MeasurementConstruct = Field(alias="construct") language: str = Field(default="ko-KR", min_length=2, max_length=35) license_id: str | None = Field(default=None, max_length=200) validation_basis: str = Field(min_length=1, max_length=1000) scoring_schema: dict[str, Any] = Field(default_factory=dict) metadata: dict[str, Any] = Field(default_factory=dict) created_at: datetime = Field(default_factory=_utc_now) @field_validator("instrument_id", "instrument_version") @classmethod def strip_identifiers(cls, value: str) -> str: stripped = value.strip() if not stripped: raise ValueError("instrument identifiers must not be blank") return stripped class ModelRun(BaseModel): """측정을 만든 모델 실행의 재현·드리프트 감사 계약.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) model_run_id: UUID = Field(default_factory=uuid4) session_id: UUID | None = None turn_id: UUID | None = None agent_role: Literal["client", "evaluator", "coach", "scenario", "research"] provider: str = Field(min_length=1, max_length=80) model: str = Field(min_length=1, max_length=160) prompt_bundle_id: str = Field(min_length=1, max_length=160) prompt_bundle_version: str = Field(min_length=1, max_length=40) prompt_bundle_hash: str = Field(pattern=r"^[a-f0-9]{64}$") structured_schema_version: str = Field(min_length=1, max_length=80) input_evidence_hash: str = Field(pattern=r"^[a-f0-9]{64}$") status: ModelRunStatus = "ready" error_code: str | None = Field(default=None, max_length=120) metadata: dict[str, Any] = Field(default_factory=dict) created_at: datetime = Field(default_factory=_utc_now) @model_validator(mode="after") def require_error_code_for_failed_run(self) -> "ModelRun": if self.status == "error" and not self.error_code: raise ValueError("error model runs require error_code") if self.status == "ready" and self.error_code: raise ValueError("ready model runs cannot carry error_code") return self class MeasurementEvent(BaseModel): """append-only 측정 이벤트. 정정은 기존 행 변경이 아니라 새 이벤트의 ``supersedes_id``로 표현한다. """ model_config = ConfigDict( extra="forbid", frozen=True, populate_by_name=True, protected_namespaces=(), ) measurement_id: UUID = Field(default_factory=uuid4) session_id: UUID pulse_id: UUID | None = None turn_id: UUID | None = None supersedes_id: UUID | None = None construct_key: MeasurementConstruct = Field(alias="construct") dimension: str = Field(min_length=1, max_length=120) perspective: MeasurementPerspective source_kind: SourceKind instrument_id: str = Field(min_length=1, max_length=120) instrument_version: str = Field(min_length=1, max_length=40) value: float | None = None scale_min: float scale_max: float confidence: float | None = Field(default=None, ge=0.0, le=1.0) status: MeasurementStatus = "ready" error_code: str | None = Field(default=None, max_length=120) evidence_turn_ids: tuple[UUID, ...] = () model_run_id: UUID | None = None visible_to: tuple[AIView, ...] = ("evaluator",) metadata: dict[str, Any] = Field(default_factory=dict) created_at: datetime = Field(default_factory=_utc_now) @field_validator("dimension", "instrument_id", "instrument_version") @classmethod def strip_required_text(cls, value: str) -> str: stripped = value.strip() if not stripped: raise ValueError("measurement identifiers must not be blank") return stripped @model_validator(mode="after") def enforce_measurement_truth(self) -> "MeasurementEvent": allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind] if self.perspective not in allowed: raise ValueError( f"source_kind={self.source_kind} is incompatible with perspective={self.perspective}" ) if self.scale_max <= self.scale_min: raise ValueError("scale_max must be greater than scale_min") if self.value is not None and not self.scale_min <= self.value <= self.scale_max: raise ValueError("measurement value must stay inside its declared scale") if self.status == "ready" and self.value is None: raise ValueError("ready measurements require a value") if self.status in {"error", "rejected"}: if self.value is not None: raise ValueError("error/rejected measurements cannot carry a score") if not self.error_code: raise ValueError("error/rejected measurements require error_code") if self.status == "ready" and self.error_code: raise ValueError("ready measurements cannot carry error_code") if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None: raise ValueError("model/agent measurements require model_run_id provenance") if self.supersedes_id == self.measurement_id: raise ValueError("a measurement cannot supersede itself") if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids): raise ValueError("evidence_turn_ids must be unique") if not self.visible_to: raise ValueError("visible_to must contain at least one audience") if len(set(self.visible_to)) != len(self.visible_to): raise ValueError("visible_to must not contain duplicates") return self class AllianceScores(BaseModel): """goal/task/bond를 서로 가리지 않는 독립 0..1 점수.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) goal: float = Field(ge=0.0, le=1.0) task: float = Field(ge=0.0, le=1.0) bond: float = Field(ge=0.0, le=1.0) class AllianceDimensionAssessment(BaseModel): """한 관점의 한 동맹 축 평가와 transcript 근거.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) score: float = Field(ge=0.0, le=1.0) confidence: float = Field(ge=0.0, le=1.0) evidence_turn_indices: tuple[int, ...] = Field(min_length=1, max_length=12) rationale: str = Field(min_length=1, max_length=1200) @field_validator("evidence_turn_indices") @classmethod def keep_evidence_indices_unique(cls, value: tuple[int, ...]) -> tuple[int, ...]: if min(value) < 0: raise ValueError("alliance evidence indices must be non-negative") if len(set(value)) != len(value): raise ValueError("alliance evidence indices must be unique") return value class AllianceAgentAssessment(BaseModel): """client-agent 또는 observer가 독립 실행으로 만든 3축 평가.""" model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) goal: AllianceDimensionAssessment task: AllianceDimensionAssessment bond: AllianceDimensionAssessment def by_dimension(self) -> dict[AllianceDimension, AllianceDimensionAssessment]: return {"goal": self.goal, "task": self.task, "bond": self.bond} class BenchmarkTurn(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) speaker: Literal["counselor", "client"] text: str = Field(min_length=1) class BenchmarkExpectation(BaseModel): model_config = ConfigDict( extra="forbid", frozen=True, populate_by_name=True, protected_namespaces=(), ) construct_key: MeasurementConstruct = Field(alias="construct") dimension: str = Field(min_length=1) perspective: MeasurementPerspective source_kind: SourceKind direction: Literal["low", "mid", "high", "drop", "rise", "detected", "not_detected"] evidence_turn_indices: tuple[int, ...] = Field(min_length=1) @model_validator(mode="after") def enforce_expectation_truth(self) -> "BenchmarkExpectation": if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]: raise ValueError("benchmark expectation mixes source and perspective layers") if min(self.evidence_turn_indices) < 0: raise ValueError("benchmark turn indices must be zero-based and non-negative") return self class BenchmarkCase(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) case_id: str = Field(pattern=r"^oas-g0-[0-9]{3}$") version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$") scene_type: Literal[ "goal_mismatch", "task_mismatch", "empathic_miss", "withdrawal", "confrontation", "successful_repair", "failed_repair", "warm_but_directionless", ] title_ko: str = Field(min_length=1) description_ko: str = Field(min_length=1) turns: tuple[BenchmarkTurn, ...] = Field(min_length=2) expected: tuple[BenchmarkExpectation, ...] = Field(min_length=1) forbidden_claims: tuple[str, ...] = Field(min_length=1) tags: tuple[str, ...] = () @model_validator(mode="after") def keep_evidence_inside_scene(self) -> "BenchmarkCase": for expectation in self.expected: if max(expectation.evidence_turn_indices) >= len(self.turns): raise ValueError("benchmark evidence points outside the scene") return self __all__ = [ "AI_VIEWS", "ALLIANCE_CHECKPOINTS", "ALLIANCE_DIMENSIONS", "AllianceAgentAssessment", "AllianceDimensionAssessment", "AllianceScores", "BenchmarkCase", "BenchmarkExpectation", "BenchmarkTurn", "INSTRUMENT_KINDS", "MEASUREMENT_CONSTRUCTS", "MEASUREMENT_PERSPECTIVES", "MEASUREMENT_STATUSES", "MODEL_RUN_STATUSES", "MeasurementEvent", "MeasurementInstrument", "ModelRun", "SOURCE_KINDS", "SOURCE_PERSPECTIVE_COMPATIBILITY", ]