G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
564
apps/api/app/contracts/calibration_transfer.py
Normal file
564
apps/api/app/contracts/calibration_transfer.py
Normal file
|
|
@ -0,0 +1,564 @@
|
|||
"""G5 Calibration Mirror & Transfer의 버전 고정 순수 도메인 계약.
|
||||
|
||||
외부 평가 공개 전에 잠근 자기예측과 독립 관찰을 역량별로만 대조한다.
|
||||
전이는 익숙한 문장 재현과 분리하고, 합성 subgroup 차이는 표본 근거가
|
||||
충분할 때만 drift 신호로 남긴다. 단일 총점이나 임상 주장은 허용하지 않는다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
CalibrationBias = Literal[
|
||||
"overconfident", "underconfident", "aligned", "insufficient_evidence"
|
||||
]
|
||||
ImprovementStatus = Literal["improved", "not_improved", "insufficient_evidence"]
|
||||
PerformanceStatus = Literal["passed", "failed", "insufficient_evidence"]
|
||||
RelationshipStyle = Literal[
|
||||
"collaborative", "withdrawn", "confrontational", "ambivalent"
|
||||
]
|
||||
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
||||
ActualTransferStatus = Literal["verified", "not_verified", "insufficient_evidence"]
|
||||
|
||||
|
||||
class SelfPredictionRevision(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
prediction_id: str = Field(pattern=r"^oas-g5-prediction-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
revision_no: int = Field(ge=1)
|
||||
supersedes_prediction_id: str | None = None
|
||||
recorded_sequence: int = Field(ge=1)
|
||||
revision_reason: str = Field(min_length=1, max_length=300)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_revision_link(self) -> "SelfPredictionRevision":
|
||||
if self.revision_no == 1 and self.supersedes_prediction_id is not None:
|
||||
raise ValueError("first self-prediction revision cannot supersede another")
|
||||
if self.revision_no > 1 and self.supersedes_prediction_id is None:
|
||||
raise ValueError(
|
||||
"later self-prediction revision must supersede its predecessor"
|
||||
)
|
||||
if self.supersedes_prediction_id == self.prediction_id:
|
||||
raise ValueError("self-prediction revision cannot supersede itself")
|
||||
return self
|
||||
|
||||
|
||||
class LockedSelfPredictionHistory(BaseModel):
|
||||
"""외부 관찰 공개 시퀀스 이전에만 수정 가능한 append-only 자기예측."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
lock_id: str = Field(pattern=r"^oas-g5-lock-[a-z0-9-]+$")
|
||||
revisions: tuple[SelfPredictionRevision, ...] = Field(min_length=1)
|
||||
locked_sequence: int = Field(ge=1)
|
||||
external_reveal_sequence: int | None = Field(default=None, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_append_only_pre_reveal_history(self) -> "LockedSelfPredictionHistory":
|
||||
first = self.revisions[0]
|
||||
expected_revisions = list(range(1, len(self.revisions) + 1))
|
||||
if [item.revision_no for item in self.revisions] != expected_revisions:
|
||||
raise ValueError("self-prediction revisions must be contiguous and ordered")
|
||||
sequences = [item.recorded_sequence for item in self.revisions]
|
||||
if sequences != sorted(set(sequences)):
|
||||
raise ValueError(
|
||||
"self-prediction revision sequences must be unique and ordered"
|
||||
)
|
||||
for index, item in enumerate(self.revisions):
|
||||
if (
|
||||
item.competency_id != first.competency_id
|
||||
or item.practice_block_id != first.practice_block_id
|
||||
or item.scenario_variant_id != first.scenario_variant_id
|
||||
or item.phrase_family_id != first.phrase_family_id
|
||||
):
|
||||
raise ValueError(
|
||||
"one prediction history must keep one assessment target"
|
||||
)
|
||||
if (
|
||||
index
|
||||
and item.supersedes_prediction_id
|
||||
!= self.revisions[index - 1].prediction_id
|
||||
):
|
||||
raise ValueError(
|
||||
"prediction revision must supersede the immediate predecessor"
|
||||
)
|
||||
if self.locked_sequence < sequences[-1]:
|
||||
raise ValueError("prediction lock cannot precede its latest revision")
|
||||
if self.external_reveal_sequence is not None:
|
||||
if self.external_reveal_sequence <= self.locked_sequence:
|
||||
raise ValueError(
|
||||
"external evaluation must be revealed after prediction lock"
|
||||
)
|
||||
if any(
|
||||
item.recorded_sequence >= self.external_reveal_sequence
|
||||
for item in self.revisions
|
||||
):
|
||||
raise ValueError(
|
||||
"self-prediction cannot be revised after external reveal"
|
||||
)
|
||||
return self
|
||||
|
||||
@property
|
||||
def locked_prediction(self) -> SelfPredictionRevision:
|
||||
return self.revisions[-1]
|
||||
|
||||
|
||||
class IndependentPerformanceObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
observation_id: str = Field(pattern=r"^oas-g5-observation-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
practice_block_id: str = Field(pattern=r"^oas-g5-block-[a-z0-9-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
status: PerformanceStatus
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective
|
||||
model_run_id: UUID | None = None
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
revealed_sequence: int = Field(ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_independent_observation_truth(
|
||||
self,
|
||||
) -> "IndependentPerformanceObservation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError(
|
||||
"performance observation mixes source and perspective layers"
|
||||
)
|
||||
if (
|
||||
self.source_kind in {"model_inferred", "agent_reported"}
|
||||
and self.model_run_id is None
|
||||
):
|
||||
raise ValueError("model/agent observation requires model_run_id")
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_refs:
|
||||
raise ValueError(
|
||||
"insufficient performance evidence must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_refs:
|
||||
raise ValueError("ready performance observation requires evidence")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed performance observation requires counterevidence")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationBlockInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
block_sequence: int = Field(ge=1)
|
||||
prediction_history: LockedSelfPredictionHistory
|
||||
observation: IndependentPerformanceObservation
|
||||
|
||||
@model_validator(mode="after")
|
||||
def align_prediction_and_observation(self) -> "CalibrationBlockInput":
|
||||
prediction = self.prediction_history.locked_prediction
|
||||
observed = self.observation
|
||||
for field in (
|
||||
"competency_id",
|
||||
"practice_block_id",
|
||||
"scenario_variant_id",
|
||||
"phrase_family_id",
|
||||
):
|
||||
if getattr(prediction, field) != getattr(observed, field):
|
||||
raise ValueError(f"calibration block mismatches {field}")
|
||||
if (
|
||||
self.prediction_history.external_reveal_sequence
|
||||
!= observed.revealed_sequence
|
||||
):
|
||||
raise ValueError("prediction history must record the exact external reveal")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationPair(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
practice_block_id: str
|
||||
prediction_id: str
|
||||
observation_id: str
|
||||
predicted_success_probability: float = Field(ge=0.0, le=1.0)
|
||||
observed_success: bool
|
||||
signed_error: float = Field(ge=-1.0, le=1.0)
|
||||
absolute_error: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class ConfidenceInterval(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
method: Literal["normal_95_bounded", "wilson_95"]
|
||||
lower: float = Field(ge=0.0, le=1.0)
|
||||
upper: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_order(self) -> "ConfidenceInterval":
|
||||
if self.lower > self.upper:
|
||||
raise ValueError("confidence interval lower bound exceeds upper bound")
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyCalibrationAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
pair_count: int = Field(ge=0)
|
||||
mean_absolute_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
mean_signed_error: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
error_interval: ConfidenceInterval | None = None
|
||||
bias: CalibrationBias
|
||||
baseline_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
recent_error: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
improvement: ImprovementStatus
|
||||
pairs: tuple[CalibrationPair, ...]
|
||||
excluded_block_ids: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_insufficient_calibration_scoreless(
|
||||
self,
|
||||
) -> "CompetencyCalibrationAssessment":
|
||||
if self.pair_count != len(self.pairs):
|
||||
raise ValueError("calibration pair_count must match pair evidence")
|
||||
if self.bias == "insufficient_evidence":
|
||||
if any(
|
||||
item is not None
|
||||
for item in (
|
||||
self.mean_absolute_error,
|
||||
self.mean_signed_error,
|
||||
self.error_interval,
|
||||
)
|
||||
):
|
||||
raise ValueError("insufficient calibration must remain scoreless")
|
||||
elif self.mean_absolute_error is None or self.mean_signed_error is None:
|
||||
raise ValueError("classified calibration requires error estimates")
|
||||
return self
|
||||
|
||||
|
||||
class MetacognitivePrescription(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
bias: CalibrationBias
|
||||
practice_mode: Literal[
|
||||
"counterevidence_forecast",
|
||||
"evidence_recall",
|
||||
"uncertainty_range",
|
||||
"collect_more_evidence",
|
||||
]
|
||||
instruction_ko: str = Field(min_length=10, max_length=500)
|
||||
completion_evidence: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class TransferVariation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
context_variant: str = Field(min_length=1, max_length=120)
|
||||
relationship_style: RelationshipStyle
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
expression_variant: str = Field(min_length=1, max_length=120)
|
||||
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
||||
scenario_family_id: str = Field(min_length=1, max_length=180)
|
||||
phrase_family_id: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class TransferTrial(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
trial_id: str = Field(pattern=r"^oas-g5-transfer-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
||||
variation: TransferVariation
|
||||
status: PerformanceStatus
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_transfer_evidence(self) -> "TransferTrial":
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_refs:
|
||||
raise ValueError(
|
||||
"insufficient transfer trial must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_refs:
|
||||
raise ValueError("ready transfer trial requires evidence")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed transfer trial requires counterevidence")
|
||||
return self
|
||||
|
||||
|
||||
class NormalizedEvaluatorLabels(BaseModel):
|
||||
"""원문 없이 실제 회기 판정에 사용한 정규화 evaluator 라벨."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
technique_codes: tuple[str, ...] = ()
|
||||
client_state_codes: tuple[str, ...] = ()
|
||||
appropriateness: tuple[Literal["pos", "neutral", "warn"], ...] = ()
|
||||
intent_deviation_dimensions: tuple[str, ...] = ()
|
||||
evaluator_error_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ActualTransferExecution(BaseModel):
|
||||
"""서버 원장에서 재구성한 한 번의 실제 transfer 연습 실행."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
execution_event_id: UUID
|
||||
original_transfer_trial_record_id: UUID
|
||||
practice_session_id: UUID
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: Literal["unseen_transfer"] = "unseen_transfer"
|
||||
variation: TransferVariation
|
||||
status: PerformanceStatus
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_turn_ids: tuple[UUID, ...] = ()
|
||||
normalized_evaluator_labels: NormalizedEvaluatorLabels
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
model_run_id: UUID
|
||||
source_kind: Literal["model_inferred"] = "model_inferred"
|
||||
perspective: Literal["independent_observer"] = "independent_observer"
|
||||
instrument_id: Literal["unseen-transfer-g5"] = "unseen-transfer-g5"
|
||||
instrument_version: Literal["1.0.0"] = "1.0.0"
|
||||
observer_version: Literal["calibration-actual-transfer-observer-v1"] = (
|
||||
"calibration-actual-transfer-observer-v1"
|
||||
)
|
||||
training_phrase_collision: bool = False
|
||||
created_at: datetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_actual_execution_truth(self) -> "ActualTransferExecution":
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.uncertainty != 1.0 or self.evidence_turn_ids:
|
||||
raise ValueError(
|
||||
"insufficient actual execution must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence_turn_ids:
|
||||
raise ValueError("observed actual execution requires durable turn UUIDs")
|
||||
if self.status == "failed" and not self.counterevidence:
|
||||
raise ValueError("failed actual execution requires counterevidence")
|
||||
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
||||
raise ValueError("actual execution turn evidence must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TransferSuiteInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
suite_id: str = Field(pattern=r"^oas-g5-suite-[a-z0-9-]+$")
|
||||
training_phrase_family_ids: tuple[str, ...] = Field(min_length=1)
|
||||
trials: tuple[TransferTrial, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_trials(self) -> "TransferSuiteInput":
|
||||
ids = [item.trial_id for item in self.trials]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("transfer trial ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TransferAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
trial_count: int = Field(ge=0)
|
||||
observed_trial_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
success_interval: ConfidenceInterval | None = None
|
||||
coverage: dict[str, int]
|
||||
eligible: bool
|
||||
transfer_verified: bool
|
||||
blockers: tuple[str, ...]
|
||||
evidence_refs: tuple[str, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_transfer_gate(self) -> "TransferAssessment":
|
||||
if self.transfer_verified and (
|
||||
not self.eligible
|
||||
or self.success_rate is None
|
||||
or self.success_rate < 0.85
|
||||
or not self.evidence_refs
|
||||
):
|
||||
raise ValueError(
|
||||
"transfer verification requires eligible evidence at target rate"
|
||||
)
|
||||
if not self.observed_trial_count and (
|
||||
self.success_rate is not None or self.success_interval is not None
|
||||
):
|
||||
raise ValueError("unobserved transfer must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class ActualTransferAssessment(BaseModel):
|
||||
"""실제 완료 회기만으로 계산한 역량별 transfer 상태."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
evidence_source: Literal["actual_practice_execution"] = "actual_practice_execution"
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
execution_count: int = Field(ge=0)
|
||||
independent_execution_count: int = Field(ge=0)
|
||||
observed_execution_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
success_interval: ConfidenceInterval | None = None
|
||||
coverage: dict[str, int]
|
||||
phrase_family_collision_count: int = Field(ge=0)
|
||||
eligible: bool
|
||||
actual_transfer_status: ActualTransferStatus
|
||||
blockers: tuple[str, ...]
|
||||
source_execution_event_ids: tuple[UUID, ...]
|
||||
evidence_turn_ids: tuple[UUID, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_actual_transfer_gate(self) -> "ActualTransferAssessment":
|
||||
if self.independent_execution_count > self.execution_count:
|
||||
raise ValueError("independent actual execution count exceeds total")
|
||||
if self.observed_execution_count > self.independent_execution_count:
|
||||
raise ValueError("observed actual execution count exceeds independent count")
|
||||
if self.actual_transfer_status == "verified" and (
|
||||
not self.eligible
|
||||
or self.success_rate is None
|
||||
or self.success_rate < 0.85
|
||||
or not self.evidence_turn_ids
|
||||
):
|
||||
raise ValueError(
|
||||
"actual transfer verification requires eligible durable evidence"
|
||||
)
|
||||
if self.actual_transfer_status == "not_verified" and not self.eligible:
|
||||
raise ValueError("classified actual transfer requires eligible evidence")
|
||||
if self.actual_transfer_status == "insufficient_evidence" and self.eligible:
|
||||
raise ValueError("eligible actual transfer cannot remain insufficient")
|
||||
if not self.observed_execution_count and (
|
||||
self.success_rate is not None or self.success_interval is not None
|
||||
):
|
||||
raise ValueError("unobserved actual transfer must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class SyntheticSubgroupResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
subgroup: str
|
||||
observed_count: int = Field(ge=0)
|
||||
success_rate: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
interval: ConfidenceInterval | None = None
|
||||
|
||||
|
||||
class SubgroupDriftReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str
|
||||
status: DriftStatus
|
||||
max_rate_gap: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
compared_subgroups: tuple[str, ...]
|
||||
subgroup_results: tuple[SyntheticSubgroupResult, ...]
|
||||
threshold: float = Field(default=0.2, ge=0.0, le=1.0)
|
||||
notice_ko: str = Field(min_length=10, max_length=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_underpowered_drift_scoreless(self) -> "SubgroupDriftReport":
|
||||
if self.status == "insufficient_evidence" and self.max_rate_gap is not None:
|
||||
raise ValueError("underpowered subgroup report cannot claim a rate gap")
|
||||
if self.status != "insufficient_evidence" and self.max_rate_gap is None:
|
||||
raise ValueError("classified subgroup report requires an observed gap")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
calibration_improved: dict[str, bool] = Field(default_factory=dict)
|
||||
transfer_verified: dict[str, bool] = Field(default_factory=dict)
|
||||
drift_status: dict[str, DriftStatus] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g5-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
tags: tuple[str, ...] = Field(min_length=1)
|
||||
calibration_blocks: tuple[CalibrationBlockInput, ...] = ()
|
||||
transfer_suite: TransferSuiteInput | None = None
|
||||
expected: CalibrationTransferBenchmarkExpectation
|
||||
|
||||
|
||||
class CalibrationTransferBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.calibration-transfer-benchmark.v1"]
|
||||
version: Literal["1.0.0"]
|
||||
data_classification: Literal["synthetic_educational"]
|
||||
clinical_claim_allowed: Literal[False]
|
||||
cases: tuple[CalibrationTransferBenchmarkCase, ...] = Field(min_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_adversarial_coverage(self) -> "CalibrationTransferBenchmarkPack":
|
||||
ids = [item.case_id for item in self.cases]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("calibration benchmark case ids must be unique")
|
||||
tags = {tag for item in self.cases for tag in item.tags}
|
||||
required = {
|
||||
"post_reveal_contamination",
|
||||
"memorized_phrase_transfer",
|
||||
"calibration_improvement",
|
||||
"synthetic_subgroup_drift",
|
||||
}
|
||||
if not required.issubset(tags):
|
||||
raise ValueError(
|
||||
"calibration benchmark lacks required adversarial coverage"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActualTransferAssessment",
|
||||
"ActualTransferExecution",
|
||||
"ActualTransferStatus",
|
||||
"CalibrationBias",
|
||||
"CalibrationBlockInput",
|
||||
"CalibrationPair",
|
||||
"CalibrationTransferBenchmarkCase",
|
||||
"CalibrationTransferBenchmarkExpectation",
|
||||
"CalibrationTransferBenchmarkPack",
|
||||
"CompetencyCalibrationAssessment",
|
||||
"ConfidenceInterval",
|
||||
"DriftStatus",
|
||||
"ImprovementStatus",
|
||||
"IndependentPerformanceObservation",
|
||||
"LockedSelfPredictionHistory",
|
||||
"MetacognitivePrescription",
|
||||
"NormalizedEvaluatorLabels",
|
||||
"PerformanceStatus",
|
||||
"RelationshipStyle",
|
||||
"SelfPredictionRevision",
|
||||
"SubgroupDriftReport",
|
||||
"SyntheticSubgroupResult",
|
||||
"TransferAssessment",
|
||||
"TransferSuiteInput",
|
||||
"TransferTrial",
|
||||
"TransferVariation",
|
||||
]
|
||||
255
apps/api/app/contracts/continuous_improvement.py
Normal file
255
apps/api/app/contracts/continuous_improvement.py
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
"""G8 자율 콘텐츠 생성·적대 검토·승격·운영 환류 계약."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
ReviewDimension = Literal[
|
||||
"safety",
|
||||
"identity",
|
||||
"answer_leakage",
|
||||
"cultural_bias",
|
||||
"difficulty",
|
||||
"pii",
|
||||
"grounding",
|
||||
]
|
||||
FindingSeverity = Literal["blocker", "high", "moderate", "low"]
|
||||
FindingState = Literal["open", "resolved", "accepted_risk"]
|
||||
ModelChangeDecision = Literal["promote", "rollback", "quarantine"]
|
||||
|
||||
|
||||
class ContentSourceArtifact(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
source_id: str = Field(pattern=r"^oas-g8-source-[a-z0-9-]+$")
|
||||
version: str = Field(min_length=1, max_length=80)
|
||||
content_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
provenance_uri: str = Field(pattern=r"^(repo|db|audit)://[a-zA-Z0-9_./:-]+$")
|
||||
usage_status: Literal["approved", "restricted", "rejected"]
|
||||
citation_label: str = Field(min_length=1, max_length=300)
|
||||
|
||||
|
||||
class GeneratedContentDraft(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
draft_id: str = Field(pattern=r"^oas-g8-draft-[a-z0-9-]+$")
|
||||
content_kind: Literal["case", "rupture", "practice", "benchmark"]
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
generation_model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
prompt_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
payload_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
synthetic_identity_id: str = Field(pattern=r"^synthetic-identity-[a-z0-9-]+$")
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
hidden_answer_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
visible_answer_overlap_tokens: int = Field(ge=0)
|
||||
pii_findings: int = Field(ge=0)
|
||||
unsupported_clinical_claims: int = Field(ge=0)
|
||||
|
||||
|
||||
class RedTeamFinding(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
finding_id: str = Field(pattern=r"^oas-g8-finding-[a-z0-9-]+$")
|
||||
dimension: ReviewDimension
|
||||
severity: FindingSeverity
|
||||
state: FindingState
|
||||
evidence_ref: str = Field(min_length=1, max_length=220)
|
||||
remediation_ref: str | None = Field(default=None, max_length=220)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_resolution_evidence(self) -> "RedTeamFinding":
|
||||
if self.state == "resolved" and not self.remediation_ref:
|
||||
raise ValueError("resolved red-team finding requires remediation evidence")
|
||||
if self.state == "accepted_risk" and self.severity in {"blocker", "high"}:
|
||||
raise ValueError("blocker/high finding cannot be accepted as residual risk")
|
||||
return self
|
||||
|
||||
|
||||
class IndependentRedTeamReview(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
review_id: str = Field(pattern=r"^oas-g8-review-[a-z0-9-]+$")
|
||||
draft_id: str
|
||||
reviewer_agent_id: str = Field(min_length=1, max_length=180)
|
||||
dimensions: tuple[ReviewDimension, ...] = Field(min_length=3)
|
||||
findings: tuple[RedTeamFinding, ...]
|
||||
reviewed_payload_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_coverage(self) -> "IndependentRedTeamReview":
|
||||
if len(set(self.dimensions)) != len(self.dimensions):
|
||||
raise ValueError("red-team review dimensions must be unique")
|
||||
if any(item.dimension not in self.dimensions for item in self.findings):
|
||||
raise ValueError("red-team finding must belong to a reviewed dimension")
|
||||
return self
|
||||
|
||||
|
||||
class ContentBenchmarkQualification(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
benchmark_id: str = Field(pattern=r"^oas-g8-benchmark-[a-z0-9-]+$")
|
||||
draft_id: str
|
||||
variant_count: int = Field(ge=3)
|
||||
variant_pass_rate: float = Field(ge=0.0, le=1.0)
|
||||
answer_leakage_count: int = Field(ge=0)
|
||||
pii_finding_count: int = Field(ge=0)
|
||||
unsupported_claim_count: int = Field(ge=0)
|
||||
safety_failure_count: int = Field(ge=0)
|
||||
reward_hacking_count: int = Field(ge=0)
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
@property
|
||||
def qualified(self) -> bool:
|
||||
return (
|
||||
self.variant_pass_rate >= 0.85
|
||||
and self.answer_leakage_count == 0
|
||||
and self.pii_finding_count == 0
|
||||
and self.unsupported_claim_count == 0
|
||||
and self.safety_failure_count == 0
|
||||
and self.reward_hacking_count == 0
|
||||
)
|
||||
|
||||
|
||||
class ApprovedCatalogEntry(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
catalog_entry_id: str = Field(pattern=r"^oas-g8-catalog-[a-z0-9-]+$")
|
||||
draft_id: str
|
||||
payload_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
review_ids: tuple[str, ...] = Field(min_length=2)
|
||||
benchmark_id: str
|
||||
status: Literal["approved"] = "approved"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
|
||||
|
||||
class ModelCalibrationSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
snapshot_id: str = Field(pattern=r"^oas-g8-model-snapshot-[a-z0-9-]+$")
|
||||
model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
benchmark_version: str = Field(min_length=1, max_length=80)
|
||||
task_accuracy: float = Field(ge=0.0, le=1.0)
|
||||
critical_miss_count: int = Field(ge=0)
|
||||
leakage_count: int = Field(ge=0)
|
||||
pii_count: int = Field(ge=0)
|
||||
calibration_error: float = Field(ge=0.0, le=1.0)
|
||||
subgroup_max_gap: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class ModelChangeGateResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
baseline_snapshot_id: str
|
||||
candidate_snapshot_id: str
|
||||
decision: ModelChangeDecision
|
||||
reasons: tuple[str, ...] = Field(min_length=1)
|
||||
rollback_target_snapshot_id: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_rollback_target(self) -> "ModelChangeGateResult":
|
||||
if self.decision == "rollback" and not self.rollback_target_snapshot_id:
|
||||
raise ValueError("rollback decision requires a target snapshot")
|
||||
if self.decision != "rollback" and self.rollback_target_snapshot_id:
|
||||
raise ValueError("non-rollback decision cannot carry rollback target")
|
||||
return self
|
||||
|
||||
|
||||
class OperationalIncident(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
incident_id: str = Field(pattern=r"^oas-g8-incident-[a-z0-9-]+$")
|
||||
error_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
affected_contract: str = Field(min_length=1, max_length=180)
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
pii_included: Literal[False] = False
|
||||
|
||||
|
||||
class RegressionBacklogNode(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
node_id: str = Field(pattern=r"^oas-g8-node-[a-z0-9-]+$")
|
||||
node_type: Literal["reproduction_test", "implementation", "e2e", "runtime_proof"]
|
||||
depends_on: tuple[str, ...]
|
||||
evidence_ref: str | None = Field(default=None, max_length=220)
|
||||
status: Literal["pending", "passed", "failed"]
|
||||
|
||||
|
||||
class IncidentRegressionDag(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
incident_id: str
|
||||
nodes: tuple[RegressionBacklogNode, ...] = Field(min_length=4, max_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_closed_loop(self) -> "IncidentRegressionDag":
|
||||
ids = [item.node_id for item in self.nodes]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("incident DAG node ids must be unique")
|
||||
by_type = {item.node_type: item for item in self.nodes}
|
||||
if set(by_type) != {
|
||||
"reproduction_test",
|
||||
"implementation",
|
||||
"e2e",
|
||||
"runtime_proof",
|
||||
}:
|
||||
raise ValueError(
|
||||
"incident DAG requires reproduction, implementation, E2E, runtime"
|
||||
)
|
||||
known: set[str] = set()
|
||||
for item in self.nodes:
|
||||
if any(parent not in known for parent in item.depends_on):
|
||||
raise ValueError("incident DAG dependencies must point backward")
|
||||
known.add(item.node_id)
|
||||
return self
|
||||
|
||||
|
||||
class AgenticReleaseManifest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
release_id: str = Field(pattern=r"^oas-g8-release-[a-z0-9-]+$")
|
||||
red_green_passed: bool
|
||||
contract_passed: bool
|
||||
e2e_passed: bool
|
||||
runtime_proof_passed: bool
|
||||
public_proof_passed: bool
|
||||
ssot_synced: bool
|
||||
evidence_refs: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def releasable(self) -> bool:
|
||||
return all(
|
||||
(
|
||||
self.red_green_passed,
|
||||
self.contract_passed,
|
||||
self.e2e_passed,
|
||||
self.runtime_proof_passed,
|
||||
self.public_proof_passed,
|
||||
self.ssot_synced,
|
||||
)
|
||||
) and bool(self.evidence_refs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AgenticReleaseManifest",
|
||||
"ApprovedCatalogEntry",
|
||||
"ContentBenchmarkQualification",
|
||||
"ContentSourceArtifact",
|
||||
"FindingSeverity",
|
||||
"FindingState",
|
||||
"GeneratedContentDraft",
|
||||
"IncidentRegressionDag",
|
||||
"IndependentRedTeamReview",
|
||||
"ModelCalibrationSnapshot",
|
||||
"ModelChangeDecision",
|
||||
"ModelChangeGateResult",
|
||||
"OperationalIncident",
|
||||
"RedTeamFinding",
|
||||
"RegressionBacklogNode",
|
||||
"ReviewDimension",
|
||||
]
|
||||
639
apps/api/app/contracts/deliberate_practice.py
Normal file
639
apps/api/app/contracts/deliberate_practice.py
Normal file
|
|
@ -0,0 +1,639 @@
|
|||
"""G4 Deliberate Practice Engine의 버전 고정 순수 도메인 계약.
|
||||
|
||||
코칭 장면을 하나의 관찰 가능한 행동으로 분해한 실행형 처방, 시도 근거,
|
||||
역량 그래프와 전이 게이트를 정의한다. 단일 총점이나 보상 점수로 숙련을
|
||||
승격하지 않으며, 익숙한 장면의 성공과 미지 사례 전이를 물리적으로 구분한다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
PRACTICE_MODES = (
|
||||
"replay",
|
||||
"branch",
|
||||
"constrained_response",
|
||||
"voice_retry",
|
||||
"difficulty_ladder",
|
||||
)
|
||||
PracticeMode = Literal[
|
||||
"replay",
|
||||
"branch",
|
||||
"constrained_response",
|
||||
"voice_retry",
|
||||
"difficulty_ladder",
|
||||
]
|
||||
|
||||
COMPETENCY_BANDS = (
|
||||
"unassessed",
|
||||
"fragile",
|
||||
"developing",
|
||||
"consistent_local",
|
||||
"transfer_verified",
|
||||
)
|
||||
CompetencyBand = Literal[
|
||||
"unassessed",
|
||||
"fragile",
|
||||
"developing",
|
||||
"consistent_local",
|
||||
"transfer_verified",
|
||||
]
|
||||
|
||||
CriterionStatus = Literal["observed", "not_observed", "error"]
|
||||
AttemptOutcome = Literal["passed", "needs_retry", "insufficient_evidence"]
|
||||
PracticeProgress = Literal["practicing", "transfer_pending", "mastered"]
|
||||
ScenarioNovelty = Literal["familiar", "unseen_transfer"]
|
||||
ClientPracticeResponse = Literal[
|
||||
"rejecting",
|
||||
"withdrawn",
|
||||
"compliance_only",
|
||||
"mixed",
|
||||
"engaged",
|
||||
"explicit_alignment",
|
||||
]
|
||||
EvidenceKind = Literal[
|
||||
"scene_context",
|
||||
"learner_behavior",
|
||||
"client_response",
|
||||
"evaluator_decision",
|
||||
"voice_feature",
|
||||
]
|
||||
|
||||
|
||||
class PracticeEvidenceRef(BaseModel):
|
||||
"""원문을 복제하지 않는 장면·행동·반응 원장 포인터."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
ref_id: str = Field(min_length=1, max_length=180)
|
||||
scene_id: str = Field(min_length=1, max_length=180)
|
||||
turn_index: int = Field(ge=0)
|
||||
actor: Literal["learner", "client", "observer", "runtime"]
|
||||
kind: EvidenceKind
|
||||
|
||||
|
||||
class ReplayActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["replay"] = "replay"
|
||||
launch_intent: Literal["practice.replay.launch"] = "practice.replay.launch"
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty = "familiar"
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
pause_at_evidence_ref: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class BranchActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["branch"] = "branch"
|
||||
launch_intent: Literal["practice.branch.launch"] = "practice.branch.launch"
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
branch_options: tuple[str, ...] = Field(min_length=2, max_length=5)
|
||||
client_responses_hidden: Literal[True] = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_distinct_branches(self) -> "BranchActivity":
|
||||
if len(set(self.branch_options)) != len(self.branch_options):
|
||||
raise ValueError("branch options must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class ConstrainedResponseActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["constrained_response"] = "constrained_response"
|
||||
launch_intent: Literal["practice.constrained-response.launch"] = (
|
||||
"practice.constrained-response.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
max_words: int = Field(ge=5, le=80)
|
||||
required_moves: tuple[str, ...] = Field(min_length=1, max_length=4)
|
||||
|
||||
|
||||
class VoiceRetryActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["voice_retry"] = "voice_retry"
|
||||
launch_intent: Literal["practice.voice-retry.launch"] = (
|
||||
"practice.voice-retry.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
max_seconds: int = Field(ge=5, le=120)
|
||||
acoustic_focus: tuple[str, ...] = Field(min_length=1, max_length=4)
|
||||
|
||||
|
||||
class DifficultyStep(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
level: int = Field(ge=1, le=5)
|
||||
variation: str = Field(min_length=1, max_length=240)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
|
||||
|
||||
class DifficultyLadderActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["difficulty_ladder"] = "difficulty_ladder"
|
||||
launch_intent: Literal["practice.difficulty-ladder.launch"] = (
|
||||
"practice.difficulty-ladder.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
steps: tuple[DifficultyStep, ...] = Field(min_length=2, max_length=5)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_ladder_with_transfer(self) -> "DifficultyLadderActivity":
|
||||
levels = [item.level for item in self.steps]
|
||||
if levels != sorted(set(levels)):
|
||||
raise ValueError("difficulty ladder levels must be unique and ascending")
|
||||
if not any(item.scenario_novelty == "unseen_transfer" for item in self.steps):
|
||||
raise ValueError("difficulty ladder must end in an unseen transfer step")
|
||||
return self
|
||||
|
||||
|
||||
PracticeActivity = Annotated[
|
||||
ReplayActivity
|
||||
| BranchActivity
|
||||
| ConstrainedResponseActivity
|
||||
| VoiceRetryActivity
|
||||
| DifficultyLadderActivity,
|
||||
Field(discriminator="mode"),
|
||||
]
|
||||
|
||||
|
||||
class PracticeTargetSpec(BaseModel):
|
||||
"""한 역량의 한 관찰 행동만 소유하는 원자적 연습 명세."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
||||
observable_behavior: str = Field(min_length=10, max_length=500)
|
||||
activity: PracticeActivity
|
||||
|
||||
|
||||
class CoachingCard(BaseModel):
|
||||
"""실행 가능한 재연습이 없는 코칭 카드를 구조적으로 거부한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
card_id: str = Field(pattern=r"^oas-g4-card-[a-z0-9-]+$")
|
||||
scene_id: str = Field(min_length=1, max_length=180)
|
||||
coach_claim: str = Field(min_length=10, max_length=800)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
targets: tuple[PracticeTargetSpec, ...] = Field(min_length=1, max_length=3)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_atomic_actionable_targets(self) -> "CoachingCard":
|
||||
if any(item.scene_id != self.scene_id for item in self.evidence_refs):
|
||||
raise ValueError("coaching card evidence must belong to its scene")
|
||||
prescription_ids = [item.prescription_id for item in self.targets]
|
||||
if len(set(prescription_ids)) != len(prescription_ids):
|
||||
raise ValueError("coaching card prescription ids must be unique")
|
||||
target_keys = [(item.competency_id, item.criterion_id) for item in self.targets]
|
||||
if len(set(target_keys)) != len(target_keys):
|
||||
raise ValueError("coaching card targets must be atomic and unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticePrescription(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.practice-prescription.v1"] = (
|
||||
"vignette.practice-prescription.v1"
|
||||
)
|
||||
event_name: Literal["practice.prescribed"] = "practice.prescribed"
|
||||
prescription_id: str
|
||||
coaching_card_id: str
|
||||
scene_id: str
|
||||
competency_id: str
|
||||
criterion_id: str
|
||||
observable_behavior: str
|
||||
activity: PracticeActivity
|
||||
can_launch: Literal[True] = True
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class CriterionObservation(BaseModel):
|
||||
"""시도의 관찰 결과. 오류와 미관찰을 성공값으로 보간하지 않는다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
||||
status: CriterionStatus
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective
|
||||
model_run_id: UUID | None = None
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_observation_truth(self) -> "CriterionObservation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError("practice observation mixes source and perspective layers")
|
||||
if (
|
||||
self.source_kind in {"model_inferred", "agent_reported"}
|
||||
and self.model_run_id is None
|
||||
):
|
||||
raise ValueError("model/agent practice observation requires model_run_id")
|
||||
if self.status == "observed":
|
||||
if not self.evidence_refs:
|
||||
raise ValueError("observed practice criterion requires evidence")
|
||||
if self.error_code:
|
||||
raise ValueError("observed practice criterion cannot carry error_code")
|
||||
elif self.status == "not_observed":
|
||||
if not self.counterevidence:
|
||||
raise ValueError(
|
||||
"not-observed practice criterion requires counterevidence"
|
||||
)
|
||||
if self.error_code:
|
||||
raise ValueError(
|
||||
"not-observed practice criterion cannot carry error_code"
|
||||
)
|
||||
else:
|
||||
if not self.error_code or self.uncertainty != 1.0:
|
||||
raise ValueError(
|
||||
"error practice criterion requires error_code and maximum uncertainty"
|
||||
)
|
||||
if len({(item.ref_id, item.kind) for item in self.evidence_refs}) != len(
|
||||
self.evidence_refs
|
||||
):
|
||||
raise ValueError("practice criterion evidence refs must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeAttemptObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str = Field(pattern=r"^oas-g4-attempt-[a-z0-9-]+$")
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
sequence_no: int = Field(ge=1)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
criterion: CriterionObservation
|
||||
client_response: ClientPracticeResponse | None = None
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
utterance_template_id: str | None = Field(default=None, max_length=180)
|
||||
learner_claimed_success: bool = False
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_behavior_and_impact_evidence(self) -> "PracticeAttemptObservation":
|
||||
refs = (*self.evidence_refs, *self.criterion.evidence_refs)
|
||||
if len({(item.ref_id, item.kind) for item in refs}) != len(refs):
|
||||
raise ValueError("practice attempt evidence refs must be unique")
|
||||
if self.criterion.status == "error":
|
||||
if (
|
||||
not self.error_code
|
||||
or self.client_response is not None
|
||||
or self.uncertainty != 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"error practice attempt must remain impact-free with maximum uncertainty"
|
||||
)
|
||||
return self
|
||||
if self.error_code:
|
||||
raise ValueError("ready practice attempt cannot carry error_code")
|
||||
kinds = {item.kind for item in refs}
|
||||
if "learner_behavior" not in kinds or "client_response" not in kinds:
|
||||
raise ValueError(
|
||||
"ready practice attempt requires learner behavior and client response evidence"
|
||||
)
|
||||
if self.client_response is None:
|
||||
raise ValueError("ready practice attempt requires observed client response")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeEpisodeInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
episode_id: str = Field(pattern=r"^oas-g4-episode-[a-z0-9-]+$")
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
attempts: tuple[PracticeAttemptObservation, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_attempt_history(self) -> "PracticeEpisodeInput":
|
||||
if any(item.prescription_id != self.prescription_id for item in self.attempts):
|
||||
raise ValueError(
|
||||
"practice episode attempts must reference one prescription"
|
||||
)
|
||||
sequences = [item.sequence_no for item in self.attempts]
|
||||
if sequences != list(range(1, len(sequences) + 1)):
|
||||
raise ValueError("practice attempts must have contiguous sequence numbers")
|
||||
ids = [item.attempt_id for item in self.attempts]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("practice attempt ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeAttemptAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str
|
||||
outcome: AttemptOutcome
|
||||
criterion_status: CriterionStatus
|
||||
client_response: ClientPracticeResponse | None
|
||||
scenario_novelty: ScenarioNovelty
|
||||
scenario_variant_id: str
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
utterance_template_id: str | None
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
|
||||
class BeforeAfterComparison(BaseModel):
|
||||
"""총점 차이가 아니라 동일 기준의 전후 관찰과 근거를 보존한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
criterion_id: str
|
||||
before_attempt_id: str
|
||||
after_attempt_id: str
|
||||
change: Literal["improved", "unchanged", "regressed", "inconclusive"]
|
||||
before_status: CriterionStatus
|
||||
after_status: CriterionStatus
|
||||
before_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
after_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
|
||||
class PracticeEpisodeAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.practice-episode-assessment.v1"] = (
|
||||
"vignette.practice-episode-assessment.v1"
|
||||
)
|
||||
event_names: tuple[
|
||||
Literal["practice.attempted", "practice.mastered", "transfer.verified"], ...
|
||||
]
|
||||
episode_id: str
|
||||
prescription_id: str
|
||||
competency_id: str
|
||||
attempts: tuple[PracticeAttemptAssessment, ...]
|
||||
comparison: BeforeAfterComparison
|
||||
prior_familiar_demonstrations: int = Field(default=0, ge=0)
|
||||
progress: PracticeProgress
|
||||
mastery_allowed: bool
|
||||
mastery_blockers: tuple[str, ...]
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_transfer_before_mastery(self) -> "PracticeEpisodeAssessment":
|
||||
if self.progress == "mastered":
|
||||
if not self.mastery_allowed or "transfer.verified" not in self.event_names:
|
||||
raise ValueError("mastery requires an explicit transfer.verified event")
|
||||
has_familiar_basis = self.prior_familiar_demonstrations > 0 or any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "familiar"
|
||||
for item in self.attempts
|
||||
)
|
||||
if not has_familiar_basis:
|
||||
raise ValueError(
|
||||
"mastery requires current or prior familiar demonstration evidence"
|
||||
)
|
||||
if not any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
||||
for item in self.attempts
|
||||
):
|
||||
raise ValueError("mastery requires passed unseen transfer evidence")
|
||||
elif self.mastery_allowed:
|
||||
raise ValueError("non-mastered practice cannot allow mastery")
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
label_ko: str = Field(min_length=1, max_length=120)
|
||||
description: str = Field(min_length=10, max_length=500)
|
||||
prerequisite_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class CompetencyState(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
band: CompetencyBand
|
||||
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
attempt_count: int = Field(ge=0)
|
||||
familiar_demonstrations: int = Field(ge=0)
|
||||
unseen_transfer_demonstrations: int = Field(ge=0)
|
||||
highest_familiar_difficulty: int = Field(ge=0, le=5)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def prevent_unverified_mastery(self) -> "CompetencyState":
|
||||
if (
|
||||
self.familiar_demonstrations + self.unseen_transfer_demonstrations
|
||||
> self.attempt_count
|
||||
):
|
||||
raise ValueError("competency demonstrations cannot exceed attempt count")
|
||||
if self.band == "unassessed" and self.attempt_count:
|
||||
raise ValueError("attempted competency cannot remain unassessed")
|
||||
if self.band == "consistent_local" and self.familiar_demonstrations < 1:
|
||||
raise ValueError(
|
||||
"consistent_local requires familiar demonstration evidence"
|
||||
)
|
||||
if self.band == "transfer_verified":
|
||||
if self.unseen_transfer_demonstrations < 1 or not self.evidence_refs:
|
||||
raise ValueError("transfer_verified requires unseen transfer evidence")
|
||||
elif self.unseen_transfer_demonstrations:
|
||||
raise ValueError(
|
||||
"unseen transfer demonstration must promote transfer_verified"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyGraph(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.competency-graph.v1"] = (
|
||||
"vignette.competency-graph.v1"
|
||||
)
|
||||
definitions: tuple[CompetencyDefinition, ...] = Field(min_length=1)
|
||||
states: tuple[CompetencyState, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_acyclic_graph(self) -> "CompetencyGraph":
|
||||
definitions = {item.competency_id: item for item in self.definitions}
|
||||
states = {item.competency_id: item for item in self.states}
|
||||
if len(definitions) != len(self.definitions) or len(states) != len(self.states):
|
||||
raise ValueError("competency graph ids must be unique")
|
||||
if definitions.keys() != states.keys():
|
||||
raise ValueError(
|
||||
"competency graph requires exactly one state per definition"
|
||||
)
|
||||
if any(
|
||||
prerequisite not in definitions
|
||||
for item in self.definitions
|
||||
for prerequisite in item.prerequisite_ids
|
||||
):
|
||||
raise ValueError("competency prerequisite must exist in graph")
|
||||
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def visit(competency_id: str) -> None:
|
||||
if competency_id in visiting:
|
||||
raise ValueError("competency graph prerequisites must be acyclic")
|
||||
if competency_id in visited:
|
||||
return
|
||||
visiting.add(competency_id)
|
||||
for prerequisite in definitions[competency_id].prerequisite_ids:
|
||||
visit(prerequisite)
|
||||
visiting.remove(competency_id)
|
||||
visited.add(competency_id)
|
||||
|
||||
for competency_id in definitions:
|
||||
visit(competency_id)
|
||||
return self
|
||||
|
||||
|
||||
class CurriculumDecision(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.curriculum-decision.v1"] = (
|
||||
"vignette.curriculum-decision.v1"
|
||||
)
|
||||
selected_prescription_id: str
|
||||
competency_id: str
|
||||
competency_band: CompetencyBand
|
||||
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
||||
mode: PracticeMode
|
||||
selection_basis: tuple[str, ...] = Field(min_length=2)
|
||||
deferred_prescription_ids: tuple[str, ...]
|
||||
blocked_prescription_reasons: tuple[str, ...]
|
||||
|
||||
|
||||
class PracticeBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
episode_progress: tuple[PracticeProgress, ...]
|
||||
final_competency_id: str
|
||||
final_band: CompetencyBand
|
||||
selected_prescription_id: str
|
||||
|
||||
|
||||
class PracticeBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g4-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
coaching_cards: tuple[CoachingCard, ...] = Field(min_length=1)
|
||||
graph: CompetencyGraph
|
||||
episodes: tuple[PracticeEpisodeInput, ...] = ()
|
||||
expected: PracticeBenchmarkExpectation
|
||||
tags: tuple[str, ...] = ()
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class PracticeBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.deliberate-practice-benchmark.v1"] = (
|
||||
"vignette.deliberate-practice-benchmark.v1"
|
||||
)
|
||||
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
cases: tuple[PracticeBenchmarkCase, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_adversarial_and_mode_coverage(self) -> "PracticeBenchmarkPack":
|
||||
case_ids = [item.case_id for item in self.cases]
|
||||
if len(set(case_ids)) != len(case_ids):
|
||||
raise ValueError("practice benchmark case ids must be unique")
|
||||
modes = {
|
||||
target.activity.mode
|
||||
for case in self.cases
|
||||
for card in case.coaching_cards
|
||||
for target in card.targets
|
||||
}
|
||||
if modes != set(PRACTICE_MODES):
|
||||
raise ValueError("practice benchmark must cover every practice mode")
|
||||
required_tags = {
|
||||
"reward_hacking",
|
||||
"easy_repeat_hacking",
|
||||
"memorized_phrase_hacking",
|
||||
"unseen_transfer_gate",
|
||||
}
|
||||
tags = {tag for case in self.cases for tag in case.tags}
|
||||
if not required_tags.issubset(tags):
|
||||
raise ValueError("practice benchmark lacks required adversarial coverage")
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttemptOutcome",
|
||||
"BeforeAfterComparison",
|
||||
"BranchActivity",
|
||||
"COMPETENCY_BANDS",
|
||||
"ClientPracticeResponse",
|
||||
"CoachingCard",
|
||||
"CompetencyBand",
|
||||
"CompetencyDefinition",
|
||||
"CompetencyGraph",
|
||||
"CompetencyState",
|
||||
"ConstrainedResponseActivity",
|
||||
"CriterionObservation",
|
||||
"CriterionStatus",
|
||||
"CurriculumDecision",
|
||||
"DifficultyLadderActivity",
|
||||
"DifficultyStep",
|
||||
"EvidenceKind",
|
||||
"PRACTICE_MODES",
|
||||
"PracticeActivity",
|
||||
"PracticeAttemptAssessment",
|
||||
"PracticeAttemptObservation",
|
||||
"PracticeBenchmarkCase",
|
||||
"PracticeBenchmarkExpectation",
|
||||
"PracticeBenchmarkPack",
|
||||
"PracticeEpisodeAssessment",
|
||||
"PracticeEpisodeInput",
|
||||
"PracticeEvidenceRef",
|
||||
"PracticeMode",
|
||||
"PracticePrescription",
|
||||
"PracticeProgress",
|
||||
"PracticeTargetSpec",
|
||||
"ReplayActivity",
|
||||
"ScenarioNovelty",
|
||||
"VoiceRetryActivity",
|
||||
]
|
||||
|
|
@ -288,10 +288,45 @@ def _json_object_or_none(value: str) -> dict[str, Any] | None:
|
|||
try:
|
||||
parsed = json.loads(value)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
# CLI 기반 provider는 스키마 지시를 따르면서도 간헐적으로 객체/배열의
|
||||
# 마지막 항목 뒤에 쉼표 하나를 남긴다. 값이나 필드를 추정하지 않고,
|
||||
# 문자열 밖의 닫는 괄호 직전 쉼표만 제거하는 보수적 기계 복구를 허용한다.
|
||||
try:
|
||||
parsed = json.loads(_remove_json_trailing_commas(value))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
|
||||
|
||||
def _remove_json_trailing_commas(value: str) -> str:
|
||||
result: list[str] = []
|
||||
in_string = False
|
||||
escaped = False
|
||||
for index, char in enumerate(value):
|
||||
if in_string:
|
||||
result.append(char)
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
in_string = True
|
||||
result.append(char)
|
||||
continue
|
||||
if char == ",":
|
||||
next_index = index + 1
|
||||
while next_index < len(value) and value[next_index].isspace():
|
||||
next_index += 1
|
||||
if next_index < len(value) and value[next_index] in "}]":
|
||||
continue
|
||||
result.append(char)
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
|
|
|
|||
258
apps/api/app/contracts/g7_external_evidence.py
Normal file
258
apps/api/app/contracts/g7_external_evidence.py
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
"""독립 human-labeled G7 voice-gain 종료 증거 계약.
|
||||
|
||||
이 계약은 repository synthetic benchmark와 의도적으로 분리되어 있다. 원음과
|
||||
축어록을 받지 않고, 동의·분할·모델·라벨링 provenance와 비식별 수치 관측만 받는다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
AllianceAxis = Literal["goal", "task", "bond"]
|
||||
PredictionStatus = Literal["observed", "missing", "error"]
|
||||
ParticipantSplit = Literal["calibration", "held_out"]
|
||||
Sha256 = str
|
||||
|
||||
|
||||
class G7EvidenceProvenance(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
protocol_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
consent_protocol_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
dataset_manifest_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
split_manifest_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
labeling_protocol_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
analysis_plan_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
registered_at: datetime
|
||||
held_out_labels_opened_at: datetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_preregistered_analysis(self) -> "G7EvidenceProvenance":
|
||||
if self.registered_at > self.held_out_labels_opened_at:
|
||||
raise ValueError("analysis protocol must precede held-out label access")
|
||||
return self
|
||||
|
||||
|
||||
class G7ModelProvenance(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
role: Literal["text_only_baseline", "voice_enabled_candidate"]
|
||||
provider: str = Field(min_length=1, max_length=80)
|
||||
model_id: str = Field(min_length=1, max_length=160)
|
||||
model_version: str = Field(min_length=1, max_length=120)
|
||||
artifact_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
configuration_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class G7PowerPlan(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
primary_metric: Literal["paired_one_minus_mae_gain"] = (
|
||||
"paired_one_minus_mae_gain"
|
||||
)
|
||||
clustering_unit: Literal["participant"] = "participant"
|
||||
required_held_out_participants: int = Field(ge=1)
|
||||
required_held_out_sessions: int = Field(ge=1)
|
||||
required_paired_axis_observations: int = Field(ge=3)
|
||||
alpha: float = Field(gt=0.0, le=0.05)
|
||||
target_power: float = Field(ge=0.8, lt=1.0)
|
||||
minimally_detectable_gain: float = Field(gt=0.0, le=1.0)
|
||||
planned_bootstrap_samples: Literal[10000] = 10000
|
||||
|
||||
|
||||
class G7ParticipantAssignment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
participant_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
|
||||
split: ParticipantSplit
|
||||
consent_receipt_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class G7LabelerAttestation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
labeler_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
|
||||
blinded_to_model_condition: Literal[True] = True
|
||||
blinded_to_other_labelers: Literal[True] = True
|
||||
labeled_independently: Literal[True] = True
|
||||
attestation_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class G7HumanAxisLabel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
labeler_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
|
||||
score: float = Field(ge=0.0, le=1.0)
|
||||
category: str | None = Field(
|
||||
default=None,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$",
|
||||
)
|
||||
|
||||
|
||||
class G7ReliabilityClaim(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
method: Literal["ICC(A,1)"] = "ICC(A,1)"
|
||||
labeler_keys: tuple[str, ...] = Field(min_length=2)
|
||||
reported_icc: float = Field(ge=-1.0, le=1.0)
|
||||
reported_categorical_kappa: float | None = Field(
|
||||
default=None,
|
||||
ge=-1.0,
|
||||
le=1.0,
|
||||
)
|
||||
report_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_labelers(self) -> "G7ReliabilityClaim":
|
||||
if len(set(self.labeler_keys)) != len(self.labeler_keys):
|
||||
raise ValueError("reliability labeler keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class G7PairedAxisObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
observation_id: str = Field(pattern=r"^g7-human-observation-[A-Za-z0-9._:-]+$")
|
||||
participant_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
|
||||
session_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
|
||||
axis: AllianceAxis
|
||||
text_only_status: PredictionStatus
|
||||
text_only_score: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
voice_enabled_status: PredictionStatus
|
||||
voice_enabled_score: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
labels: tuple[G7HumanAxisLabel, ...] = Field(min_length=2)
|
||||
raw_audio_included: Literal[False] = False
|
||||
transcript_included: Literal[False] = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_status_consistent_scores(self) -> "G7PairedAxisObservation":
|
||||
pairs = (
|
||||
(self.text_only_status, self.text_only_score, "text-only"),
|
||||
(self.voice_enabled_status, self.voice_enabled_score, "voice-enabled"),
|
||||
)
|
||||
for status, score, label in pairs:
|
||||
if status == "observed" and score is None:
|
||||
raise ValueError(f"{label} observed status requires a score")
|
||||
if status != "observed" and score is not None:
|
||||
raise ValueError(f"{label} missing/error status must remain scoreless")
|
||||
labeler_keys = [item.labeler_key for item in self.labels]
|
||||
if len(set(labeler_keys)) != len(labeler_keys):
|
||||
raise ValueError("observation labeler keys must be unique")
|
||||
has_category = [item.category is not None for item in self.labels]
|
||||
if any(has_category) and not all(has_category):
|
||||
raise ValueError("categorical labels must be complete within an observation")
|
||||
return self
|
||||
|
||||
|
||||
class G7HumanVoiceGainEvidencePack(BaseModel):
|
||||
"""합성 자료로 대체할 수 없는 독립 held-out G7 증거 묶음."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
schema_version: Literal["g7_human_voice_gain_v1"] = "g7_human_voice_gain_v1"
|
||||
evidence_kind: Literal["independent_human_held_out_voice_gain"] = (
|
||||
"independent_human_held_out_voice_gain"
|
||||
)
|
||||
data_classification: Literal["consented_deidentified_research_metrics"] = (
|
||||
"consented_deidentified_research_metrics"
|
||||
)
|
||||
synthetic_pack: Literal[False] = False
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
provenance: G7EvidenceProvenance
|
||||
text_only_model: G7ModelProvenance
|
||||
voice_enabled_model: G7ModelProvenance
|
||||
power_plan: G7PowerPlan
|
||||
participants: tuple[G7ParticipantAssignment, ...] = Field(min_length=2)
|
||||
labeler_attestations: tuple[G7LabelerAttestation, ...] = Field(min_length=2)
|
||||
reliability: G7ReliabilityClaim
|
||||
observations: tuple[G7PairedAxisObservation, ...] = Field(min_length=3)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_independent_holdout_boundaries(self) -> "G7HumanVoiceGainEvidencePack":
|
||||
if self.text_only_model.role != "text_only_baseline":
|
||||
raise ValueError("text-only model provenance role is invalid")
|
||||
if self.voice_enabled_model.role != "voice_enabled_candidate":
|
||||
raise ValueError("voice-enabled model provenance role is invalid")
|
||||
baseline_identity = (
|
||||
self.text_only_model.artifact_sha256,
|
||||
self.text_only_model.configuration_sha256,
|
||||
)
|
||||
candidate_identity = (
|
||||
self.voice_enabled_model.artifact_sha256,
|
||||
self.voice_enabled_model.configuration_sha256,
|
||||
)
|
||||
if baseline_identity == candidate_identity:
|
||||
raise ValueError("baseline and candidate model provenance must differ")
|
||||
|
||||
participant_keys = [item.participant_key for item in self.participants]
|
||||
if len(set(participant_keys)) != len(participant_keys):
|
||||
raise ValueError("participant assignments must be unique")
|
||||
receipt_hashes = [item.consent_receipt_sha256 for item in self.participants]
|
||||
if len(set(receipt_hashes)) != len(receipt_hashes):
|
||||
raise ValueError("participant consent receipts must be unique")
|
||||
split_by_participant = {
|
||||
item.participant_key: item.split for item in self.participants
|
||||
}
|
||||
if "calibration" not in split_by_participant.values():
|
||||
raise ValueError("participant-level calibration split is required")
|
||||
if "held_out" not in split_by_participant.values():
|
||||
raise ValueError("participant-level held-out split is required")
|
||||
|
||||
attested_labelers = {
|
||||
item.labeler_key for item in self.labeler_attestations
|
||||
}
|
||||
if len(attested_labelers) != len(self.labeler_attestations):
|
||||
raise ValueError("labeler attestations must be unique")
|
||||
reliability_panel = set(self.reliability.labeler_keys)
|
||||
if not reliability_panel.issubset(attested_labelers):
|
||||
raise ValueError("reliability panel requires blind independent attestations")
|
||||
|
||||
observation_ids = [item.observation_id for item in self.observations]
|
||||
if len(set(observation_ids)) != len(observation_ids):
|
||||
raise ValueError("observation ids must be unique")
|
||||
composite_keys = [
|
||||
(item.participant_key, item.session_key, item.axis)
|
||||
for item in self.observations
|
||||
]
|
||||
if len(set(composite_keys)) != len(composite_keys):
|
||||
raise ValueError("paired participant/session/axis observations must be unique")
|
||||
|
||||
participant_by_session: dict[str, str] = {}
|
||||
axes_by_session: dict[str, set[AllianceAxis]] = {}
|
||||
categorical_modes: set[bool] = set()
|
||||
for observation in self.observations:
|
||||
if split_by_participant.get(observation.participant_key) != "held_out":
|
||||
raise ValueError("evaluation observations must use held-out participants")
|
||||
prior_participant = participant_by_session.setdefault(
|
||||
observation.session_key,
|
||||
observation.participant_key,
|
||||
)
|
||||
if prior_participant != observation.participant_key:
|
||||
raise ValueError("a session cannot belong to multiple participants")
|
||||
axes_by_session.setdefault(observation.session_key, set()).add(
|
||||
observation.axis
|
||||
)
|
||||
row_labelers = {item.labeler_key for item in observation.labels}
|
||||
if row_labelers != reliability_panel:
|
||||
raise ValueError("every row must use the declared reliability panel")
|
||||
categorical_modes.add(observation.labels[0].category is not None)
|
||||
|
||||
required_axes: set[AllianceAxis] = {"goal", "task", "bond"}
|
||||
if any(axes != required_axes for axes in axes_by_session.values()):
|
||||
raise ValueError("every held-out session must cover goal, task, and bond")
|
||||
if len(categorical_modes) != 1:
|
||||
raise ValueError("categorical labels must be all-present or all-absent")
|
||||
has_categories = True in categorical_modes
|
||||
if has_categories != (
|
||||
self.reliability.reported_categorical_kappa is not None
|
||||
):
|
||||
raise ValueError("categorical labels and reported kappa must appear together")
|
||||
return self
|
||||
373
apps/api/app/contracts/measurement.py
Normal file
373
apps/api/app/contracts/measurement.py
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
"""Outcome & Alliance OS 측정 원장의 Python 계약.
|
||||
|
||||
점수의 숫자 자체보다 출처, 관점, 도구, 모델 실행, 근거를 먼저 고정한다.
|
||||
이 모듈은 DB/API/프론트 계약이 맞춰야 하는 의미론적 SSOT다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
SOURCE_KINDS = (
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime",
|
||||
)
|
||||
SourceKind = Literal[
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime",
|
||||
]
|
||||
|
||||
MEASUREMENT_CONSTRUCTS = (
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress",
|
||||
)
|
||||
MeasurementConstruct = Literal[
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress",
|
||||
]
|
||||
|
||||
MEASUREMENT_PERSPECTIVES = (
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation",
|
||||
)
|
||||
MeasurementPerspective = Literal[
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation",
|
||||
]
|
||||
|
||||
MEASUREMENT_STATUSES = ("ready", "degraded", "error", "rejected")
|
||||
MeasurementStatus = Literal["ready", "degraded", "error", "rejected"]
|
||||
|
||||
INSTRUMENT_KINDS = (
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric",
|
||||
)
|
||||
InstrumentKind = Literal[
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric",
|
||||
]
|
||||
|
||||
AI_VIEWS = ("client", "counselor", "evaluator", "supervisor", "research")
|
||||
AIView = Literal["client", "counselor", "evaluator", "supervisor", "research"]
|
||||
|
||||
MODEL_RUN_STATUSES = ("ready", "degraded", "error")
|
||||
ModelRunStatus = Literal["ready", "degraded", "error"]
|
||||
|
||||
ALLIANCE_DIMENSIONS = ("goal", "task", "bond")
|
||||
AllianceDimension = Literal["goal", "task", "bond"]
|
||||
|
||||
ALLIANCE_CHECKPOINTS = ("pre", "mid", "post")
|
||||
AllianceCheckpoint = Literal["pre", "mid", "post"]
|
||||
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY: dict[str, frozenset[str]] = {
|
||||
"simulated_state": frozenset({"client_simulation"}),
|
||||
"model_inferred": frozenset({"independent_observer"}),
|
||||
"agent_reported": frozenset({"client_agent_report"}),
|
||||
"learner_reported": frozenset({"learner_self_report"}),
|
||||
"human_rated": frozenset({"supervisor_human"}),
|
||||
"observed_runtime": frozenset({"runtime_observation"}),
|
||||
}
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class MeasurementInstrument(BaseModel):
|
||||
"""척도·훈련지표·시뮬레이션 신호의 버전 고정 레지스트리."""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
instrument_id: str = Field(min_length=1, max_length=120)
|
||||
instrument_version: str = Field(min_length=1, max_length=40)
|
||||
name_ko: str = Field(min_length=1, max_length=200)
|
||||
instrument_kind: InstrumentKind
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
language: str = Field(default="ko-KR", min_length=2, max_length=35)
|
||||
license_id: str | None = Field(default=None, max_length=200)
|
||||
validation_basis: str = Field(min_length=1, max_length=1000)
|
||||
scoring_schema: dict[str, Any] = Field(default_factory=dict)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@field_validator("instrument_id", "instrument_version")
|
||||
@classmethod
|
||||
def strip_identifiers(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("instrument identifiers must not be blank")
|
||||
return stripped
|
||||
|
||||
|
||||
class ModelRun(BaseModel):
|
||||
"""측정을 만든 모델 실행의 재현·드리프트 감사 계약."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
model_run_id: UUID = Field(default_factory=uuid4)
|
||||
session_id: UUID | None = None
|
||||
turn_id: UUID | None = None
|
||||
agent_role: Literal["client", "evaluator", "coach", "scenario", "research"]
|
||||
provider: str = Field(min_length=1, max_length=80)
|
||||
model: str = Field(min_length=1, max_length=160)
|
||||
prompt_bundle_id: str = Field(min_length=1, max_length=160)
|
||||
prompt_bundle_version: str = Field(min_length=1, max_length=40)
|
||||
prompt_bundle_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
structured_schema_version: str = Field(min_length=1, max_length=80)
|
||||
input_evidence_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
status: ModelRunStatus = "ready"
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_error_code_for_failed_run(self) -> "ModelRun":
|
||||
if self.status == "error" and not self.error_code:
|
||||
raise ValueError("error model runs require error_code")
|
||||
if self.status == "ready" and self.error_code:
|
||||
raise ValueError("ready model runs cannot carry error_code")
|
||||
return self
|
||||
|
||||
|
||||
class MeasurementEvent(BaseModel):
|
||||
"""append-only 측정 이벤트.
|
||||
|
||||
정정은 기존 행 변경이 아니라 새 이벤트의 ``supersedes_id``로 표현한다.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
measurement_id: UUID = Field(default_factory=uuid4)
|
||||
session_id: UUID
|
||||
pulse_id: UUID | None = None
|
||||
turn_id: UUID | None = None
|
||||
supersedes_id: UUID | None = None
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
dimension: str = Field(min_length=1, max_length=120)
|
||||
perspective: MeasurementPerspective
|
||||
source_kind: SourceKind
|
||||
instrument_id: str = Field(min_length=1, max_length=120)
|
||||
instrument_version: str = Field(min_length=1, max_length=40)
|
||||
value: float | None = None
|
||||
scale_min: float
|
||||
scale_max: float
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
status: MeasurementStatus = "ready"
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
evidence_turn_ids: tuple[UUID, ...] = ()
|
||||
model_run_id: UUID | None = None
|
||||
visible_to: tuple[AIView, ...] = ("evaluator",)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@field_validator("dimension", "instrument_id", "instrument_version")
|
||||
@classmethod
|
||||
def strip_required_text(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("measurement identifiers must not be blank")
|
||||
return stripped
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_measurement_truth(self) -> "MeasurementEvent":
|
||||
allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]
|
||||
if self.perspective not in allowed:
|
||||
raise ValueError(
|
||||
f"source_kind={self.source_kind} is incompatible with perspective={self.perspective}"
|
||||
)
|
||||
if self.scale_max <= self.scale_min:
|
||||
raise ValueError("scale_max must be greater than scale_min")
|
||||
if self.value is not None and not self.scale_min <= self.value <= self.scale_max:
|
||||
raise ValueError("measurement value must stay inside its declared scale")
|
||||
if self.status == "ready" and self.value is None:
|
||||
raise ValueError("ready measurements require a value")
|
||||
if self.status in {"error", "rejected"}:
|
||||
if self.value is not None:
|
||||
raise ValueError("error/rejected measurements cannot carry a score")
|
||||
if not self.error_code:
|
||||
raise ValueError("error/rejected measurements require error_code")
|
||||
if self.status == "ready" and self.error_code:
|
||||
raise ValueError("ready measurements cannot carry error_code")
|
||||
if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None:
|
||||
raise ValueError("model/agent measurements require model_run_id provenance")
|
||||
if self.supersedes_id == self.measurement_id:
|
||||
raise ValueError("a measurement cannot supersede itself")
|
||||
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
||||
raise ValueError("evidence_turn_ids must be unique")
|
||||
if not self.visible_to:
|
||||
raise ValueError("visible_to must contain at least one audience")
|
||||
if len(set(self.visible_to)) != len(self.visible_to):
|
||||
raise ValueError("visible_to must not contain duplicates")
|
||||
return self
|
||||
|
||||
|
||||
class AllianceScores(BaseModel):
|
||||
"""goal/task/bond를 서로 가리지 않는 독립 0..1 점수."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
goal: float = Field(ge=0.0, le=1.0)
|
||||
task: float = Field(ge=0.0, le=1.0)
|
||||
bond: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class AllianceDimensionAssessment(BaseModel):
|
||||
"""한 관점의 한 동맹 축 평가와 transcript 근거."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
score: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
evidence_turn_indices: tuple[int, ...] = Field(min_length=1, max_length=12)
|
||||
rationale: str = Field(min_length=1, max_length=1200)
|
||||
|
||||
@field_validator("evidence_turn_indices")
|
||||
@classmethod
|
||||
def keep_evidence_indices_unique(cls, value: tuple[int, ...]) -> tuple[int, ...]:
|
||||
if min(value) < 0:
|
||||
raise ValueError("alliance evidence indices must be non-negative")
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("alliance evidence indices must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class AllianceAgentAssessment(BaseModel):
|
||||
"""client-agent 또는 observer가 독립 실행으로 만든 3축 평가."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
goal: AllianceDimensionAssessment
|
||||
task: AllianceDimensionAssessment
|
||||
bond: AllianceDimensionAssessment
|
||||
|
||||
def by_dimension(self) -> dict[AllianceDimension, AllianceDimensionAssessment]:
|
||||
return {"goal": self.goal, "task": self.task, "bond": self.bond}
|
||||
|
||||
|
||||
class BenchmarkTurn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
speaker: Literal["counselor", "client"]
|
||||
text: str = Field(min_length=1)
|
||||
|
||||
|
||||
class BenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
dimension: str = Field(min_length=1)
|
||||
perspective: MeasurementPerspective
|
||||
source_kind: SourceKind
|
||||
direction: Literal["low", "mid", "high", "drop", "rise", "detected", "not_detected"]
|
||||
evidence_turn_indices: tuple[int, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_expectation_truth(self) -> "BenchmarkExpectation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError("benchmark expectation mixes source and perspective layers")
|
||||
if min(self.evidence_turn_indices) < 0:
|
||||
raise ValueError("benchmark turn indices must be zero-based and non-negative")
|
||||
return self
|
||||
|
||||
|
||||
class BenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g0-[0-9]{3}$")
|
||||
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
scene_type: Literal[
|
||||
"goal_mismatch",
|
||||
"task_mismatch",
|
||||
"empathic_miss",
|
||||
"withdrawal",
|
||||
"confrontation",
|
||||
"successful_repair",
|
||||
"failed_repair",
|
||||
"warm_but_directionless",
|
||||
]
|
||||
title_ko: str = Field(min_length=1)
|
||||
description_ko: str = Field(min_length=1)
|
||||
turns: tuple[BenchmarkTurn, ...] = Field(min_length=2)
|
||||
expected: tuple[BenchmarkExpectation, ...] = Field(min_length=1)
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
tags: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_evidence_inside_scene(self) -> "BenchmarkCase":
|
||||
for expectation in self.expected:
|
||||
if max(expectation.evidence_turn_indices) >= len(self.turns):
|
||||
raise ValueError("benchmark evidence points outside the scene")
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AI_VIEWS",
|
||||
"ALLIANCE_CHECKPOINTS",
|
||||
"ALLIANCE_DIMENSIONS",
|
||||
"AllianceAgentAssessment",
|
||||
"AllianceDimensionAssessment",
|
||||
"AllianceScores",
|
||||
"BenchmarkCase",
|
||||
"BenchmarkExpectation",
|
||||
"BenchmarkTurn",
|
||||
"INSTRUMENT_KINDS",
|
||||
"MEASUREMENT_CONSTRUCTS",
|
||||
"MEASUREMENT_PERSPECTIVES",
|
||||
"MEASUREMENT_STATUSES",
|
||||
"MODEL_RUN_STATUSES",
|
||||
"MeasurementEvent",
|
||||
"MeasurementInstrument",
|
||||
"ModelRun",
|
||||
"SOURCE_KINDS",
|
||||
"SOURCE_PERSPECTIVE_COMPATIBILITY",
|
||||
]
|
||||
829
apps/api/app/contracts/measurement_contract.v1.json
Normal file
829
apps/api/app/contracts/measurement_contract.v1.json
Normal file
|
|
@ -0,0 +1,829 @@
|
|||
{
|
||||
"$defs": {
|
||||
"AllianceAgentAssessment": {
|
||||
"$defs": {
|
||||
"AllianceDimensionAssessment": {
|
||||
"additionalProperties": false,
|
||||
"description": "한 관점의 한 동맹 축 평가와 transcript 근거.",
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"title": "Confidence",
|
||||
"type": "number"
|
||||
},
|
||||
"evidence_turn_indices": {
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 12,
|
||||
"minItems": 1,
|
||||
"title": "Evidence Turn Indices",
|
||||
"type": "array"
|
||||
},
|
||||
"rationale": {
|
||||
"maxLength": 1200,
|
||||
"minLength": 1,
|
||||
"title": "Rationale",
|
||||
"type": "string"
|
||||
},
|
||||
"score": {
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"title": "Score",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"score",
|
||||
"confidence",
|
||||
"evidence_turn_indices",
|
||||
"rationale"
|
||||
],
|
||||
"title": "AllianceDimensionAssessment",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"description": "client-agent 또는 observer가 독립 실행으로 만든 3축 평가.",
|
||||
"properties": {
|
||||
"bond": {
|
||||
"$ref": "#/$defs/AllianceDimensionAssessment"
|
||||
},
|
||||
"goal": {
|
||||
"$ref": "#/$defs/AllianceDimensionAssessment"
|
||||
},
|
||||
"task": {
|
||||
"$ref": "#/$defs/AllianceDimensionAssessment"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"goal",
|
||||
"task",
|
||||
"bond"
|
||||
],
|
||||
"title": "AllianceAgentAssessment",
|
||||
"type": "object"
|
||||
},
|
||||
"AllianceScores": {
|
||||
"additionalProperties": false,
|
||||
"description": "goal/task/bond를 서로 가리지 않는 독립 0..1 점수.",
|
||||
"properties": {
|
||||
"bond": {
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"title": "Bond",
|
||||
"type": "number"
|
||||
},
|
||||
"goal": {
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"title": "Goal",
|
||||
"type": "number"
|
||||
},
|
||||
"task": {
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"title": "Task",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"goal",
|
||||
"task",
|
||||
"bond"
|
||||
],
|
||||
"title": "AllianceScores",
|
||||
"type": "object"
|
||||
},
|
||||
"BenchmarkCase": {
|
||||
"$defs": {
|
||||
"BenchmarkExpectation": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"construct": {
|
||||
"enum": [
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress"
|
||||
],
|
||||
"title": "Construct",
|
||||
"type": "string"
|
||||
},
|
||||
"dimension": {
|
||||
"minLength": 1,
|
||||
"title": "Dimension",
|
||||
"type": "string"
|
||||
},
|
||||
"direction": {
|
||||
"enum": [
|
||||
"low",
|
||||
"mid",
|
||||
"high",
|
||||
"drop",
|
||||
"rise",
|
||||
"detected",
|
||||
"not_detected"
|
||||
],
|
||||
"title": "Direction",
|
||||
"type": "string"
|
||||
},
|
||||
"evidence_turn_indices": {
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"minItems": 1,
|
||||
"title": "Evidence Turn Indices",
|
||||
"type": "array"
|
||||
},
|
||||
"perspective": {
|
||||
"enum": [
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation"
|
||||
],
|
||||
"title": "Perspective",
|
||||
"type": "string"
|
||||
},
|
||||
"source_kind": {
|
||||
"enum": [
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime"
|
||||
],
|
||||
"title": "Source Kind",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"construct",
|
||||
"dimension",
|
||||
"perspective",
|
||||
"source_kind",
|
||||
"direction",
|
||||
"evidence_turn_indices"
|
||||
],
|
||||
"title": "BenchmarkExpectation",
|
||||
"type": "object"
|
||||
},
|
||||
"BenchmarkTurn": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"speaker": {
|
||||
"enum": [
|
||||
"counselor",
|
||||
"client"
|
||||
],
|
||||
"title": "Speaker",
|
||||
"type": "string"
|
||||
},
|
||||
"text": {
|
||||
"minLength": 1,
|
||||
"title": "Text",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"speaker",
|
||||
"text"
|
||||
],
|
||||
"title": "BenchmarkTurn",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"case_id": {
|
||||
"pattern": "^oas-g0-[0-9]{3}$",
|
||||
"title": "Case Id",
|
||||
"type": "string"
|
||||
},
|
||||
"description_ko": {
|
||||
"minLength": 1,
|
||||
"title": "Description Ko",
|
||||
"type": "string"
|
||||
},
|
||||
"expected": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/BenchmarkExpectation"
|
||||
},
|
||||
"minItems": 1,
|
||||
"title": "Expected",
|
||||
"type": "array"
|
||||
},
|
||||
"forbidden_claims": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"minItems": 1,
|
||||
"title": "Forbidden Claims",
|
||||
"type": "array"
|
||||
},
|
||||
"scene_type": {
|
||||
"enum": [
|
||||
"goal_mismatch",
|
||||
"task_mismatch",
|
||||
"empathic_miss",
|
||||
"withdrawal",
|
||||
"confrontation",
|
||||
"successful_repair",
|
||||
"failed_repair",
|
||||
"warm_but_directionless"
|
||||
],
|
||||
"title": "Scene Type",
|
||||
"type": "string"
|
||||
},
|
||||
"tags": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Tags",
|
||||
"type": "array"
|
||||
},
|
||||
"title_ko": {
|
||||
"minLength": 1,
|
||||
"title": "Title Ko",
|
||||
"type": "string"
|
||||
},
|
||||
"turns": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/BenchmarkTurn"
|
||||
},
|
||||
"minItems": 2,
|
||||
"title": "Turns",
|
||||
"type": "array"
|
||||
},
|
||||
"version": {
|
||||
"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$",
|
||||
"title": "Version",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"case_id",
|
||||
"version",
|
||||
"scene_type",
|
||||
"title_ko",
|
||||
"description_ko",
|
||||
"turns",
|
||||
"expected",
|
||||
"forbidden_claims"
|
||||
],
|
||||
"title": "BenchmarkCase",
|
||||
"type": "object"
|
||||
},
|
||||
"MeasurementEvent": {
|
||||
"additionalProperties": false,
|
||||
"description": "append-only 측정 이벤트.\n\n정정은 기존 행 변경이 아니라 새 이벤트의 ``supersedes_id``로 표현한다.",
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maximum": 1.0,
|
||||
"minimum": 0.0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Confidence"
|
||||
},
|
||||
"construct": {
|
||||
"enum": [
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress"
|
||||
],
|
||||
"title": "Construct",
|
||||
"type": "string"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"dimension": {
|
||||
"maxLength": 120,
|
||||
"minLength": 1,
|
||||
"title": "Dimension",
|
||||
"type": "string"
|
||||
},
|
||||
"error_code": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 120,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Error Code"
|
||||
},
|
||||
"evidence_turn_ids": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Evidence Turn Ids",
|
||||
"type": "array"
|
||||
},
|
||||
"instrument_id": {
|
||||
"maxLength": 120,
|
||||
"minLength": 1,
|
||||
"title": "Instrument Id",
|
||||
"type": "string"
|
||||
},
|
||||
"instrument_version": {
|
||||
"maxLength": 40,
|
||||
"minLength": 1,
|
||||
"title": "Instrument Version",
|
||||
"type": "string"
|
||||
},
|
||||
"measurement_id": {
|
||||
"format": "uuid",
|
||||
"title": "Measurement Id",
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"title": "Metadata",
|
||||
"type": "object"
|
||||
},
|
||||
"model_run_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Model Run Id"
|
||||
},
|
||||
"perspective": {
|
||||
"enum": [
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation"
|
||||
],
|
||||
"title": "Perspective",
|
||||
"type": "string"
|
||||
},
|
||||
"pulse_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Pulse Id"
|
||||
},
|
||||
"scale_max": {
|
||||
"title": "Scale Max",
|
||||
"type": "number"
|
||||
},
|
||||
"scale_min": {
|
||||
"title": "Scale Min",
|
||||
"type": "number"
|
||||
},
|
||||
"session_id": {
|
||||
"format": "uuid",
|
||||
"title": "Session Id",
|
||||
"type": "string"
|
||||
},
|
||||
"source_kind": {
|
||||
"enum": [
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime"
|
||||
],
|
||||
"title": "Source Kind",
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"default": "ready",
|
||||
"enum": [
|
||||
"ready",
|
||||
"degraded",
|
||||
"error",
|
||||
"rejected"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"supersedes_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Supersedes Id"
|
||||
},
|
||||
"turn_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Turn Id"
|
||||
},
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Value"
|
||||
},
|
||||
"visible_to": {
|
||||
"default": [
|
||||
"evaluator"
|
||||
],
|
||||
"items": {
|
||||
"enum": [
|
||||
"client",
|
||||
"counselor",
|
||||
"evaluator",
|
||||
"supervisor",
|
||||
"research"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Visible To",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"session_id",
|
||||
"construct",
|
||||
"dimension",
|
||||
"perspective",
|
||||
"source_kind",
|
||||
"instrument_id",
|
||||
"instrument_version",
|
||||
"scale_min",
|
||||
"scale_max"
|
||||
],
|
||||
"title": "MeasurementEvent",
|
||||
"type": "object"
|
||||
},
|
||||
"MeasurementInstrument": {
|
||||
"additionalProperties": false,
|
||||
"description": "척도·훈련지표·시뮬레이션 신호의 버전 고정 레지스트리.",
|
||||
"properties": {
|
||||
"construct": {
|
||||
"enum": [
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress"
|
||||
],
|
||||
"title": "Construct",
|
||||
"type": "string"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"instrument_id": {
|
||||
"maxLength": 120,
|
||||
"minLength": 1,
|
||||
"title": "Instrument Id",
|
||||
"type": "string"
|
||||
},
|
||||
"instrument_kind": {
|
||||
"enum": [
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric"
|
||||
],
|
||||
"title": "Instrument Kind",
|
||||
"type": "string"
|
||||
},
|
||||
"instrument_version": {
|
||||
"maxLength": 40,
|
||||
"minLength": 1,
|
||||
"title": "Instrument Version",
|
||||
"type": "string"
|
||||
},
|
||||
"language": {
|
||||
"default": "ko-KR",
|
||||
"maxLength": 35,
|
||||
"minLength": 2,
|
||||
"title": "Language",
|
||||
"type": "string"
|
||||
},
|
||||
"license_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 200,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "License Id"
|
||||
},
|
||||
"metadata": {
|
||||
"title": "Metadata",
|
||||
"type": "object"
|
||||
},
|
||||
"name_ko": {
|
||||
"maxLength": 200,
|
||||
"minLength": 1,
|
||||
"title": "Name Ko",
|
||||
"type": "string"
|
||||
},
|
||||
"scoring_schema": {
|
||||
"title": "Scoring Schema",
|
||||
"type": "object"
|
||||
},
|
||||
"validation_basis": {
|
||||
"maxLength": 1000,
|
||||
"minLength": 1,
|
||||
"title": "Validation Basis",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"instrument_id",
|
||||
"instrument_version",
|
||||
"name_ko",
|
||||
"instrument_kind",
|
||||
"construct",
|
||||
"validation_basis"
|
||||
],
|
||||
"title": "MeasurementInstrument",
|
||||
"type": "object"
|
||||
},
|
||||
"ModelRun": {
|
||||
"additionalProperties": false,
|
||||
"description": "측정을 만든 모델 실행의 재현·드리프트 감사 계약.",
|
||||
"properties": {
|
||||
"agent_role": {
|
||||
"enum": [
|
||||
"client",
|
||||
"evaluator",
|
||||
"coach",
|
||||
"scenario",
|
||||
"research"
|
||||
],
|
||||
"title": "Agent Role",
|
||||
"type": "string"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"error_code": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 120,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Error Code"
|
||||
},
|
||||
"input_evidence_hash": {
|
||||
"pattern": "^[a-f0-9]{64}$",
|
||||
"title": "Input Evidence Hash",
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"title": "Metadata",
|
||||
"type": "object"
|
||||
},
|
||||
"model": {
|
||||
"maxLength": 160,
|
||||
"minLength": 1,
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"model_run_id": {
|
||||
"format": "uuid",
|
||||
"title": "Model Run Id",
|
||||
"type": "string"
|
||||
},
|
||||
"prompt_bundle_hash": {
|
||||
"pattern": "^[a-f0-9]{64}$",
|
||||
"title": "Prompt Bundle Hash",
|
||||
"type": "string"
|
||||
},
|
||||
"prompt_bundle_id": {
|
||||
"maxLength": 160,
|
||||
"minLength": 1,
|
||||
"title": "Prompt Bundle Id",
|
||||
"type": "string"
|
||||
},
|
||||
"prompt_bundle_version": {
|
||||
"maxLength": 40,
|
||||
"minLength": 1,
|
||||
"title": "Prompt Bundle Version",
|
||||
"type": "string"
|
||||
},
|
||||
"provider": {
|
||||
"maxLength": 80,
|
||||
"minLength": 1,
|
||||
"title": "Provider",
|
||||
"type": "string"
|
||||
},
|
||||
"session_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Session Id"
|
||||
},
|
||||
"status": {
|
||||
"default": "ready",
|
||||
"enum": [
|
||||
"ready",
|
||||
"degraded",
|
||||
"error"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"structured_schema_version": {
|
||||
"maxLength": 80,
|
||||
"minLength": 1,
|
||||
"title": "Structured Schema Version",
|
||||
"type": "string"
|
||||
},
|
||||
"turn_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Turn Id"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"agent_role",
|
||||
"provider",
|
||||
"model",
|
||||
"prompt_bundle_id",
|
||||
"prompt_bundle_version",
|
||||
"prompt_bundle_hash",
|
||||
"structured_schema_version",
|
||||
"input_evidence_hash"
|
||||
],
|
||||
"title": "ModelRun",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"$id": "https://vignette.local/contracts/measurement_contract.v1.json",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"enums": {
|
||||
"aiViews": [
|
||||
"client",
|
||||
"counselor",
|
||||
"evaluator",
|
||||
"supervisor",
|
||||
"research"
|
||||
],
|
||||
"allianceCheckpoints": [
|
||||
"pre",
|
||||
"mid",
|
||||
"post"
|
||||
],
|
||||
"allianceDimensions": [
|
||||
"goal",
|
||||
"task",
|
||||
"bond"
|
||||
],
|
||||
"constructs": [
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress"
|
||||
],
|
||||
"instrumentKinds": [
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric"
|
||||
],
|
||||
"measurementStatuses": [
|
||||
"ready",
|
||||
"degraded",
|
||||
"error",
|
||||
"rejected"
|
||||
],
|
||||
"modelRunStatuses": [
|
||||
"ready",
|
||||
"degraded",
|
||||
"error"
|
||||
],
|
||||
"perspectives": [
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation"
|
||||
],
|
||||
"sourceKinds": [
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime"
|
||||
]
|
||||
},
|
||||
"sourcePerspectiveCompatibility": {
|
||||
"agent_reported": [
|
||||
"client_agent_report"
|
||||
],
|
||||
"human_rated": [
|
||||
"supervisor_human"
|
||||
],
|
||||
"learner_reported": [
|
||||
"learner_self_report"
|
||||
],
|
||||
"model_inferred": [
|
||||
"independent_observer"
|
||||
],
|
||||
"observed_runtime": [
|
||||
"runtime_observation"
|
||||
],
|
||||
"simulated_state": [
|
||||
"client_simulation"
|
||||
]
|
||||
},
|
||||
"title": "Vignette Outcome & Alliance Measurement Contract",
|
||||
"version": 1
|
||||
}
|
||||
265
apps/api/app/contracts/multimodal_alliance.py
Normal file
265
apps/api/app/contracts/multimodal_alliance.py
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
"""G7 Multimodal Alliance의 시간정렬·독립측정·보존 계약."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
AllianceAxis = Literal["goal", "task", "bond"]
|
||||
Modality = Literal["text", "voice"]
|
||||
MeasurementStatus = Literal["ready", "missing", "error"]
|
||||
VoiceEventType = Literal[
|
||||
"silence", "overlap", "interruption", "prosody", "pace", "audio_quality"
|
||||
]
|
||||
|
||||
|
||||
class WordTimestamp(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
word_index: int = Field(ge=0)
|
||||
start_ms: int = Field(ge=0)
|
||||
end_ms: int = Field(gt=0)
|
||||
speaker: Literal["learner", "client"]
|
||||
token_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_positive_span(self) -> "WordTimestamp":
|
||||
if self.end_ms <= self.start_ms:
|
||||
raise ValueError("word timestamp end must follow start")
|
||||
return self
|
||||
|
||||
|
||||
class VoiceInteractionEvent(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
event_id: str = Field(pattern=r"^oas-g7-event-[a-z0-9-]+$")
|
||||
event_type: VoiceEventType
|
||||
start_ms: int = Field(ge=0)
|
||||
end_ms: int = Field(gt=0)
|
||||
actor: Literal["learner", "client", "both", "channel"]
|
||||
observed_feature: str = Field(min_length=1, max_length=200)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
source: Literal["observed_audio_runtime", "stt_word_timestamps"]
|
||||
claim_scope: Literal["interaction_signal"] = "interaction_signal"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def prevent_clinical_interpretation(self) -> "VoiceInteractionEvent":
|
||||
if self.end_ms <= self.start_ms:
|
||||
raise ValueError("voice interaction event end must follow start")
|
||||
forbidden = {
|
||||
"diagnosis",
|
||||
"depression",
|
||||
"anxiety disorder",
|
||||
"is sad",
|
||||
"is angry",
|
||||
"feels anxious",
|
||||
"진단",
|
||||
"우울증",
|
||||
"불안장애",
|
||||
"자살 위험",
|
||||
"감정은",
|
||||
"감정이",
|
||||
"기분이",
|
||||
"슬픔을 느",
|
||||
"불안을 느",
|
||||
"화가 났",
|
||||
}
|
||||
normalized = self.observed_feature.casefold()
|
||||
if any(term in normalized for term in forbidden):
|
||||
raise ValueError("voice event must not infer a clinical condition")
|
||||
return self
|
||||
|
||||
|
||||
class AlignedVoiceTimeline(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
audio_duration_ms: int = Field(gt=0)
|
||||
words: tuple[WordTimestamp, ...]
|
||||
events: tuple[VoiceInteractionEvent, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_in_bounds_timeline(self) -> "AlignedVoiceTimeline":
|
||||
indices = [item.word_index for item in self.words]
|
||||
if indices != list(range(len(indices))):
|
||||
raise ValueError("word timestamps must be contiguous and ordered")
|
||||
if [item.start_ms for item in self.words] != sorted(
|
||||
item.start_ms for item in self.words
|
||||
):
|
||||
raise ValueError("word timestamps must be time ordered")
|
||||
if any(item.end_ms > self.audio_duration_ms for item in self.words):
|
||||
raise ValueError("word timestamp exceeds audio duration")
|
||||
if any(item.end_ms > self.audio_duration_ms for item in self.events):
|
||||
raise ValueError("voice event exceeds audio duration")
|
||||
if len({item.event_id for item in self.events}) != len(self.events):
|
||||
raise ValueError("voice event ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class ModalityAxisMeasurement(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
measurement_id: str = Field(pattern=r"^oas-g7-measurement-[a-z0-9-]+$")
|
||||
axis: AllianceAxis
|
||||
modality: Modality
|
||||
status: MeasurementStatus
|
||||
value: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
model_run_id: UUID | None = None
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_modality_measurement_truth(self) -> "ModalityAxisMeasurement":
|
||||
if self.status == "ready":
|
||||
if (
|
||||
self.value is None
|
||||
or self.confidence is None
|
||||
or not self.evidence_refs
|
||||
or self.model_run_id is None
|
||||
or self.error_code is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"ready modality measurement requires model and evidence"
|
||||
)
|
||||
else:
|
||||
if self.value is not None or self.confidence is not None:
|
||||
raise ValueError(
|
||||
"missing/error modality measurement must remain scoreless"
|
||||
)
|
||||
if self.status == "error" and (
|
||||
not self.error_code or self.uncertainty != 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"error modality measurement requires maximum uncertainty"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class FusionCalibration(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
calibration_id: str = Field(pattern=r"^oas-g7-fusion-[a-z0-9-]+$")
|
||||
axis: AllianceAxis
|
||||
text_weight: float = Field(ge=0.0, le=1.0)
|
||||
voice_weight: float = Field(ge=0.0, le=1.0)
|
||||
text_only_accuracy: float = Field(ge=0.0, le=1.0)
|
||||
fused_accuracy: float = Field(ge=0.0, le=1.0)
|
||||
benchmark_version: str = Field(min_length=1, max_length=80)
|
||||
minimum_incremental_gain: float = Field(default=0.01, ge=0.0, le=1.0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_normalized_weights(self) -> "FusionCalibration":
|
||||
if abs(self.text_weight + self.voice_weight - 1.0) > 1e-9:
|
||||
raise ValueError("fusion weights must sum to one")
|
||||
return self
|
||||
|
||||
@property
|
||||
def incremental_gain(self) -> float:
|
||||
return self.fused_accuracy - self.text_only_accuracy
|
||||
|
||||
|
||||
class CalibratedAxisReadModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
axis: AllianceAxis
|
||||
status: MeasurementStatus
|
||||
value: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
modalities_used: tuple[Modality, ...]
|
||||
measurement_ids: tuple[str, ...]
|
||||
fusion_applied: bool
|
||||
fusion_calibration_id: str | None = None
|
||||
incremental_gain: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_traceable_fusion(self) -> "CalibratedAxisReadModel":
|
||||
if self.status != "ready" and self.value is not None:
|
||||
raise ValueError("non-ready calibrated axis must remain scoreless")
|
||||
if self.fusion_applied and (
|
||||
set(self.modalities_used) != {"text", "voice"}
|
||||
or self.fusion_calibration_id is None
|
||||
or self.incremental_gain is None
|
||||
):
|
||||
raise ValueError(
|
||||
"fusion requires both modalities and calibration provenance"
|
||||
)
|
||||
if not self.fusion_applied and self.fusion_calibration_id is not None:
|
||||
raise ValueError("text-only read model cannot claim fusion calibration")
|
||||
return self
|
||||
|
||||
|
||||
class AudioRetentionRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_id: str = Field(min_length=1, max_length=180)
|
||||
consent_status: Literal["granted", "withdrawn", "not_granted"]
|
||||
audio_ref: str | None = Field(default=None, max_length=300)
|
||||
audio_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
||||
retained_until_sequence: int | None = Field(default=None, ge=1)
|
||||
deletion_event_id: str | None = Field(default=None, max_length=180)
|
||||
transcript_retained: bool
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_consent_and_deletion(self) -> "AudioRetentionRecord":
|
||||
if self.consent_status == "granted":
|
||||
if (
|
||||
not self.audio_ref
|
||||
or not self.audio_sha256
|
||||
or self.retained_until_sequence is None
|
||||
):
|
||||
raise ValueError(
|
||||
"granted audio retention requires ref, hash, and expiry"
|
||||
)
|
||||
if self.deletion_event_id is not None:
|
||||
raise ValueError("retained audio cannot already have deletion evidence")
|
||||
else:
|
||||
if self.audio_ref or self.audio_sha256 or self.retained_until_sequence:
|
||||
raise ValueError("unconsented audio must not retain audio material")
|
||||
if self.consent_status == "withdrawn" and not self.deletion_event_id:
|
||||
raise ValueError("withdrawn audio requires deletion evidence")
|
||||
return self
|
||||
|
||||
|
||||
class MultimodalBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g7-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
text_measurement: ModalityAxisMeasurement
|
||||
voice_measurement: ModalityAxisMeasurement
|
||||
calibration: FusionCalibration
|
||||
target_value: float = Field(ge=0.0, le=1.0)
|
||||
expected_fusion_applied: bool
|
||||
|
||||
|
||||
class MultimodalBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.multimodal-alliance-benchmark.v1"]
|
||||
version: Literal["1.0.0"]
|
||||
data_classification: Literal["synthetic_educational"]
|
||||
clinical_claim_allowed: Literal[False]
|
||||
cases: tuple[MultimodalBenchmarkCase, ...] = Field(min_length=3)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AlignedVoiceTimeline",
|
||||
"AllianceAxis",
|
||||
"AudioRetentionRecord",
|
||||
"CalibratedAxisReadModel",
|
||||
"FusionCalibration",
|
||||
"MeasurementStatus",
|
||||
"Modality",
|
||||
"ModalityAxisMeasurement",
|
||||
"MultimodalBenchmarkCase",
|
||||
"MultimodalBenchmarkPack",
|
||||
"VoiceEventType",
|
||||
"VoiceInteractionEvent",
|
||||
"WordTimestamp",
|
||||
]
|
||||
410
apps/api/app/contracts/outcome_trajectory.py
Normal file
410
apps/api/app/contracts/outcome_trajectory.py
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
"""G2 Longitudinal Outcome Trajectory의 순수 도메인 계약.
|
||||
|
||||
이 계약은 교육용 합성 사례의 1~5회기 진행 궤적을 다룬다. 임상 진단·치료 효과
|
||||
예측 계약이 아니며, 실제 데이터 부재를 정상값이나 평균값으로 대체하지 않는다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
AIView,
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
OUTCOME_AXES = ("distress_load", "daily_functioning", "learning_engagement")
|
||||
OutcomeAxis = Literal["distress_load", "daily_functioning", "learning_engagement"]
|
||||
|
||||
TRAJECTORY_STATUSES = (
|
||||
"on_track",
|
||||
"watch",
|
||||
"off_track",
|
||||
"deteriorating",
|
||||
"insufficient_evidence",
|
||||
)
|
||||
TrajectoryStatus = Literal[
|
||||
"on_track",
|
||||
"watch",
|
||||
"off_track",
|
||||
"deteriorating",
|
||||
"insufficient_evidence",
|
||||
]
|
||||
|
||||
OBSERVATION_STATUSES = ("observed", "missing", "error")
|
||||
ObservationStatus = Literal["observed", "missing", "error"]
|
||||
|
||||
RELATIONSHIP_EVENT_TYPES = (
|
||||
"goal_agreement",
|
||||
"task_agreement",
|
||||
"rupture_withdrawal",
|
||||
"rupture_confrontation",
|
||||
"repair_attempt",
|
||||
"repair_confirmed",
|
||||
"unresolved_rupture",
|
||||
)
|
||||
RelationshipEventType = Literal[
|
||||
"goal_agreement",
|
||||
"task_agreement",
|
||||
"rupture_withdrawal",
|
||||
"rupture_confrontation",
|
||||
"repair_attempt",
|
||||
"repair_confirmed",
|
||||
"unresolved_rupture",
|
||||
]
|
||||
|
||||
|
||||
class SyntheticExpectedDistribution(BaseModel):
|
||||
"""한 축·한 회기의 교육용 예상 분포.
|
||||
|
||||
``mean``은 관측값의 폴백이 아니다. 관측값이 없으면 비교 자체를 하지 않는다.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
axis: OutcomeAxis
|
||||
mean: float = Field(ge=0.0, le=1.0)
|
||||
standard_deviation: float = Field(gt=0.0, le=0.5)
|
||||
lower_reference: float = Field(ge=0.0, le=1.0)
|
||||
upper_reference: float = Field(ge=0.0, le=1.0)
|
||||
sample_size: int = Field(ge=1)
|
||||
expected_direction: Literal["lower_is_better", "higher_is_better"]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_reference_band_ordered(self) -> "SyntheticExpectedDistribution":
|
||||
if not self.lower_reference <= self.mean <= self.upper_reference:
|
||||
raise ValueError("expected mean must stay inside its reference band")
|
||||
return self
|
||||
|
||||
|
||||
class SyntheticExpectedArc(BaseModel):
|
||||
"""임상 오표시를 구조적으로 막는 5회기 교육용 예상 궤적."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.synthetic-outcome-arc.v1"] = (
|
||||
"vignette.synthetic-outcome-arc.v1"
|
||||
)
|
||||
arc_id: str = Field(pattern=r"^oas-g2-arc-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
provenance_note: str = Field(min_length=20, max_length=1000)
|
||||
distributions: tuple[SyntheticExpectedDistribution, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_five_session_distribution(self) -> "SyntheticExpectedArc":
|
||||
keys = {(item.session_no, item.axis) for item in self.distributions}
|
||||
expected = {
|
||||
(session_no, axis)
|
||||
for session_no in range(1, 6)
|
||||
for axis in OUTCOME_AXES
|
||||
}
|
||||
if keys != expected or len(self.distributions) != len(expected):
|
||||
raise ValueError("synthetic arc requires every outcome axis for sessions 1..5")
|
||||
for item in self.distributions:
|
||||
if item.axis == "distress_load" and item.expected_direction != "lower_is_better":
|
||||
raise ValueError("distress_load must use lower_is_better direction")
|
||||
if item.axis != "distress_load" and item.expected_direction != "higher_is_better":
|
||||
raise ValueError(f"{item.axis} must use higher_is_better direction")
|
||||
return self
|
||||
|
||||
def distribution_for(
|
||||
self, session_no: int, axis: OutcomeAxis
|
||||
) -> SyntheticExpectedDistribution:
|
||||
for item in self.distributions:
|
||||
if item.session_no == session_no and item.axis == axis:
|
||||
return item
|
||||
raise KeyError((session_no, axis))
|
||||
|
||||
|
||||
class OutcomeAxisObservation(BaseModel):
|
||||
"""실제 관측 여부를 보존하는 한 축의 회기 관측."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
axis: OutcomeAxis
|
||||
status: ObservationStatus = "observed"
|
||||
value: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective = "client_simulation"
|
||||
instrument_id: str = Field(min_length=1, max_length=120)
|
||||
instrument_version: str = Field(min_length=1, max_length=40)
|
||||
model_run_id: UUID | None = None
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
missing_reason: str | None = Field(default=None, max_length=300)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def never_impute_missing_observations(self) -> "OutcomeAxisObservation":
|
||||
if self.status == "observed":
|
||||
if self.value is None or self.confidence is None:
|
||||
raise ValueError("observed outcomes require value and confidence")
|
||||
if self.missing_reason:
|
||||
raise ValueError("observed outcomes cannot carry missing_reason")
|
||||
if not self.evidence_refs:
|
||||
raise ValueError("observed outcomes require evidence_refs")
|
||||
else:
|
||||
if self.value is not None or self.confidence is not None:
|
||||
raise ValueError("missing/error outcomes must remain scoreless")
|
||||
if not self.missing_reason:
|
||||
raise ValueError("missing/error outcomes require missing_reason")
|
||||
if len(set(self.evidence_refs)) != len(self.evidence_refs):
|
||||
raise ValueError("outcome evidence_refs must be unique")
|
||||
allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]
|
||||
if self.perspective not in allowed:
|
||||
raise ValueError("outcome observation mixes source and perspective layers")
|
||||
if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None:
|
||||
raise ValueError("model/agent outcome observations require model_run_id provenance")
|
||||
return self
|
||||
|
||||
|
||||
class SafetySignalReference(BaseModel):
|
||||
"""성과 악화와 별도로 전달되는 기존 safety 원장 참조."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
safety_event_id: str = Field(min_length=1, max_length=160)
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
risk_level: Literal["low", "moderate", "high", "imminent"]
|
||||
escalated: bool
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class RelationshipMemoryEvent(BaseModel):
|
||||
"""역할별 문장을 서로 섞지 않는 관계 기억 원본.
|
||||
|
||||
범용 ``metadata`` 필드를 의도적으로 두지 않는다. 각 역할은 자기 키의 summary만
|
||||
받을 수 있어 내담자 내부 설정이나 평가자 정답 라벨이 상담자 뷰로 새지 않는다.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
event_id: str = Field(min_length=1, max_length=160)
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
event_type: RelationshipEventType
|
||||
visible_to: tuple[AIView, ...] = Field(min_length=1)
|
||||
summaries: dict[AIView, str] = Field(min_length=1)
|
||||
evidence_refs: tuple[str, ...] = Field(min_length=1)
|
||||
resolved_by_event_id: str | None = Field(default=None, max_length=160)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_role_safe_summary_keys(self) -> "RelationshipMemoryEvent":
|
||||
if len(set(self.visible_to)) != len(self.visible_to):
|
||||
raise ValueError("relationship visible_to must be unique")
|
||||
if set(self.summaries) != set(self.visible_to):
|
||||
raise ValueError("relationship summaries must exactly match visible_to roles")
|
||||
if any(not text.strip() for text in self.summaries.values()):
|
||||
raise ValueError("relationship summaries must not be blank")
|
||||
if len(set(self.evidence_refs)) != len(self.evidence_refs):
|
||||
raise ValueError("relationship evidence_refs must be unique")
|
||||
if self.resolved_by_event_id == self.event_id:
|
||||
raise ValueError("relationship event cannot resolve itself")
|
||||
return self
|
||||
|
||||
|
||||
class ObservedSessionOutcome(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
axes: tuple[OutcomeAxisObservation, ...] = Field(min_length=3, max_length=3)
|
||||
safety_signals: tuple[SafetySignalReference, ...] = ()
|
||||
relationship_events: tuple[RelationshipMemoryEvent, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_explicit_axis_presence(self) -> "ObservedSessionOutcome":
|
||||
axes = [item.axis for item in self.axes]
|
||||
if set(axes) != set(OUTCOME_AXES) or len(set(axes)) != len(OUTCOME_AXES):
|
||||
raise ValueError("every session must explicitly represent all outcome axes")
|
||||
if any(item.session_no != self.session_no for item in self.safety_signals):
|
||||
raise ValueError("safety references must belong to the observed session")
|
||||
if any(item.session_no != self.session_no for item in self.relationship_events):
|
||||
raise ValueError("relationship events must belong to the observed session")
|
||||
return self
|
||||
|
||||
def observation_for(self, axis: OutcomeAxis) -> OutcomeAxisObservation:
|
||||
for item in self.axes:
|
||||
if item.axis == axis:
|
||||
return item
|
||||
raise KeyError(axis)
|
||||
|
||||
|
||||
class LongitudinalOutcomeInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
expected_arc: SyntheticExpectedArc
|
||||
sessions: tuple[ObservedSessionOutcome, ...] = Field(min_length=1, max_length=5)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_contiguous_early_sessions(self) -> "LongitudinalOutcomeInput":
|
||||
session_numbers = [item.session_no for item in self.sessions]
|
||||
if session_numbers != list(range(1, len(self.sessions) + 1)):
|
||||
raise ValueError("outcome sessions must be contiguous and ordered from session 1")
|
||||
safety_ids = [
|
||||
signal.safety_event_id
|
||||
for session in self.sessions
|
||||
for signal in session.safety_signals
|
||||
]
|
||||
if len(set(safety_ids)) != len(safety_ids):
|
||||
raise ValueError("safety event references must be unique across the arc")
|
||||
events = [
|
||||
event
|
||||
for session in self.sessions
|
||||
for event in session.relationship_events
|
||||
]
|
||||
event_by_id = {event.event_id: event for event in events}
|
||||
if len(event_by_id) != len(events):
|
||||
raise ValueError("relationship event ids must be unique across the arc")
|
||||
for event in events:
|
||||
if not event.resolved_by_event_id:
|
||||
continue
|
||||
resolution = event_by_id.get(event.resolved_by_event_id)
|
||||
if resolution is None:
|
||||
raise ValueError("relationship resolution reference must exist in the arc")
|
||||
if resolution.event_type != "repair_confirmed":
|
||||
raise ValueError("relationship resolution must point to repair_confirmed")
|
||||
if resolution.session_no < event.session_no:
|
||||
raise ValueError("relationship resolution cannot precede the rupture")
|
||||
return self
|
||||
|
||||
|
||||
class AxisTrajectoryAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
axis: OutcomeAxis
|
||||
status: TrajectoryStatus
|
||||
observed_value: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
expected_mean: float = Field(ge=0.0, le=1.0)
|
||||
adverse_z: float | None = None
|
||||
adverse_z_change: float | None = None
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
decision_basis: tuple[str, ...] = Field(min_length=1)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_missing_assessment_scoreless(self) -> "AxisTrajectoryAssessment":
|
||||
if self.status == "insufficient_evidence":
|
||||
if self.observed_value is not None or self.adverse_z is not None:
|
||||
raise ValueError("insufficient evidence assessment must remain scoreless")
|
||||
if self.uncertainty != 1.0:
|
||||
raise ValueError("insufficient evidence must expose maximum uncertainty")
|
||||
elif self.observed_value is None or self.adverse_z is None:
|
||||
raise ValueError("classified trajectory axes require observed evidence")
|
||||
return self
|
||||
|
||||
|
||||
class SessionTrajectoryAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
status: TrajectoryStatus
|
||||
axes: tuple[AxisTrajectoryAssessment, ...] = Field(min_length=3, max_length=3)
|
||||
missing_axes: tuple[OutcomeAxis, ...] = ()
|
||||
next_check_questions: tuple[str, ...] = ()
|
||||
safety_signals: tuple[SafetySignalReference, ...] = ()
|
||||
|
||||
|
||||
class LongitudinalOutcomeAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.outcome-trajectory-assessment.v1"] = (
|
||||
"vignette.outcome-trajectory-assessment.v1"
|
||||
)
|
||||
expected_arc_id: str
|
||||
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
sessions: tuple[SessionTrajectoryAssessment, ...]
|
||||
|
||||
|
||||
class RelationshipMemoryProjection(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
event_id: str
|
||||
session_no: int
|
||||
event_type: RelationshipEventType
|
||||
summary: str
|
||||
evidence_refs: tuple[str, ...]
|
||||
resolved_by_event_id: str | None = None
|
||||
|
||||
|
||||
class RoleSafeTrajectoryReadModel(BaseModel):
|
||||
"""성과·safety·관계 기억을 합산하지 않고 나란히 전달하는 읽기 모델."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
assessment: LongitudinalOutcomeAssessment
|
||||
safety_signals: tuple[SafetySignalReference, ...]
|
||||
relationship_memory: tuple[RelationshipMemoryProjection, ...]
|
||||
|
||||
|
||||
class TrajectoryBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
session_no: int = Field(ge=1, le=5)
|
||||
status: TrajectoryStatus
|
||||
|
||||
|
||||
class TrajectoryBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g2-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
sessions: tuple[ObservedSessionOutcome, ...] = Field(min_length=5, max_length=5)
|
||||
expected: tuple[TrajectoryBenchmarkExpectation, ...] = Field(
|
||||
min_length=5, max_length=5
|
||||
)
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_benchmark_timeline(self) -> "TrajectoryBenchmarkCase":
|
||||
session_numbers = [item.session_no for item in self.sessions]
|
||||
expectation_numbers = [item.session_no for item in self.expected]
|
||||
if session_numbers != [1, 2, 3, 4, 5]:
|
||||
raise ValueError("benchmark sessions must be ordered 1..5")
|
||||
if expectation_numbers != [1, 2, 3, 4, 5]:
|
||||
raise ValueError("benchmark expectations must be ordered 1..5")
|
||||
return self
|
||||
|
||||
|
||||
class TrajectoryBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.outcome-trajectory-benchmark.v1"] = (
|
||||
"vignette.outcome-trajectory-benchmark.v1"
|
||||
)
|
||||
expected_arc: SyntheticExpectedArc
|
||||
cases: tuple[TrajectoryBenchmarkCase, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OBSERVATION_STATUSES",
|
||||
"OUTCOME_AXES",
|
||||
"RELATIONSHIP_EVENT_TYPES",
|
||||
"TRAJECTORY_STATUSES",
|
||||
"AxisTrajectoryAssessment",
|
||||
"LongitudinalOutcomeAssessment",
|
||||
"LongitudinalOutcomeInput",
|
||||
"ObservedSessionOutcome",
|
||||
"OutcomeAxis",
|
||||
"OutcomeAxisObservation",
|
||||
"RelationshipMemoryEvent",
|
||||
"RelationshipMemoryProjection",
|
||||
"RoleSafeTrajectoryReadModel",
|
||||
"SafetySignalReference",
|
||||
"SessionTrajectoryAssessment",
|
||||
"SyntheticExpectedArc",
|
||||
"SyntheticExpectedDistribution",
|
||||
"TrajectoryBenchmarkCase",
|
||||
"TrajectoryBenchmarkExpectation",
|
||||
"TrajectoryBenchmarkPack",
|
||||
"TrajectoryStatus",
|
||||
]
|
||||
434
apps/api/app/contracts/rupture_repair.py
Normal file
434
apps/api/app/contracts/rupture_repair.py
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
"""G3 Rupture & Repair Lab의 버전 고정 순수 도메인 계약.
|
||||
|
||||
균열 유형, 탐지 근거, 상담자 복구 행동, 내담자 후속 반응을 서로 분리한다.
|
||||
단일 총점으로 합산하지 않으며 safety 신호는 균열 판정의 입력 feature가 아니다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
RUPTURE_TYPES = (
|
||||
"withdrawal",
|
||||
"confrontation",
|
||||
"goal_mismatch",
|
||||
"task_mismatch",
|
||||
"empathic_miss",
|
||||
"cultural_miss",
|
||||
"boundary_tension",
|
||||
"premature_advice",
|
||||
"over_disclosure",
|
||||
)
|
||||
RuptureType = Literal[
|
||||
"withdrawal",
|
||||
"confrontation",
|
||||
"goal_mismatch",
|
||||
"task_mismatch",
|
||||
"empathic_miss",
|
||||
"cultural_miss",
|
||||
"boundary_tension",
|
||||
"premature_advice",
|
||||
"over_disclosure",
|
||||
]
|
||||
|
||||
RUPTURE_LIFECYCLE_STATES = (
|
||||
"onset",
|
||||
"recognized",
|
||||
"repair_attempted",
|
||||
"missed",
|
||||
"partial",
|
||||
"resolved",
|
||||
)
|
||||
RuptureLifecycleState = Literal[
|
||||
"onset",
|
||||
"recognized",
|
||||
"repair_attempted",
|
||||
"missed",
|
||||
"partial",
|
||||
"resolved",
|
||||
]
|
||||
FinalRuptureStatus = Literal[
|
||||
"missed",
|
||||
"partial",
|
||||
"resolved",
|
||||
"not_applicable",
|
||||
"insufficient_evidence",
|
||||
]
|
||||
|
||||
REPAIR_BEHAVIORS = (
|
||||
"noticing",
|
||||
"naming",
|
||||
"curiosity",
|
||||
"impact_acknowledgement",
|
||||
"goal_reagreement",
|
||||
"task_reagreement",
|
||||
"follow_up_check",
|
||||
)
|
||||
RepairBehavior = Literal[
|
||||
"noticing",
|
||||
"naming",
|
||||
"curiosity",
|
||||
"impact_acknowledgement",
|
||||
"goal_reagreement",
|
||||
"task_reagreement",
|
||||
"follow_up_check",
|
||||
]
|
||||
|
||||
ClientRepairResponse = Literal[
|
||||
"rejecting",
|
||||
"withdrawn",
|
||||
"compliance_only",
|
||||
"mixed",
|
||||
"engaged",
|
||||
"explicit_alignment",
|
||||
]
|
||||
SignalLoop = Literal["fast", "deep"]
|
||||
SignalStatus = Literal["detected", "not_detected", "error"]
|
||||
ReconciliationDisposition = Literal[
|
||||
"not_applicable",
|
||||
"confirmed",
|
||||
"superseded_resolved",
|
||||
"superseded_partial",
|
||||
"dismissed",
|
||||
]
|
||||
|
||||
|
||||
class RuptureEvidenceRef(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
ref_id: str = Field(min_length=1, max_length=180)
|
||||
turn_index: int = Field(ge=0)
|
||||
speaker: Literal["learner", "client", "observer", "runtime"]
|
||||
|
||||
|
||||
class SafetySignalReference(BaseModel):
|
||||
"""균열/복구 판정과 합산하지 않는 기존 safety 원장 포인터."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
safety_event_id: str = Field(min_length=1, max_length=180)
|
||||
risk_level: Literal["low", "moderate", "high", "imminent"]
|
||||
escalated: bool
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class RuptureDetectionSignal(BaseModel):
|
||||
"""fast/deep observer가 만든 독립 탐지 가설."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
signal_id: str = Field(min_length=1, max_length=180)
|
||||
loop: SignalLoop
|
||||
status: SignalStatus
|
||||
rupture_type: RuptureType | None = None
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
observed_at_turn: int = Field(ge=0)
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective
|
||||
model_run_id: UUID | None = None
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_detection_truth(self) -> "RuptureDetectionSignal":
|
||||
allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]
|
||||
if self.perspective not in allowed:
|
||||
raise ValueError("rupture detection mixes source and perspective layers")
|
||||
if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None:
|
||||
raise ValueError("model/agent rupture detection requires model_run_id")
|
||||
if self.status == "detected":
|
||||
if self.rupture_type is None or self.confidence is None or not self.evidence_refs:
|
||||
raise ValueError("detected rupture requires type, confidence, and evidence")
|
||||
if self.error_code:
|
||||
raise ValueError("detected rupture cannot carry error_code")
|
||||
elif self.status == "not_detected":
|
||||
if self.rupture_type is not None or self.confidence is not None:
|
||||
raise ValueError("not_detected rupture must remain type/scoreless")
|
||||
if self.error_code:
|
||||
raise ValueError("not_detected rupture cannot carry error_code")
|
||||
else:
|
||||
if self.rupture_type is not None or self.confidence is not None:
|
||||
raise ValueError("error rupture signal must remain type/scoreless")
|
||||
if not self.error_code:
|
||||
raise ValueError("error rupture signal requires error_code")
|
||||
if len({item.ref_id for item in self.evidence_refs}) != len(self.evidence_refs):
|
||||
raise ValueError("rupture evidence refs must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class FastLoopWarning(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
warning_id: str = Field(min_length=1, max_length=180)
|
||||
signal_id: str = Field(min_length=1, max_length=180)
|
||||
provisional_status: Literal["missed", "partial"]
|
||||
emitted_at_turn: int = Field(ge=0)
|
||||
|
||||
|
||||
class RepairAttemptObservation(BaseModel):
|
||||
"""문구가 아니라 관찰된 복구 행동과 내담자 후속 반응을 보존한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str = Field(min_length=1, max_length=180)
|
||||
turn_index: int = Field(ge=0)
|
||||
behaviors: tuple[RepairBehavior, ...] = ()
|
||||
client_response: ClientRepairResponse
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...] = Field(min_length=1)
|
||||
response_evidence_refs: tuple[RuptureEvidenceRef, ...] = Field(min_length=1)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
utterance_template_id: str | None = Field(default=None, max_length=180)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_attempt_evidence(self) -> "RepairAttemptObservation":
|
||||
if len(set(self.behaviors)) != len(self.behaviors):
|
||||
raise ValueError("repair behaviors must be unique")
|
||||
refs = (*self.evidence_refs, *self.response_evidence_refs)
|
||||
if len({item.ref_id for item in refs}) != len(refs):
|
||||
raise ValueError("repair attempt evidence refs must be unique")
|
||||
if any(item.turn_index > self.turn_index for item in self.evidence_refs):
|
||||
raise ValueError("repair behavior evidence cannot follow the attempt")
|
||||
if any(item.turn_index <= self.turn_index for item in self.response_evidence_refs):
|
||||
raise ValueError("client response evidence must follow the attempt")
|
||||
return self
|
||||
|
||||
|
||||
class RuptureEpisodeInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
episode_id: str = Field(pattern=r"^oas-g3-episode-[a-z0-9-]+$")
|
||||
detection_signals: tuple[RuptureDetectionSignal, ...] = Field(min_length=1)
|
||||
fast_warning: FastLoopWarning | None = None
|
||||
recognized_at_turn: int | None = Field(default=None, ge=0)
|
||||
recognition_evidence_refs: tuple[RuptureEvidenceRef, ...] = ()
|
||||
repair_attempts: tuple[RepairAttemptObservation, ...] = ()
|
||||
safety_signals: tuple[SafetySignalReference, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_episode(self) -> "RuptureEpisodeInput":
|
||||
signal_ids = [item.signal_id for item in self.detection_signals]
|
||||
if len(set(signal_ids)) != len(signal_ids):
|
||||
raise ValueError("rupture signal ids must be unique")
|
||||
if self.fast_warning:
|
||||
linked = next(
|
||||
(item for item in self.detection_signals if item.signal_id == self.fast_warning.signal_id),
|
||||
None,
|
||||
)
|
||||
if linked is None or linked.loop != "fast" or linked.status != "detected":
|
||||
raise ValueError("fast warning must reference a detected fast-loop signal")
|
||||
if self.fast_warning.emitted_at_turn < linked.observed_at_turn:
|
||||
raise ValueError("fast warning cannot precede its signal")
|
||||
if self.recognized_at_turn is None and self.recognition_evidence_refs:
|
||||
raise ValueError("recognition evidence requires recognized_at_turn")
|
||||
if self.recognized_at_turn is not None and not self.recognition_evidence_refs:
|
||||
raise ValueError("recognized rupture requires recognition evidence")
|
||||
attempts = [item.attempt_id for item in self.repair_attempts]
|
||||
if len(set(attempts)) != len(attempts):
|
||||
raise ValueError("repair attempt ids must be unique")
|
||||
attempt_turns = [item.turn_index for item in self.repair_attempts]
|
||||
if attempt_turns != sorted(attempt_turns):
|
||||
raise ValueError("repair attempts must be ordered by turn")
|
||||
if self.repair_attempts and self.recognized_at_turn is None:
|
||||
raise ValueError("repair attempts require rupture recognition")
|
||||
if self.recognized_at_turn is not None and any(
|
||||
item.turn_index < self.recognized_at_turn for item in self.repair_attempts
|
||||
):
|
||||
raise ValueError("repair attempt cannot precede rupture recognition")
|
||||
return self
|
||||
|
||||
|
||||
class RuptureLedgerEntry(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
sequence_no: int = Field(ge=1)
|
||||
event_name: Literal[
|
||||
"rupture.detected",
|
||||
"rupture.recognized",
|
||||
"rupture.missed",
|
||||
"repair.attempted",
|
||||
"repair.partial",
|
||||
"repair.resolved",
|
||||
"repair.missed",
|
||||
"rupture.reconciled",
|
||||
]
|
||||
from_state: RuptureLifecycleState | None = None
|
||||
to_state: RuptureLifecycleState
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
source_ref_id: str = Field(min_length=1, max_length=180)
|
||||
reconciles_event_id: str | None = Field(default=None, max_length=180)
|
||||
|
||||
|
||||
class RepairAttemptAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str
|
||||
outcome: Literal["missed", "partial", "resolved"]
|
||||
observed_behaviors: tuple[RepairBehavior, ...]
|
||||
required_behaviors: tuple[RepairBehavior, ...]
|
||||
missing_behaviors: tuple[RepairBehavior, ...]
|
||||
client_response: ClientRepairResponse
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class FastDeepReconciliation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
warning_id: str | None = None
|
||||
disposition: ReconciliationDisposition
|
||||
provisional_status: Literal["missed", "partial"] | None = None
|
||||
deep_status: FinalRuptureStatus
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...] = ()
|
||||
reason: str = Field(min_length=1, max_length=500)
|
||||
|
||||
|
||||
class RuptureEpisodeAssessment(BaseModel):
|
||||
"""점수 합산 없이 유형·상태·근거·반증을 나란히 반환한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.rupture-repair-assessment.v1"] = (
|
||||
"vignette.rupture-repair-assessment.v1"
|
||||
)
|
||||
episode_id: str
|
||||
assessment_status: Literal["ready", "error"] = "ready"
|
||||
detected: bool
|
||||
rupture_type: RuptureType | None
|
||||
final_status: FinalRuptureStatus
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[RuptureEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
repair_attempts: tuple[RepairAttemptAssessment, ...]
|
||||
ledger: tuple[RuptureLedgerEntry, ...]
|
||||
reconciliation: FastDeepReconciliation
|
||||
safety_signals: tuple[SafetySignalReference, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_negative_assessment_scoreless(self) -> "RuptureEpisodeAssessment":
|
||||
if self.assessment_status == "error":
|
||||
if self.detected or self.rupture_type is not None or self.confidence is not None:
|
||||
raise ValueError("error assessment must remain detection/type/scoreless")
|
||||
if self.final_status != "insufficient_evidence":
|
||||
raise ValueError("error assessment requires insufficient_evidence status")
|
||||
if self.repair_attempts or self.ledger:
|
||||
raise ValueError("error assessment cannot fabricate lifecycle events")
|
||||
return self
|
||||
if not self.detected:
|
||||
if self.rupture_type is not None or self.confidence is not None:
|
||||
raise ValueError("not-detected assessment must remain type/scoreless")
|
||||
if self.final_status != "not_applicable":
|
||||
raise ValueError("not-detected assessment has no repair status")
|
||||
if self.repair_attempts or self.ledger:
|
||||
raise ValueError("not-detected assessment cannot fabricate lifecycle events")
|
||||
elif self.rupture_type is None or self.confidence is None:
|
||||
raise ValueError("detected assessment requires type and confidence")
|
||||
return self
|
||||
|
||||
|
||||
class RuptureBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
detected: bool
|
||||
rupture_type: RuptureType | None
|
||||
final_status: FinalRuptureStatus
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_expectation_consistent(self) -> "RuptureBenchmarkExpectation":
|
||||
if self.detected and self.rupture_type is None:
|
||||
raise ValueError("detected benchmark expectation requires rupture_type")
|
||||
if not self.detected and (
|
||||
self.rupture_type is not None or self.final_status != "not_applicable"
|
||||
):
|
||||
raise ValueError("negative benchmark expectation must be type/repair scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class RuptureBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g3-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
episode: RuptureEpisodeInput
|
||||
expected: RuptureBenchmarkExpectation
|
||||
critical: bool = False
|
||||
tags: tuple[str, ...] = ()
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class RuptureBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.rupture-repair-benchmark.v1"] = (
|
||||
"vignette.rupture-repair-benchmark.v1"
|
||||
)
|
||||
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
cases: tuple[RuptureBenchmarkCase, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_adversarial_coverage(self) -> "RuptureBenchmarkPack":
|
||||
case_ids = [item.case_id for item in self.cases]
|
||||
if len(set(case_ids)) != len(case_ids):
|
||||
raise ValueError("rupture benchmark case ids must be unique")
|
||||
covered = {
|
||||
item.expected.rupture_type for item in self.cases if item.expected.detected
|
||||
}
|
||||
if covered != set(RUPTURE_TYPES):
|
||||
raise ValueError("rupture benchmark must cover every rupture type")
|
||||
if not any("judge_gaming" in item.tags for item in self.cases):
|
||||
raise ValueError("rupture benchmark requires judge_gaming cases")
|
||||
if not any("memorized_phrase_trap" in item.tags for item in self.cases):
|
||||
raise ValueError("rupture benchmark requires memorized phrase variants")
|
||||
template_expectations: dict[str, set[FinalRuptureStatus]] = {}
|
||||
for case in self.cases:
|
||||
for attempt in case.episode.repair_attempts:
|
||||
if attempt.utterance_template_id:
|
||||
template_expectations.setdefault(attempt.utterance_template_id, set()).add(
|
||||
case.expected.final_status
|
||||
)
|
||||
if not any(len(statuses) > 1 for statuses in template_expectations.values()):
|
||||
raise ValueError("a memorized template must have different contextual outcomes")
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ClientRepairResponse",
|
||||
"FastDeepReconciliation",
|
||||
"FastLoopWarning",
|
||||
"FinalRuptureStatus",
|
||||
"REPAIR_BEHAVIORS",
|
||||
"RUPTURE_LIFECYCLE_STATES",
|
||||
"RUPTURE_TYPES",
|
||||
"RepairAttemptAssessment",
|
||||
"RepairAttemptObservation",
|
||||
"RepairBehavior",
|
||||
"RuptureBenchmarkCase",
|
||||
"RuptureBenchmarkExpectation",
|
||||
"RuptureBenchmarkPack",
|
||||
"RuptureDetectionSignal",
|
||||
"RuptureEpisodeAssessment",
|
||||
"RuptureEpisodeInput",
|
||||
"RuptureEvidenceRef",
|
||||
"RuptureLedgerEntry",
|
||||
"RuptureLifecycleState",
|
||||
"RuptureType",
|
||||
"SafetySignalReference",
|
||||
]
|
||||
275
apps/api/app/contracts/supervision_research.py
Normal file
275
apps/api/app/contracts/supervision_research.py
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
"""G6 Supervision & Research OS의 역할 안전 운영·연구 계약."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
AttentionSignalType = Literal[
|
||||
"deterioration",
|
||||
"unresolved_rupture",
|
||||
"safety_boundary",
|
||||
"persistent_overconfidence",
|
||||
"growth_stagnation",
|
||||
"transfer_failure",
|
||||
]
|
||||
SignalSeverity = Literal["high", "moderate", "low"]
|
||||
SignalState = Literal["active", "monitoring", "resolved", "insufficient_evidence"]
|
||||
EvidenceLedger = Literal[
|
||||
"measurement_event",
|
||||
"outcome_trajectory_revision",
|
||||
"rupture_observation_event",
|
||||
"rupture_reconciliation_revision",
|
||||
"safety_event",
|
||||
"calibration_assessment",
|
||||
"transfer_assessment",
|
||||
"practice_attempt",
|
||||
]
|
||||
ManifestDomain = Literal["alliance", "rupture", "transfer", "calibration"]
|
||||
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
||||
|
||||
|
||||
class LedgerEvidencePointer(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
ledger: EvidenceLedger
|
||||
event_id: str = Field(min_length=1, max_length=180)
|
||||
session_id: str | None = Field(default=None, max_length=180)
|
||||
route_hint: str = Field(pattern=r"^/[a-zA-Z0-9_{}?&=./-]+$")
|
||||
|
||||
|
||||
class LearnerAttentionSignal(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
signal_id: str = Field(pattern=r"^oas-g6-signal-[a-z0-9-]+$")
|
||||
learner_ref: str = Field(pattern=r"^learner-[a-z0-9-]+$")
|
||||
signal_type: AttentionSignalType
|
||||
severity: SignalSeverity
|
||||
state: SignalState
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
observed_sequence: int = Field(ge=1)
|
||||
evidence: tuple[LedgerEvidencePointer, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_active_evidence(self) -> "LearnerAttentionSignal":
|
||||
if self.state == "insufficient_evidence":
|
||||
if self.evidence or self.uncertainty != 1.0:
|
||||
raise ValueError(
|
||||
"insufficient attention signal must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence:
|
||||
raise ValueError("classified attention signal requires ledger evidence")
|
||||
if self.state == "resolved" and not self.counterevidence:
|
||||
raise ValueError("resolved attention signal requires resolution evidence")
|
||||
return self
|
||||
|
||||
|
||||
class AttentionQueueReason(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
signal_id: str
|
||||
signal_type: AttentionSignalType
|
||||
severity: SignalSeverity
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class AttentionQueueItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
learner_ref: str
|
||||
queue_position: int = Field(ge=1)
|
||||
primary_signal: AttentionSignalType
|
||||
oldest_active_sequence: int = Field(ge=1)
|
||||
reasons: tuple[AttentionQueueReason, ...] = Field(min_length=1)
|
||||
drilldown_routes: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_direct_drilldown(self) -> "AttentionQueueItem":
|
||||
if len(self.drilldown_routes) > 3:
|
||||
raise ValueError(
|
||||
"attention item must reach evidence within three drilldowns"
|
||||
)
|
||||
if len(set(self.drilldown_routes)) != len(self.drilldown_routes):
|
||||
raise ValueError("attention drilldown routes must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TeacherAiDisagreement(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
disagreement_id: str = Field(pattern=r"^oas-g6-disagreement-[a-z0-9-]+$")
|
||||
case_ref: str = Field(min_length=1, max_length=180)
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
ai_label: str = Field(min_length=1, max_length=120)
|
||||
teacher_label: str = Field(min_length=1, max_length=120)
|
||||
ai_model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
instrument_id: str = Field(min_length=1, max_length=180)
|
||||
instrument_version: str = Field(min_length=1, max_length=80)
|
||||
ai_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
teacher_correction_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
correction_reason_code: str = Field(min_length=1, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_actual_disagreement(self) -> "TeacherAiDisagreement":
|
||||
if self.ai_label == self.teacher_label:
|
||||
raise ValueError("calibration disagreement requires different labels")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationDatasetRow(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
row_id: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
disagreement_id: str
|
||||
case_ref: str
|
||||
competency_id: str
|
||||
ai_label: str
|
||||
teacher_label: str
|
||||
ai_model: str
|
||||
prompt_version: str
|
||||
instrument_id: str
|
||||
instrument_version: str
|
||||
evidence_event_ids: tuple[str, ...] = Field(min_length=2)
|
||||
correction_reason_code: str
|
||||
raw_transcript_included: Literal[False] = False
|
||||
|
||||
|
||||
class VersionedEvaluationObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_ref: str = Field(min_length=1, max_length=180)
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
||||
gold_label: str = Field(min_length=1, max_length=120)
|
||||
predicted_label: str = Field(min_length=1, max_length=120)
|
||||
evidence_event_id: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class EvaluationVersionBatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
batch_id: str = Field(pattern=r"^oas-g6-batch-[a-z0-9-]+$")
|
||||
model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
instrument_id: str = Field(min_length=1, max_length=180)
|
||||
instrument_version: str = Field(min_length=1, max_length=80)
|
||||
observations: tuple[VersionedEvaluationObservation, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_case_competency(self) -> "EvaluationVersionBatch":
|
||||
keys = [(item.case_ref, item.competency_id) for item in self.observations]
|
||||
if len(set(keys)) != len(keys):
|
||||
raise ValueError("version batch case/competency keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class SubgroupVersionMetric(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
subgroup: str
|
||||
matched_count: int = Field(ge=0)
|
||||
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
|
||||
|
||||
class EvaluationVersionDriftReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
baseline_batch_id: str
|
||||
candidate_batch_id: str
|
||||
matched_count: int = Field(ge=0)
|
||||
status: DriftStatus
|
||||
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
disagreement_case_refs: tuple[str, ...]
|
||||
subgroup_metrics: tuple[SubgroupVersionMetric, ...]
|
||||
alerts: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_underpowered_report(self) -> "EvaluationVersionDriftReport":
|
||||
if self.status == "insufficient_evidence" and any(
|
||||
item is not None
|
||||
for item in (
|
||||
self.baseline_accuracy,
|
||||
self.candidate_accuracy,
|
||||
self.accuracy_delta,
|
||||
)
|
||||
):
|
||||
raise ValueError("underpowered version comparison must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class Phase3EvidenceArtifact(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
domain: ManifestDomain
|
||||
artifact_id: str = Field(min_length=1, max_length=180)
|
||||
schema_version: str = Field(min_length=1, max_length=120)
|
||||
content_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
record_count: int = Field(ge=1)
|
||||
provenance_uri: str = Field(pattern=r"^(repo|db|audit)://[a-zA-Z0-9_./:-]+$")
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
|
||||
|
||||
class Phase3OutcomeEvidenceManifest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.phase3-outcome-evidence-manifest.v1"]
|
||||
artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_unique_domains(self) -> "Phase3OutcomeEvidenceManifest":
|
||||
domains = [item.domain for item in self.artifacts]
|
||||
if set(domains) != {"alliance", "rupture", "transfer", "calibration"}:
|
||||
raise ValueError(
|
||||
"Phase 3 outcome manifest requires all four evidence domains"
|
||||
)
|
||||
if len(domains) != len(set(domains)):
|
||||
raise ValueError("Phase 3 outcome manifest domains must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class SupervisionResearchBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.supervision-research-benchmark.v1"]
|
||||
version: Literal["1.0.0"]
|
||||
data_classification: Literal["synthetic_educational"]
|
||||
clinical_claim_allowed: Literal[False]
|
||||
attention_signals: tuple[LearnerAttentionSignal, ...] = Field(min_length=6)
|
||||
expected_queue_order: tuple[str, ...] = Field(min_length=1)
|
||||
disagreements: tuple[TeacherAiDisagreement, ...] = Field(min_length=1)
|
||||
baseline_batch: EvaluationVersionBatch
|
||||
candidate_batch: EvaluationVersionBatch
|
||||
expected_drift_status: DriftStatus
|
||||
phase3_artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttentionQueueItem",
|
||||
"AttentionQueueReason",
|
||||
"AttentionSignalType",
|
||||
"CalibrationDatasetRow",
|
||||
"DriftStatus",
|
||||
"EvaluationVersionBatch",
|
||||
"EvaluationVersionDriftReport",
|
||||
"EvidenceLedger",
|
||||
"LedgerEvidencePointer",
|
||||
"LearnerAttentionSignal",
|
||||
"ManifestDomain",
|
||||
"Phase3EvidenceArtifact",
|
||||
"Phase3OutcomeEvidenceManifest",
|
||||
"SignalSeverity",
|
||||
"SignalState",
|
||||
"SubgroupVersionMetric",
|
||||
"SupervisionResearchBenchmarkPack",
|
||||
"TeacherAiDisagreement",
|
||||
"VersionedEvaluationObservation",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue