16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
275 lines
10 KiB
Python
275 lines
10 KiB
Python
"""G6 Supervision & Research OS의 역할 안전 운영·연구 계약."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
|
|
|
AttentionSignalType = Literal[
|
|
"deterioration",
|
|
"unresolved_rupture",
|
|
"safety_boundary",
|
|
"persistent_overconfidence",
|
|
"growth_stagnation",
|
|
"transfer_failure",
|
|
]
|
|
SignalSeverity = Literal["high", "moderate", "low"]
|
|
SignalState = Literal["active", "monitoring", "resolved", "insufficient_evidence"]
|
|
EvidenceLedger = Literal[
|
|
"measurement_event",
|
|
"outcome_trajectory_revision",
|
|
"rupture_observation_event",
|
|
"rupture_reconciliation_revision",
|
|
"safety_event",
|
|
"calibration_assessment",
|
|
"transfer_assessment",
|
|
"practice_attempt",
|
|
]
|
|
ManifestDomain = Literal["alliance", "rupture", "transfer", "calibration"]
|
|
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
|
|
|
|
|
class LedgerEvidencePointer(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
ledger: EvidenceLedger
|
|
event_id: str = Field(min_length=1, max_length=180)
|
|
session_id: str | None = Field(default=None, max_length=180)
|
|
route_hint: str = Field(pattern=r"^/[a-zA-Z0-9_{}?&=./-]+$")
|
|
|
|
|
|
class LearnerAttentionSignal(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
signal_id: str = Field(pattern=r"^oas-g6-signal-[a-z0-9-]+$")
|
|
learner_ref: str = Field(pattern=r"^learner-[a-z0-9-]+$")
|
|
signal_type: AttentionSignalType
|
|
severity: SignalSeverity
|
|
state: SignalState
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
observed_sequence: int = Field(ge=1)
|
|
evidence: tuple[LedgerEvidencePointer, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
|
|
@model_validator(mode="after")
|
|
def require_active_evidence(self) -> "LearnerAttentionSignal":
|
|
if self.state == "insufficient_evidence":
|
|
if self.evidence or self.uncertainty != 1.0:
|
|
raise ValueError(
|
|
"insufficient attention signal must remain evidence-free"
|
|
)
|
|
elif not self.evidence:
|
|
raise ValueError("classified attention signal requires ledger evidence")
|
|
if self.state == "resolved" and not self.counterevidence:
|
|
raise ValueError("resolved attention signal requires resolution evidence")
|
|
return self
|
|
|
|
|
|
class AttentionQueueReason(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
signal_id: str
|
|
signal_type: AttentionSignalType
|
|
severity: SignalSeverity
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
|
|
|
|
|
class AttentionQueueItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
learner_ref: str
|
|
queue_position: int = Field(ge=1)
|
|
primary_signal: AttentionSignalType
|
|
oldest_active_sequence: int = Field(ge=1)
|
|
reasons: tuple[AttentionQueueReason, ...] = Field(min_length=1)
|
|
drilldown_routes: tuple[str, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_direct_drilldown(self) -> "AttentionQueueItem":
|
|
if len(self.drilldown_routes) > 3:
|
|
raise ValueError(
|
|
"attention item must reach evidence within three drilldowns"
|
|
)
|
|
if len(set(self.drilldown_routes)) != len(self.drilldown_routes):
|
|
raise ValueError("attention drilldown routes must be unique")
|
|
return self
|
|
|
|
|
|
class TeacherAiDisagreement(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
disagreement_id: str = Field(pattern=r"^oas-g6-disagreement-[a-z0-9-]+$")
|
|
case_ref: str = Field(min_length=1, max_length=180)
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
ai_label: str = Field(min_length=1, max_length=120)
|
|
teacher_label: str = Field(min_length=1, max_length=120)
|
|
ai_model: str = Field(min_length=1, max_length=180)
|
|
prompt_version: str = Field(min_length=1, max_length=80)
|
|
instrument_id: str = Field(min_length=1, max_length=180)
|
|
instrument_version: str = Field(min_length=1, max_length=80)
|
|
ai_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
|
teacher_correction_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
|
correction_reason_code: str = Field(min_length=1, max_length=120)
|
|
|
|
@model_validator(mode="after")
|
|
def require_actual_disagreement(self) -> "TeacherAiDisagreement":
|
|
if self.ai_label == self.teacher_label:
|
|
raise ValueError("calibration disagreement requires different labels")
|
|
return self
|
|
|
|
|
|
class CalibrationDatasetRow(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
row_id: str = Field(pattern=r"^[a-f0-9]{64}$")
|
|
disagreement_id: str
|
|
case_ref: str
|
|
competency_id: str
|
|
ai_label: str
|
|
teacher_label: str
|
|
ai_model: str
|
|
prompt_version: str
|
|
instrument_id: str
|
|
instrument_version: str
|
|
evidence_event_ids: tuple[str, ...] = Field(min_length=2)
|
|
correction_reason_code: str
|
|
raw_transcript_included: Literal[False] = False
|
|
|
|
|
|
class VersionedEvaluationObservation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
case_ref: str = Field(min_length=1, max_length=180)
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
|
gold_label: str = Field(min_length=1, max_length=120)
|
|
predicted_label: str = Field(min_length=1, max_length=120)
|
|
evidence_event_id: str = Field(min_length=1, max_length=180)
|
|
|
|
|
|
class EvaluationVersionBatch(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
batch_id: str = Field(pattern=r"^oas-g6-batch-[a-z0-9-]+$")
|
|
model: str = Field(min_length=1, max_length=180)
|
|
prompt_version: str = Field(min_length=1, max_length=80)
|
|
instrument_id: str = Field(min_length=1, max_length=180)
|
|
instrument_version: str = Field(min_length=1, max_length=80)
|
|
observations: tuple[VersionedEvaluationObservation, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def require_unique_case_competency(self) -> "EvaluationVersionBatch":
|
|
keys = [(item.case_ref, item.competency_id) for item in self.observations]
|
|
if len(set(keys)) != len(keys):
|
|
raise ValueError("version batch case/competency keys must be unique")
|
|
return self
|
|
|
|
|
|
class SubgroupVersionMetric(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
subgroup: str
|
|
matched_count: int = Field(ge=0)
|
|
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
|
|
|
|
|
class EvaluationVersionDriftReport(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
baseline_batch_id: str
|
|
candidate_batch_id: str
|
|
matched_count: int = Field(ge=0)
|
|
status: DriftStatus
|
|
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
|
disagreement_case_refs: tuple[str, ...]
|
|
subgroup_metrics: tuple[SubgroupVersionMetric, ...]
|
|
alerts: tuple[str, ...]
|
|
|
|
@model_validator(mode="after")
|
|
def preserve_underpowered_report(self) -> "EvaluationVersionDriftReport":
|
|
if self.status == "insufficient_evidence" and any(
|
|
item is not None
|
|
for item in (
|
|
self.baseline_accuracy,
|
|
self.candidate_accuracy,
|
|
self.accuracy_delta,
|
|
)
|
|
):
|
|
raise ValueError("underpowered version comparison must remain scoreless")
|
|
return self
|
|
|
|
|
|
class Phase3EvidenceArtifact(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
domain: ManifestDomain
|
|
artifact_id: str = Field(min_length=1, max_length=180)
|
|
schema_version: str = Field(min_length=1, max_length=120)
|
|
content_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
|
record_count: int = Field(ge=1)
|
|
provenance_uri: str = Field(pattern=r"^(repo|db|audit)://[a-zA-Z0-9_./:-]+$")
|
|
clinical_claim_allowed: Literal[False] = False
|
|
|
|
|
|
class Phase3OutcomeEvidenceManifest(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.phase3-outcome-evidence-manifest.v1"]
|
|
artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
|
|
|
@model_validator(mode="after")
|
|
def require_complete_unique_domains(self) -> "Phase3OutcomeEvidenceManifest":
|
|
domains = [item.domain for item in self.artifacts]
|
|
if set(domains) != {"alliance", "rupture", "transfer", "calibration"}:
|
|
raise ValueError(
|
|
"Phase 3 outcome manifest requires all four evidence domains"
|
|
)
|
|
if len(domains) != len(set(domains)):
|
|
raise ValueError("Phase 3 outcome manifest domains must be unique")
|
|
return self
|
|
|
|
|
|
class SupervisionResearchBenchmarkPack(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.supervision-research-benchmark.v1"]
|
|
version: Literal["1.0.0"]
|
|
data_classification: Literal["synthetic_educational"]
|
|
clinical_claim_allowed: Literal[False]
|
|
attention_signals: tuple[LearnerAttentionSignal, ...] = Field(min_length=6)
|
|
expected_queue_order: tuple[str, ...] = Field(min_length=1)
|
|
disagreements: tuple[TeacherAiDisagreement, ...] = Field(min_length=1)
|
|
baseline_batch: EvaluationVersionBatch
|
|
candidate_batch: EvaluationVersionBatch
|
|
expected_drift_status: DriftStatus
|
|
phase3_artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
|
|
|
|
|
__all__ = [
|
|
"AttentionQueueItem",
|
|
"AttentionQueueReason",
|
|
"AttentionSignalType",
|
|
"CalibrationDatasetRow",
|
|
"DriftStatus",
|
|
"EvaluationVersionBatch",
|
|
"EvaluationVersionDriftReport",
|
|
"EvidenceLedger",
|
|
"LedgerEvidencePointer",
|
|
"LearnerAttentionSignal",
|
|
"ManifestDomain",
|
|
"Phase3EvidenceArtifact",
|
|
"Phase3OutcomeEvidenceManifest",
|
|
"SignalSeverity",
|
|
"SignalState",
|
|
"SubgroupVersionMetric",
|
|
"SupervisionResearchBenchmarkPack",
|
|
"TeacherAiDisagreement",
|
|
"VersionedEvaluationObservation",
|
|
]
|