G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
275
apps/api/app/contracts/supervision_research.py
Normal file
275
apps/api/app/contracts/supervision_research.py
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
"""G6 Supervision & Research OS의 역할 안전 운영·연구 계약."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
AttentionSignalType = Literal[
|
||||
"deterioration",
|
||||
"unresolved_rupture",
|
||||
"safety_boundary",
|
||||
"persistent_overconfidence",
|
||||
"growth_stagnation",
|
||||
"transfer_failure",
|
||||
]
|
||||
SignalSeverity = Literal["high", "moderate", "low"]
|
||||
SignalState = Literal["active", "monitoring", "resolved", "insufficient_evidence"]
|
||||
EvidenceLedger = Literal[
|
||||
"measurement_event",
|
||||
"outcome_trajectory_revision",
|
||||
"rupture_observation_event",
|
||||
"rupture_reconciliation_revision",
|
||||
"safety_event",
|
||||
"calibration_assessment",
|
||||
"transfer_assessment",
|
||||
"practice_attempt",
|
||||
]
|
||||
ManifestDomain = Literal["alliance", "rupture", "transfer", "calibration"]
|
||||
DriftStatus = Literal["stable", "drift_flagged", "insufficient_evidence"]
|
||||
|
||||
|
||||
class LedgerEvidencePointer(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
ledger: EvidenceLedger
|
||||
event_id: str = Field(min_length=1, max_length=180)
|
||||
session_id: str | None = Field(default=None, max_length=180)
|
||||
route_hint: str = Field(pattern=r"^/[a-zA-Z0-9_{}?&=./-]+$")
|
||||
|
||||
|
||||
class LearnerAttentionSignal(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
signal_id: str = Field(pattern=r"^oas-g6-signal-[a-z0-9-]+$")
|
||||
learner_ref: str = Field(pattern=r"^learner-[a-z0-9-]+$")
|
||||
signal_type: AttentionSignalType
|
||||
severity: SignalSeverity
|
||||
state: SignalState
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
observed_sequence: int = Field(ge=1)
|
||||
evidence: tuple[LedgerEvidencePointer, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_active_evidence(self) -> "LearnerAttentionSignal":
|
||||
if self.state == "insufficient_evidence":
|
||||
if self.evidence or self.uncertainty != 1.0:
|
||||
raise ValueError(
|
||||
"insufficient attention signal must remain evidence-free"
|
||||
)
|
||||
elif not self.evidence:
|
||||
raise ValueError("classified attention signal requires ledger evidence")
|
||||
if self.state == "resolved" and not self.counterevidence:
|
||||
raise ValueError("resolved attention signal requires resolution evidence")
|
||||
return self
|
||||
|
||||
|
||||
class AttentionQueueReason(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
signal_id: str
|
||||
signal_type: AttentionSignalType
|
||||
severity: SignalSeverity
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class AttentionQueueItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
learner_ref: str
|
||||
queue_position: int = Field(ge=1)
|
||||
primary_signal: AttentionSignalType
|
||||
oldest_active_sequence: int = Field(ge=1)
|
||||
reasons: tuple[AttentionQueueReason, ...] = Field(min_length=1)
|
||||
drilldown_routes: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_direct_drilldown(self) -> "AttentionQueueItem":
|
||||
if len(self.drilldown_routes) > 3:
|
||||
raise ValueError(
|
||||
"attention item must reach evidence within three drilldowns"
|
||||
)
|
||||
if len(set(self.drilldown_routes)) != len(self.drilldown_routes):
|
||||
raise ValueError("attention drilldown routes must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class TeacherAiDisagreement(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
disagreement_id: str = Field(pattern=r"^oas-g6-disagreement-[a-z0-9-]+$")
|
||||
case_ref: str = Field(min_length=1, max_length=180)
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
ai_label: str = Field(min_length=1, max_length=120)
|
||||
teacher_label: str = Field(min_length=1, max_length=120)
|
||||
ai_model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
instrument_id: str = Field(min_length=1, max_length=180)
|
||||
instrument_version: str = Field(min_length=1, max_length=80)
|
||||
ai_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
teacher_correction_evidence: tuple[LedgerEvidencePointer, ...] = Field(min_length=1)
|
||||
correction_reason_code: str = Field(min_length=1, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_actual_disagreement(self) -> "TeacherAiDisagreement":
|
||||
if self.ai_label == self.teacher_label:
|
||||
raise ValueError("calibration disagreement requires different labels")
|
||||
return self
|
||||
|
||||
|
||||
class CalibrationDatasetRow(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
row_id: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
disagreement_id: str
|
||||
case_ref: str
|
||||
competency_id: str
|
||||
ai_label: str
|
||||
teacher_label: str
|
||||
ai_model: str
|
||||
prompt_version: str
|
||||
instrument_id: str
|
||||
instrument_version: str
|
||||
evidence_event_ids: tuple[str, ...] = Field(min_length=2)
|
||||
correction_reason_code: str
|
||||
raw_transcript_included: Literal[False] = False
|
||||
|
||||
|
||||
class VersionedEvaluationObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_ref: str = Field(min_length=1, max_length=180)
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
synthetic_subgroup: str = Field(pattern=r"^synthetic-[a-z0-9-]+$")
|
||||
gold_label: str = Field(min_length=1, max_length=120)
|
||||
predicted_label: str = Field(min_length=1, max_length=120)
|
||||
evidence_event_id: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class EvaluationVersionBatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
batch_id: str = Field(pattern=r"^oas-g6-batch-[a-z0-9-]+$")
|
||||
model: str = Field(min_length=1, max_length=180)
|
||||
prompt_version: str = Field(min_length=1, max_length=80)
|
||||
instrument_id: str = Field(min_length=1, max_length=180)
|
||||
instrument_version: str = Field(min_length=1, max_length=80)
|
||||
observations: tuple[VersionedEvaluationObservation, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_unique_case_competency(self) -> "EvaluationVersionBatch":
|
||||
keys = [(item.case_ref, item.competency_id) for item in self.observations]
|
||||
if len(set(keys)) != len(keys):
|
||||
raise ValueError("version batch case/competency keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class SubgroupVersionMetric(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
subgroup: str
|
||||
matched_count: int = Field(ge=0)
|
||||
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
|
||||
|
||||
class EvaluationVersionDriftReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
baseline_batch_id: str
|
||||
candidate_batch_id: str
|
||||
matched_count: int = Field(ge=0)
|
||||
status: DriftStatus
|
||||
baseline_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
candidate_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
accuracy_delta: float | None = Field(default=None, ge=-1.0, le=1.0)
|
||||
disagreement_case_refs: tuple[str, ...]
|
||||
subgroup_metrics: tuple[SubgroupVersionMetric, ...]
|
||||
alerts: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_underpowered_report(self) -> "EvaluationVersionDriftReport":
|
||||
if self.status == "insufficient_evidence" and any(
|
||||
item is not None
|
||||
for item in (
|
||||
self.baseline_accuracy,
|
||||
self.candidate_accuracy,
|
||||
self.accuracy_delta,
|
||||
)
|
||||
):
|
||||
raise ValueError("underpowered version comparison must remain scoreless")
|
||||
return self
|
||||
|
||||
|
||||
class Phase3EvidenceArtifact(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
domain: ManifestDomain
|
||||
artifact_id: str = Field(min_length=1, max_length=180)
|
||||
schema_version: str = Field(min_length=1, max_length=120)
|
||||
content_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
record_count: int = Field(ge=1)
|
||||
provenance_uri: str = Field(pattern=r"^(repo|db|audit)://[a-zA-Z0-9_./:-]+$")
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
|
||||
|
||||
class Phase3OutcomeEvidenceManifest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.phase3-outcome-evidence-manifest.v1"]
|
||||
artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_unique_domains(self) -> "Phase3OutcomeEvidenceManifest":
|
||||
domains = [item.domain for item in self.artifacts]
|
||||
if set(domains) != {"alliance", "rupture", "transfer", "calibration"}:
|
||||
raise ValueError(
|
||||
"Phase 3 outcome manifest requires all four evidence domains"
|
||||
)
|
||||
if len(domains) != len(set(domains)):
|
||||
raise ValueError("Phase 3 outcome manifest domains must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class SupervisionResearchBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.supervision-research-benchmark.v1"]
|
||||
version: Literal["1.0.0"]
|
||||
data_classification: Literal["synthetic_educational"]
|
||||
clinical_claim_allowed: Literal[False]
|
||||
attention_signals: tuple[LearnerAttentionSignal, ...] = Field(min_length=6)
|
||||
expected_queue_order: tuple[str, ...] = Field(min_length=1)
|
||||
disagreements: tuple[TeacherAiDisagreement, ...] = Field(min_length=1)
|
||||
baseline_batch: EvaluationVersionBatch
|
||||
candidate_batch: EvaluationVersionBatch
|
||||
expected_drift_status: DriftStatus
|
||||
phase3_artifacts: tuple[Phase3EvidenceArtifact, ...] = Field(min_length=4)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttentionQueueItem",
|
||||
"AttentionQueueReason",
|
||||
"AttentionSignalType",
|
||||
"CalibrationDatasetRow",
|
||||
"DriftStatus",
|
||||
"EvaluationVersionBatch",
|
||||
"EvaluationVersionDriftReport",
|
||||
"EvidenceLedger",
|
||||
"LedgerEvidencePointer",
|
||||
"LearnerAttentionSignal",
|
||||
"ManifestDomain",
|
||||
"Phase3EvidenceArtifact",
|
||||
"Phase3OutcomeEvidenceManifest",
|
||||
"SignalSeverity",
|
||||
"SignalState",
|
||||
"SubgroupVersionMetric",
|
||||
"SupervisionResearchBenchmarkPack",
|
||||
"TeacherAiDisagreement",
|
||||
"VersionedEvaluationObservation",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue