G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
639
apps/api/app/contracts/deliberate_practice.py
Normal file
639
apps/api/app/contracts/deliberate_practice.py
Normal file
|
|
@ -0,0 +1,639 @@
|
|||
"""G4 Deliberate Practice Engine의 버전 고정 순수 도메인 계약.
|
||||
|
||||
코칭 장면을 하나의 관찰 가능한 행동으로 분해한 실행형 처방, 시도 근거,
|
||||
역량 그래프와 전이 게이트를 정의한다. 단일 총점이나 보상 점수로 숙련을
|
||||
승격하지 않으며, 익숙한 장면의 성공과 미지 사례 전이를 물리적으로 구분한다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .measurement import (
|
||||
MeasurementPerspective,
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
||||
SourceKind,
|
||||
)
|
||||
|
||||
|
||||
PRACTICE_MODES = (
|
||||
"replay",
|
||||
"branch",
|
||||
"constrained_response",
|
||||
"voice_retry",
|
||||
"difficulty_ladder",
|
||||
)
|
||||
PracticeMode = Literal[
|
||||
"replay",
|
||||
"branch",
|
||||
"constrained_response",
|
||||
"voice_retry",
|
||||
"difficulty_ladder",
|
||||
]
|
||||
|
||||
COMPETENCY_BANDS = (
|
||||
"unassessed",
|
||||
"fragile",
|
||||
"developing",
|
||||
"consistent_local",
|
||||
"transfer_verified",
|
||||
)
|
||||
CompetencyBand = Literal[
|
||||
"unassessed",
|
||||
"fragile",
|
||||
"developing",
|
||||
"consistent_local",
|
||||
"transfer_verified",
|
||||
]
|
||||
|
||||
CriterionStatus = Literal["observed", "not_observed", "error"]
|
||||
AttemptOutcome = Literal["passed", "needs_retry", "insufficient_evidence"]
|
||||
PracticeProgress = Literal["practicing", "transfer_pending", "mastered"]
|
||||
ScenarioNovelty = Literal["familiar", "unseen_transfer"]
|
||||
ClientPracticeResponse = Literal[
|
||||
"rejecting",
|
||||
"withdrawn",
|
||||
"compliance_only",
|
||||
"mixed",
|
||||
"engaged",
|
||||
"explicit_alignment",
|
||||
]
|
||||
EvidenceKind = Literal[
|
||||
"scene_context",
|
||||
"learner_behavior",
|
||||
"client_response",
|
||||
"evaluator_decision",
|
||||
"voice_feature",
|
||||
]
|
||||
|
||||
|
||||
class PracticeEvidenceRef(BaseModel):
|
||||
"""원문을 복제하지 않는 장면·행동·반응 원장 포인터."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
ref_id: str = Field(min_length=1, max_length=180)
|
||||
scene_id: str = Field(min_length=1, max_length=180)
|
||||
turn_index: int = Field(ge=0)
|
||||
actor: Literal["learner", "client", "observer", "runtime"]
|
||||
kind: EvidenceKind
|
||||
|
||||
|
||||
class ReplayActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["replay"] = "replay"
|
||||
launch_intent: Literal["practice.replay.launch"] = "practice.replay.launch"
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty = "familiar"
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
pause_at_evidence_ref: str = Field(min_length=1, max_length=180)
|
||||
|
||||
|
||||
class BranchActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["branch"] = "branch"
|
||||
launch_intent: Literal["practice.branch.launch"] = "practice.branch.launch"
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
branch_options: tuple[str, ...] = Field(min_length=2, max_length=5)
|
||||
client_responses_hidden: Literal[True] = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_distinct_branches(self) -> "BranchActivity":
|
||||
if len(set(self.branch_options)) != len(self.branch_options):
|
||||
raise ValueError("branch options must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class ConstrainedResponseActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["constrained_response"] = "constrained_response"
|
||||
launch_intent: Literal["practice.constrained-response.launch"] = (
|
||||
"practice.constrained-response.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
max_words: int = Field(ge=5, le=80)
|
||||
required_moves: tuple[str, ...] = Field(min_length=1, max_length=4)
|
||||
|
||||
|
||||
class VoiceRetryActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["voice_retry"] = "voice_retry"
|
||||
launch_intent: Literal["practice.voice-retry.launch"] = (
|
||||
"practice.voice-retry.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
max_seconds: int = Field(ge=5, le=120)
|
||||
acoustic_focus: tuple[str, ...] = Field(min_length=1, max_length=4)
|
||||
|
||||
|
||||
class DifficultyStep(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
level: int = Field(ge=1, le=5)
|
||||
variation: str = Field(min_length=1, max_length=240)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
|
||||
|
||||
class DifficultyLadderActivity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
mode: Literal["difficulty_ladder"] = "difficulty_ladder"
|
||||
launch_intent: Literal["practice.difficulty-ladder.launch"] = (
|
||||
"practice.difficulty-ladder.launch"
|
||||
)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
steps: tuple[DifficultyStep, ...] = Field(min_length=2, max_length=5)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_ladder_with_transfer(self) -> "DifficultyLadderActivity":
|
||||
levels = [item.level for item in self.steps]
|
||||
if levels != sorted(set(levels)):
|
||||
raise ValueError("difficulty ladder levels must be unique and ascending")
|
||||
if not any(item.scenario_novelty == "unseen_transfer" for item in self.steps):
|
||||
raise ValueError("difficulty ladder must end in an unseen transfer step")
|
||||
return self
|
||||
|
||||
|
||||
PracticeActivity = Annotated[
|
||||
ReplayActivity
|
||||
| BranchActivity
|
||||
| ConstrainedResponseActivity
|
||||
| VoiceRetryActivity
|
||||
| DifficultyLadderActivity,
|
||||
Field(discriminator="mode"),
|
||||
]
|
||||
|
||||
|
||||
class PracticeTargetSpec(BaseModel):
|
||||
"""한 역량의 한 관찰 행동만 소유하는 원자적 연습 명세."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
||||
observable_behavior: str = Field(min_length=10, max_length=500)
|
||||
activity: PracticeActivity
|
||||
|
||||
|
||||
class CoachingCard(BaseModel):
|
||||
"""실행 가능한 재연습이 없는 코칭 카드를 구조적으로 거부한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
card_id: str = Field(pattern=r"^oas-g4-card-[a-z0-9-]+$")
|
||||
scene_id: str = Field(min_length=1, max_length=180)
|
||||
coach_claim: str = Field(min_length=10, max_length=800)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
targets: tuple[PracticeTargetSpec, ...] = Field(min_length=1, max_length=3)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_atomic_actionable_targets(self) -> "CoachingCard":
|
||||
if any(item.scene_id != self.scene_id for item in self.evidence_refs):
|
||||
raise ValueError("coaching card evidence must belong to its scene")
|
||||
prescription_ids = [item.prescription_id for item in self.targets]
|
||||
if len(set(prescription_ids)) != len(prescription_ids):
|
||||
raise ValueError("coaching card prescription ids must be unique")
|
||||
target_keys = [(item.competency_id, item.criterion_id) for item in self.targets]
|
||||
if len(set(target_keys)) != len(target_keys):
|
||||
raise ValueError("coaching card targets must be atomic and unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticePrescription(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.practice-prescription.v1"] = (
|
||||
"vignette.practice-prescription.v1"
|
||||
)
|
||||
event_name: Literal["practice.prescribed"] = "practice.prescribed"
|
||||
prescription_id: str
|
||||
coaching_card_id: str
|
||||
scene_id: str
|
||||
competency_id: str
|
||||
criterion_id: str
|
||||
observable_behavior: str
|
||||
activity: PracticeActivity
|
||||
can_launch: Literal[True] = True
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
||||
source_refs: tuple[str, ...] = Field(min_length=1)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class CriterionObservation(BaseModel):
|
||||
"""시도의 관찰 결과. 오류와 미관찰을 성공값으로 보간하지 않는다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
||||
status: CriterionStatus
|
||||
source_kind: SourceKind
|
||||
perspective: MeasurementPerspective
|
||||
model_run_id: UUID | None = None
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def preserve_observation_truth(self) -> "CriterionObservation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError("practice observation mixes source and perspective layers")
|
||||
if (
|
||||
self.source_kind in {"model_inferred", "agent_reported"}
|
||||
and self.model_run_id is None
|
||||
):
|
||||
raise ValueError("model/agent practice observation requires model_run_id")
|
||||
if self.status == "observed":
|
||||
if not self.evidence_refs:
|
||||
raise ValueError("observed practice criterion requires evidence")
|
||||
if self.error_code:
|
||||
raise ValueError("observed practice criterion cannot carry error_code")
|
||||
elif self.status == "not_observed":
|
||||
if not self.counterevidence:
|
||||
raise ValueError(
|
||||
"not-observed practice criterion requires counterevidence"
|
||||
)
|
||||
if self.error_code:
|
||||
raise ValueError(
|
||||
"not-observed practice criterion cannot carry error_code"
|
||||
)
|
||||
else:
|
||||
if not self.error_code or self.uncertainty != 1.0:
|
||||
raise ValueError(
|
||||
"error practice criterion requires error_code and maximum uncertainty"
|
||||
)
|
||||
if len({(item.ref_id, item.kind) for item in self.evidence_refs}) != len(
|
||||
self.evidence_refs
|
||||
):
|
||||
raise ValueError("practice criterion evidence refs must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeAttemptObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str = Field(pattern=r"^oas-g4-attempt-[a-z0-9-]+$")
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
sequence_no: int = Field(ge=1)
|
||||
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
||||
scenario_novelty: ScenarioNovelty
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
criterion: CriterionObservation
|
||||
client_response: ClientPracticeResponse | None = None
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
utterance_template_id: str | None = Field(default=None, max_length=180)
|
||||
learner_claimed_success: bool = False
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_behavior_and_impact_evidence(self) -> "PracticeAttemptObservation":
|
||||
refs = (*self.evidence_refs, *self.criterion.evidence_refs)
|
||||
if len({(item.ref_id, item.kind) for item in refs}) != len(refs):
|
||||
raise ValueError("practice attempt evidence refs must be unique")
|
||||
if self.criterion.status == "error":
|
||||
if (
|
||||
not self.error_code
|
||||
or self.client_response is not None
|
||||
or self.uncertainty != 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"error practice attempt must remain impact-free with maximum uncertainty"
|
||||
)
|
||||
return self
|
||||
if self.error_code:
|
||||
raise ValueError("ready practice attempt cannot carry error_code")
|
||||
kinds = {item.kind for item in refs}
|
||||
if "learner_behavior" not in kinds or "client_response" not in kinds:
|
||||
raise ValueError(
|
||||
"ready practice attempt requires learner behavior and client response evidence"
|
||||
)
|
||||
if self.client_response is None:
|
||||
raise ValueError("ready practice attempt requires observed client response")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeEpisodeInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
episode_id: str = Field(pattern=r"^oas-g4-episode-[a-z0-9-]+$")
|
||||
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
||||
attempts: tuple[PracticeAttemptObservation, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_ordered_attempt_history(self) -> "PracticeEpisodeInput":
|
||||
if any(item.prescription_id != self.prescription_id for item in self.attempts):
|
||||
raise ValueError(
|
||||
"practice episode attempts must reference one prescription"
|
||||
)
|
||||
sequences = [item.sequence_no for item in self.attempts]
|
||||
if sequences != list(range(1, len(sequences) + 1)):
|
||||
raise ValueError("practice attempts must have contiguous sequence numbers")
|
||||
ids = [item.attempt_id for item in self.attempts]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("practice attempt ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class PracticeAttemptAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
attempt_id: str
|
||||
outcome: AttemptOutcome
|
||||
criterion_status: CriterionStatus
|
||||
client_response: ClientPracticeResponse | None
|
||||
scenario_novelty: ScenarioNovelty
|
||||
scenario_variant_id: str
|
||||
difficulty_level: int = Field(ge=1, le=5)
|
||||
utterance_template_id: str | None
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
|
||||
class BeforeAfterComparison(BaseModel):
|
||||
"""총점 차이가 아니라 동일 기준의 전후 관찰과 근거를 보존한다."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
criterion_id: str
|
||||
before_attempt_id: str
|
||||
after_attempt_id: str
|
||||
change: Literal["improved", "unchanged", "regressed", "inconclusive"]
|
||||
before_status: CriterionStatus
|
||||
after_status: CriterionStatus
|
||||
before_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
after_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
|
||||
class PracticeEpisodeAssessment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.practice-episode-assessment.v1"] = (
|
||||
"vignette.practice-episode-assessment.v1"
|
||||
)
|
||||
event_names: tuple[
|
||||
Literal["practice.attempted", "practice.mastered", "transfer.verified"], ...
|
||||
]
|
||||
episode_id: str
|
||||
prescription_id: str
|
||||
competency_id: str
|
||||
attempts: tuple[PracticeAttemptAssessment, ...]
|
||||
comparison: BeforeAfterComparison
|
||||
prior_familiar_demonstrations: int = Field(default=0, ge=0)
|
||||
progress: PracticeProgress
|
||||
mastery_allowed: bool
|
||||
mastery_blockers: tuple[str, ...]
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
||||
counterevidence: tuple[str, ...]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_transfer_before_mastery(self) -> "PracticeEpisodeAssessment":
|
||||
if self.progress == "mastered":
|
||||
if not self.mastery_allowed or "transfer.verified" not in self.event_names:
|
||||
raise ValueError("mastery requires an explicit transfer.verified event")
|
||||
has_familiar_basis = self.prior_familiar_demonstrations > 0 or any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "familiar"
|
||||
for item in self.attempts
|
||||
)
|
||||
if not has_familiar_basis:
|
||||
raise ValueError(
|
||||
"mastery requires current or prior familiar demonstration evidence"
|
||||
)
|
||||
if not any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
||||
for item in self.attempts
|
||||
):
|
||||
raise ValueError("mastery requires passed unseen transfer evidence")
|
||||
elif self.mastery_allowed:
|
||||
raise ValueError("non-mastered practice cannot allow mastery")
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
label_ko: str = Field(min_length=1, max_length=120)
|
||||
description: str = Field(min_length=10, max_length=500)
|
||||
prerequisite_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class CompetencyState(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
||||
band: CompetencyBand
|
||||
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
||||
uncertainty: float = Field(ge=0.0, le=1.0)
|
||||
attempt_count: int = Field(ge=0)
|
||||
familiar_demonstrations: int = Field(ge=0)
|
||||
unseen_transfer_demonstrations: int = Field(ge=0)
|
||||
highest_familiar_difficulty: int = Field(ge=0, le=5)
|
||||
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
||||
counterevidence: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def prevent_unverified_mastery(self) -> "CompetencyState":
|
||||
if (
|
||||
self.familiar_demonstrations + self.unseen_transfer_demonstrations
|
||||
> self.attempt_count
|
||||
):
|
||||
raise ValueError("competency demonstrations cannot exceed attempt count")
|
||||
if self.band == "unassessed" and self.attempt_count:
|
||||
raise ValueError("attempted competency cannot remain unassessed")
|
||||
if self.band == "consistent_local" and self.familiar_demonstrations < 1:
|
||||
raise ValueError(
|
||||
"consistent_local requires familiar demonstration evidence"
|
||||
)
|
||||
if self.band == "transfer_verified":
|
||||
if self.unseen_transfer_demonstrations < 1 or not self.evidence_refs:
|
||||
raise ValueError("transfer_verified requires unseen transfer evidence")
|
||||
elif self.unseen_transfer_demonstrations:
|
||||
raise ValueError(
|
||||
"unseen transfer demonstration must promote transfer_verified"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class CompetencyGraph(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.competency-graph.v1"] = (
|
||||
"vignette.competency-graph.v1"
|
||||
)
|
||||
definitions: tuple[CompetencyDefinition, ...] = Field(min_length=1)
|
||||
states: tuple[CompetencyState, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_acyclic_graph(self) -> "CompetencyGraph":
|
||||
definitions = {item.competency_id: item for item in self.definitions}
|
||||
states = {item.competency_id: item for item in self.states}
|
||||
if len(definitions) != len(self.definitions) or len(states) != len(self.states):
|
||||
raise ValueError("competency graph ids must be unique")
|
||||
if definitions.keys() != states.keys():
|
||||
raise ValueError(
|
||||
"competency graph requires exactly one state per definition"
|
||||
)
|
||||
if any(
|
||||
prerequisite not in definitions
|
||||
for item in self.definitions
|
||||
for prerequisite in item.prerequisite_ids
|
||||
):
|
||||
raise ValueError("competency prerequisite must exist in graph")
|
||||
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def visit(competency_id: str) -> None:
|
||||
if competency_id in visiting:
|
||||
raise ValueError("competency graph prerequisites must be acyclic")
|
||||
if competency_id in visited:
|
||||
return
|
||||
visiting.add(competency_id)
|
||||
for prerequisite in definitions[competency_id].prerequisite_ids:
|
||||
visit(prerequisite)
|
||||
visiting.remove(competency_id)
|
||||
visited.add(competency_id)
|
||||
|
||||
for competency_id in definitions:
|
||||
visit(competency_id)
|
||||
return self
|
||||
|
||||
|
||||
class CurriculumDecision(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.curriculum-decision.v1"] = (
|
||||
"vignette.curriculum-decision.v1"
|
||||
)
|
||||
selected_prescription_id: str
|
||||
competency_id: str
|
||||
competency_band: CompetencyBand
|
||||
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
||||
mode: PracticeMode
|
||||
selection_basis: tuple[str, ...] = Field(min_length=2)
|
||||
deferred_prescription_ids: tuple[str, ...]
|
||||
blocked_prescription_reasons: tuple[str, ...]
|
||||
|
||||
|
||||
class PracticeBenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
episode_progress: tuple[PracticeProgress, ...]
|
||||
final_competency_id: str
|
||||
final_band: CompetencyBand
|
||||
selected_prescription_id: str
|
||||
|
||||
|
||||
class PracticeBenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g4-bench-[0-9]{3}$")
|
||||
title_ko: str = Field(min_length=1, max_length=200)
|
||||
coaching_cards: tuple[CoachingCard, ...] = Field(min_length=1)
|
||||
graph: CompetencyGraph
|
||||
episodes: tuple[PracticeEpisodeInput, ...] = ()
|
||||
expected: PracticeBenchmarkExpectation
|
||||
tags: tuple[str, ...] = ()
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class PracticeBenchmarkPack(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
schema_version: Literal["vignette.deliberate-practice-benchmark.v1"] = (
|
||||
"vignette.deliberate-practice-benchmark.v1"
|
||||
)
|
||||
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
cases: tuple[PracticeBenchmarkCase, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_adversarial_and_mode_coverage(self) -> "PracticeBenchmarkPack":
|
||||
case_ids = [item.case_id for item in self.cases]
|
||||
if len(set(case_ids)) != len(case_ids):
|
||||
raise ValueError("practice benchmark case ids must be unique")
|
||||
modes = {
|
||||
target.activity.mode
|
||||
for case in self.cases
|
||||
for card in case.coaching_cards
|
||||
for target in card.targets
|
||||
}
|
||||
if modes != set(PRACTICE_MODES):
|
||||
raise ValueError("practice benchmark must cover every practice mode")
|
||||
required_tags = {
|
||||
"reward_hacking",
|
||||
"easy_repeat_hacking",
|
||||
"memorized_phrase_hacking",
|
||||
"unseen_transfer_gate",
|
||||
}
|
||||
tags = {tag for case in self.cases for tag in case.tags}
|
||||
if not required_tags.issubset(tags):
|
||||
raise ValueError("practice benchmark lacks required adversarial coverage")
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttemptOutcome",
|
||||
"BeforeAfterComparison",
|
||||
"BranchActivity",
|
||||
"COMPETENCY_BANDS",
|
||||
"ClientPracticeResponse",
|
||||
"CoachingCard",
|
||||
"CompetencyBand",
|
||||
"CompetencyDefinition",
|
||||
"CompetencyGraph",
|
||||
"CompetencyState",
|
||||
"ConstrainedResponseActivity",
|
||||
"CriterionObservation",
|
||||
"CriterionStatus",
|
||||
"CurriculumDecision",
|
||||
"DifficultyLadderActivity",
|
||||
"DifficultyStep",
|
||||
"EvidenceKind",
|
||||
"PRACTICE_MODES",
|
||||
"PracticeActivity",
|
||||
"PracticeAttemptAssessment",
|
||||
"PracticeAttemptObservation",
|
||||
"PracticeBenchmarkCase",
|
||||
"PracticeBenchmarkExpectation",
|
||||
"PracticeBenchmarkPack",
|
||||
"PracticeEpisodeAssessment",
|
||||
"PracticeEpisodeInput",
|
||||
"PracticeEvidenceRef",
|
||||
"PracticeMode",
|
||||
"PracticePrescription",
|
||||
"PracticeProgress",
|
||||
"PracticeTargetSpec",
|
||||
"ReplayActivity",
|
||||
"ScenarioNovelty",
|
||||
"VoiceRetryActivity",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue