16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
639 lines
24 KiB
Python
639 lines
24 KiB
Python
"""G4 Deliberate Practice Engine의 버전 고정 순수 도메인 계약.
|
|
|
|
코칭 장면을 하나의 관찰 가능한 행동으로 분해한 실행형 처방, 시도 근거,
|
|
역량 그래프와 전이 게이트를 정의한다. 단일 총점이나 보상 점수로 숙련을
|
|
승격하지 않으며, 익숙한 장면의 성공과 미지 사례 전이를 물리적으로 구분한다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Annotated, Literal
|
|
from uuid import UUID
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
|
from .measurement import (
|
|
MeasurementPerspective,
|
|
SOURCE_PERSPECTIVE_COMPATIBILITY,
|
|
SourceKind,
|
|
)
|
|
|
|
|
|
PRACTICE_MODES = (
|
|
"replay",
|
|
"branch",
|
|
"constrained_response",
|
|
"voice_retry",
|
|
"difficulty_ladder",
|
|
)
|
|
PracticeMode = Literal[
|
|
"replay",
|
|
"branch",
|
|
"constrained_response",
|
|
"voice_retry",
|
|
"difficulty_ladder",
|
|
]
|
|
|
|
COMPETENCY_BANDS = (
|
|
"unassessed",
|
|
"fragile",
|
|
"developing",
|
|
"consistent_local",
|
|
"transfer_verified",
|
|
)
|
|
CompetencyBand = Literal[
|
|
"unassessed",
|
|
"fragile",
|
|
"developing",
|
|
"consistent_local",
|
|
"transfer_verified",
|
|
]
|
|
|
|
CriterionStatus = Literal["observed", "not_observed", "error"]
|
|
AttemptOutcome = Literal["passed", "needs_retry", "insufficient_evidence"]
|
|
PracticeProgress = Literal["practicing", "transfer_pending", "mastered"]
|
|
ScenarioNovelty = Literal["familiar", "unseen_transfer"]
|
|
ClientPracticeResponse = Literal[
|
|
"rejecting",
|
|
"withdrawn",
|
|
"compliance_only",
|
|
"mixed",
|
|
"engaged",
|
|
"explicit_alignment",
|
|
]
|
|
EvidenceKind = Literal[
|
|
"scene_context",
|
|
"learner_behavior",
|
|
"client_response",
|
|
"evaluator_decision",
|
|
"voice_feature",
|
|
]
|
|
|
|
|
|
class PracticeEvidenceRef(BaseModel):
|
|
"""원문을 복제하지 않는 장면·행동·반응 원장 포인터."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
ref_id: str = Field(min_length=1, max_length=180)
|
|
scene_id: str = Field(min_length=1, max_length=180)
|
|
turn_index: int = Field(ge=0)
|
|
actor: Literal["learner", "client", "observer", "runtime"]
|
|
kind: EvidenceKind
|
|
|
|
|
|
class ReplayActivity(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
mode: Literal["replay"] = "replay"
|
|
launch_intent: Literal["practice.replay.launch"] = "practice.replay.launch"
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty = "familiar"
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
pause_at_evidence_ref: str = Field(min_length=1, max_length=180)
|
|
|
|
|
|
class BranchActivity(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
mode: Literal["branch"] = "branch"
|
|
launch_intent: Literal["practice.branch.launch"] = "practice.branch.launch"
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
branch_options: tuple[str, ...] = Field(min_length=2, max_length=5)
|
|
client_responses_hidden: Literal[True] = True
|
|
|
|
@model_validator(mode="after")
|
|
def require_distinct_branches(self) -> "BranchActivity":
|
|
if len(set(self.branch_options)) != len(self.branch_options):
|
|
raise ValueError("branch options must be unique")
|
|
return self
|
|
|
|
|
|
class ConstrainedResponseActivity(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
mode: Literal["constrained_response"] = "constrained_response"
|
|
launch_intent: Literal["practice.constrained-response.launch"] = (
|
|
"practice.constrained-response.launch"
|
|
)
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
max_words: int = Field(ge=5, le=80)
|
|
required_moves: tuple[str, ...] = Field(min_length=1, max_length=4)
|
|
|
|
|
|
class VoiceRetryActivity(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
mode: Literal["voice_retry"] = "voice_retry"
|
|
launch_intent: Literal["practice.voice-retry.launch"] = (
|
|
"practice.voice-retry.launch"
|
|
)
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
max_seconds: int = Field(ge=5, le=120)
|
|
acoustic_focus: tuple[str, ...] = Field(min_length=1, max_length=4)
|
|
|
|
|
|
class DifficultyStep(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
level: int = Field(ge=1, le=5)
|
|
variation: str = Field(min_length=1, max_length=240)
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
|
|
|
|
class DifficultyLadderActivity(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
mode: Literal["difficulty_ladder"] = "difficulty_ladder"
|
|
launch_intent: Literal["practice.difficulty-ladder.launch"] = (
|
|
"practice.difficulty-ladder.launch"
|
|
)
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
steps: tuple[DifficultyStep, ...] = Field(min_length=2, max_length=5)
|
|
|
|
@model_validator(mode="after")
|
|
def require_ordered_ladder_with_transfer(self) -> "DifficultyLadderActivity":
|
|
levels = [item.level for item in self.steps]
|
|
if levels != sorted(set(levels)):
|
|
raise ValueError("difficulty ladder levels must be unique and ascending")
|
|
if not any(item.scenario_novelty == "unseen_transfer" for item in self.steps):
|
|
raise ValueError("difficulty ladder must end in an unseen transfer step")
|
|
return self
|
|
|
|
|
|
PracticeActivity = Annotated[
|
|
ReplayActivity
|
|
| BranchActivity
|
|
| ConstrainedResponseActivity
|
|
| VoiceRetryActivity
|
|
| DifficultyLadderActivity,
|
|
Field(discriminator="mode"),
|
|
]
|
|
|
|
|
|
class PracticeTargetSpec(BaseModel):
|
|
"""한 역량의 한 관찰 행동만 소유하는 원자적 연습 명세."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
|
observable_behavior: str = Field(min_length=10, max_length=500)
|
|
activity: PracticeActivity
|
|
|
|
|
|
class CoachingCard(BaseModel):
|
|
"""실행 가능한 재연습이 없는 코칭 카드를 구조적으로 거부한다."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
card_id: str = Field(pattern=r"^oas-g4-card-[a-z0-9-]+$")
|
|
scene_id: str = Field(min_length=1, max_length=180)
|
|
coach_claim: str = Field(min_length=10, max_length=800)
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
|
source_refs: tuple[str, ...] = Field(min_length=1)
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
counterevidence: tuple[str, ...] = ()
|
|
targets: tuple[PracticeTargetSpec, ...] = Field(min_length=1, max_length=3)
|
|
|
|
@model_validator(mode="after")
|
|
def require_atomic_actionable_targets(self) -> "CoachingCard":
|
|
if any(item.scene_id != self.scene_id for item in self.evidence_refs):
|
|
raise ValueError("coaching card evidence must belong to its scene")
|
|
prescription_ids = [item.prescription_id for item in self.targets]
|
|
if len(set(prescription_ids)) != len(prescription_ids):
|
|
raise ValueError("coaching card prescription ids must be unique")
|
|
target_keys = [(item.competency_id, item.criterion_id) for item in self.targets]
|
|
if len(set(target_keys)) != len(target_keys):
|
|
raise ValueError("coaching card targets must be atomic and unique")
|
|
return self
|
|
|
|
|
|
class PracticePrescription(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.practice-prescription.v1"] = (
|
|
"vignette.practice-prescription.v1"
|
|
)
|
|
event_name: Literal["practice.prescribed"] = "practice.prescribed"
|
|
prescription_id: str
|
|
coaching_card_id: str
|
|
scene_id: str
|
|
competency_id: str
|
|
criterion_id: str
|
|
observable_behavior: str
|
|
activity: PracticeActivity
|
|
can_launch: Literal[True] = True
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...] = Field(min_length=1)
|
|
source_refs: tuple[str, ...] = Field(min_length=1)
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
counterevidence: tuple[str, ...] = ()
|
|
|
|
|
|
class CriterionObservation(BaseModel):
|
|
"""시도의 관찰 결과. 오류와 미관찰을 성공값으로 보간하지 않는다."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
criterion_id: str = Field(pattern=r"^criterion\.[a-z0-9_.-]+$")
|
|
status: CriterionStatus
|
|
source_kind: SourceKind
|
|
perspective: MeasurementPerspective
|
|
model_run_id: UUID | None = None
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
error_code: str | None = Field(default=None, max_length=120)
|
|
|
|
@model_validator(mode="after")
|
|
def preserve_observation_truth(self) -> "CriterionObservation":
|
|
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
|
raise ValueError("practice observation mixes source and perspective layers")
|
|
if (
|
|
self.source_kind in {"model_inferred", "agent_reported"}
|
|
and self.model_run_id is None
|
|
):
|
|
raise ValueError("model/agent practice observation requires model_run_id")
|
|
if self.status == "observed":
|
|
if not self.evidence_refs:
|
|
raise ValueError("observed practice criterion requires evidence")
|
|
if self.error_code:
|
|
raise ValueError("observed practice criterion cannot carry error_code")
|
|
elif self.status == "not_observed":
|
|
if not self.counterevidence:
|
|
raise ValueError(
|
|
"not-observed practice criterion requires counterevidence"
|
|
)
|
|
if self.error_code:
|
|
raise ValueError(
|
|
"not-observed practice criterion cannot carry error_code"
|
|
)
|
|
else:
|
|
if not self.error_code or self.uncertainty != 1.0:
|
|
raise ValueError(
|
|
"error practice criterion requires error_code and maximum uncertainty"
|
|
)
|
|
if len({(item.ref_id, item.kind) for item in self.evidence_refs}) != len(
|
|
self.evidence_refs
|
|
):
|
|
raise ValueError("practice criterion evidence refs must be unique")
|
|
return self
|
|
|
|
|
|
class PracticeAttemptObservation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
attempt_id: str = Field(pattern=r"^oas-g4-attempt-[a-z0-9-]+$")
|
|
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
sequence_no: int = Field(ge=1)
|
|
scenario_variant_id: str = Field(min_length=1, max_length=180)
|
|
scenario_novelty: ScenarioNovelty
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
criterion: CriterionObservation
|
|
client_response: ClientPracticeResponse | None = None
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
counterevidence: tuple[str, ...] = ()
|
|
utterance_template_id: str | None = Field(default=None, max_length=180)
|
|
learner_claimed_success: bool = False
|
|
error_code: str | None = Field(default=None, max_length=120)
|
|
|
|
@model_validator(mode="after")
|
|
def require_behavior_and_impact_evidence(self) -> "PracticeAttemptObservation":
|
|
refs = (*self.evidence_refs, *self.criterion.evidence_refs)
|
|
if len({(item.ref_id, item.kind) for item in refs}) != len(refs):
|
|
raise ValueError("practice attempt evidence refs must be unique")
|
|
if self.criterion.status == "error":
|
|
if (
|
|
not self.error_code
|
|
or self.client_response is not None
|
|
or self.uncertainty != 1.0
|
|
):
|
|
raise ValueError(
|
|
"error practice attempt must remain impact-free with maximum uncertainty"
|
|
)
|
|
return self
|
|
if self.error_code:
|
|
raise ValueError("ready practice attempt cannot carry error_code")
|
|
kinds = {item.kind for item in refs}
|
|
if "learner_behavior" not in kinds or "client_response" not in kinds:
|
|
raise ValueError(
|
|
"ready practice attempt requires learner behavior and client response evidence"
|
|
)
|
|
if self.client_response is None:
|
|
raise ValueError("ready practice attempt requires observed client response")
|
|
return self
|
|
|
|
|
|
class PracticeEpisodeInput(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
episode_id: str = Field(pattern=r"^oas-g4-episode-[a-z0-9-]+$")
|
|
prescription_id: str = Field(pattern=r"^oas-g4-practice-[a-z0-9-]+$")
|
|
attempts: tuple[PracticeAttemptObservation, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def require_ordered_attempt_history(self) -> "PracticeEpisodeInput":
|
|
if any(item.prescription_id != self.prescription_id for item in self.attempts):
|
|
raise ValueError(
|
|
"practice episode attempts must reference one prescription"
|
|
)
|
|
sequences = [item.sequence_no for item in self.attempts]
|
|
if sequences != list(range(1, len(sequences) + 1)):
|
|
raise ValueError("practice attempts must have contiguous sequence numbers")
|
|
ids = [item.attempt_id for item in self.attempts]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError("practice attempt ids must be unique")
|
|
return self
|
|
|
|
|
|
class PracticeAttemptAssessment(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
attempt_id: str
|
|
outcome: AttemptOutcome
|
|
criterion_status: CriterionStatus
|
|
client_response: ClientPracticeResponse | None
|
|
scenario_novelty: ScenarioNovelty
|
|
scenario_variant_id: str
|
|
difficulty_level: int = Field(ge=1, le=5)
|
|
utterance_template_id: str | None
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
|
counterevidence: tuple[str, ...]
|
|
|
|
|
|
class BeforeAfterComparison(BaseModel):
|
|
"""총점 차이가 아니라 동일 기준의 전후 관찰과 근거를 보존한다."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
criterion_id: str
|
|
before_attempt_id: str
|
|
after_attempt_id: str
|
|
change: Literal["improved", "unchanged", "regressed", "inconclusive"]
|
|
before_status: CriterionStatus
|
|
after_status: CriterionStatus
|
|
before_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
|
after_evidence_refs: tuple[PracticeEvidenceRef, ...]
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
counterevidence: tuple[str, ...]
|
|
|
|
|
|
class PracticeEpisodeAssessment(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.practice-episode-assessment.v1"] = (
|
|
"vignette.practice-episode-assessment.v1"
|
|
)
|
|
event_names: tuple[
|
|
Literal["practice.attempted", "practice.mastered", "transfer.verified"], ...
|
|
]
|
|
episode_id: str
|
|
prescription_id: str
|
|
competency_id: str
|
|
attempts: tuple[PracticeAttemptAssessment, ...]
|
|
comparison: BeforeAfterComparison
|
|
prior_familiar_demonstrations: int = Field(default=0, ge=0)
|
|
progress: PracticeProgress
|
|
mastery_allowed: bool
|
|
mastery_blockers: tuple[str, ...]
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...]
|
|
counterevidence: tuple[str, ...]
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_transfer_before_mastery(self) -> "PracticeEpisodeAssessment":
|
|
if self.progress == "mastered":
|
|
if not self.mastery_allowed or "transfer.verified" not in self.event_names:
|
|
raise ValueError("mastery requires an explicit transfer.verified event")
|
|
has_familiar_basis = self.prior_familiar_demonstrations > 0 or any(
|
|
item.outcome == "passed" and item.scenario_novelty == "familiar"
|
|
for item in self.attempts
|
|
)
|
|
if not has_familiar_basis:
|
|
raise ValueError(
|
|
"mastery requires current or prior familiar demonstration evidence"
|
|
)
|
|
if not any(
|
|
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
|
for item in self.attempts
|
|
):
|
|
raise ValueError("mastery requires passed unseen transfer evidence")
|
|
elif self.mastery_allowed:
|
|
raise ValueError("non-mastered practice cannot allow mastery")
|
|
return self
|
|
|
|
|
|
class CompetencyDefinition(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
label_ko: str = Field(min_length=1, max_length=120)
|
|
description: str = Field(min_length=10, max_length=500)
|
|
prerequisite_ids: tuple[str, ...] = ()
|
|
|
|
|
|
class CompetencyState(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
competency_id: str = Field(pattern=r"^competency\.[a-z0-9_.-]+$")
|
|
band: CompetencyBand
|
|
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
|
uncertainty: float = Field(ge=0.0, le=1.0)
|
|
attempt_count: int = Field(ge=0)
|
|
familiar_demonstrations: int = Field(ge=0)
|
|
unseen_transfer_demonstrations: int = Field(ge=0)
|
|
highest_familiar_difficulty: int = Field(ge=0, le=5)
|
|
evidence_refs: tuple[PracticeEvidenceRef, ...] = ()
|
|
counterevidence: tuple[str, ...] = ()
|
|
|
|
@model_validator(mode="after")
|
|
def prevent_unverified_mastery(self) -> "CompetencyState":
|
|
if (
|
|
self.familiar_demonstrations + self.unseen_transfer_demonstrations
|
|
> self.attempt_count
|
|
):
|
|
raise ValueError("competency demonstrations cannot exceed attempt count")
|
|
if self.band == "unassessed" and self.attempt_count:
|
|
raise ValueError("attempted competency cannot remain unassessed")
|
|
if self.band == "consistent_local" and self.familiar_demonstrations < 1:
|
|
raise ValueError(
|
|
"consistent_local requires familiar demonstration evidence"
|
|
)
|
|
if self.band == "transfer_verified":
|
|
if self.unseen_transfer_demonstrations < 1 or not self.evidence_refs:
|
|
raise ValueError("transfer_verified requires unseen transfer evidence")
|
|
elif self.unseen_transfer_demonstrations:
|
|
raise ValueError(
|
|
"unseen transfer demonstration must promote transfer_verified"
|
|
)
|
|
return self
|
|
|
|
|
|
class CompetencyGraph(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.competency-graph.v1"] = (
|
|
"vignette.competency-graph.v1"
|
|
)
|
|
definitions: tuple[CompetencyDefinition, ...] = Field(min_length=1)
|
|
states: tuple[CompetencyState, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def require_complete_acyclic_graph(self) -> "CompetencyGraph":
|
|
definitions = {item.competency_id: item for item in self.definitions}
|
|
states = {item.competency_id: item for item in self.states}
|
|
if len(definitions) != len(self.definitions) or len(states) != len(self.states):
|
|
raise ValueError("competency graph ids must be unique")
|
|
if definitions.keys() != states.keys():
|
|
raise ValueError(
|
|
"competency graph requires exactly one state per definition"
|
|
)
|
|
if any(
|
|
prerequisite not in definitions
|
|
for item in self.definitions
|
|
for prerequisite in item.prerequisite_ids
|
|
):
|
|
raise ValueError("competency prerequisite must exist in graph")
|
|
|
|
visiting: set[str] = set()
|
|
visited: set[str] = set()
|
|
|
|
def visit(competency_id: str) -> None:
|
|
if competency_id in visiting:
|
|
raise ValueError("competency graph prerequisites must be acyclic")
|
|
if competency_id in visited:
|
|
return
|
|
visiting.add(competency_id)
|
|
for prerequisite in definitions[competency_id].prerequisite_ids:
|
|
visit(prerequisite)
|
|
visiting.remove(competency_id)
|
|
visited.add(competency_id)
|
|
|
|
for competency_id in definitions:
|
|
visit(competency_id)
|
|
return self
|
|
|
|
|
|
class CurriculumDecision(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.curriculum-decision.v1"] = (
|
|
"vignette.curriculum-decision.v1"
|
|
)
|
|
selected_prescription_id: str
|
|
competency_id: str
|
|
competency_band: CompetencyBand
|
|
forgetting_risk: float = Field(ge=0.0, le=1.0)
|
|
mode: PracticeMode
|
|
selection_basis: tuple[str, ...] = Field(min_length=2)
|
|
deferred_prescription_ids: tuple[str, ...]
|
|
blocked_prescription_reasons: tuple[str, ...]
|
|
|
|
|
|
class PracticeBenchmarkExpectation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
episode_progress: tuple[PracticeProgress, ...]
|
|
final_competency_id: str
|
|
final_band: CompetencyBand
|
|
selected_prescription_id: str
|
|
|
|
|
|
class PracticeBenchmarkCase(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
case_id: str = Field(pattern=r"^oas-g4-bench-[0-9]{3}$")
|
|
title_ko: str = Field(min_length=1, max_length=200)
|
|
coaching_cards: tuple[CoachingCard, ...] = Field(min_length=1)
|
|
graph: CompetencyGraph
|
|
episodes: tuple[PracticeEpisodeInput, ...] = ()
|
|
expected: PracticeBenchmarkExpectation
|
|
tags: tuple[str, ...] = ()
|
|
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
|
|
|
|
|
class PracticeBenchmarkPack(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["vignette.deliberate-practice-benchmark.v1"] = (
|
|
"vignette.deliberate-practice-benchmark.v1"
|
|
)
|
|
data_classification: Literal["synthetic_educational"] = "synthetic_educational"
|
|
clinical_claim_allowed: Literal[False] = False
|
|
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
|
cases: tuple[PracticeBenchmarkCase, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def require_adversarial_and_mode_coverage(self) -> "PracticeBenchmarkPack":
|
|
case_ids = [item.case_id for item in self.cases]
|
|
if len(set(case_ids)) != len(case_ids):
|
|
raise ValueError("practice benchmark case ids must be unique")
|
|
modes = {
|
|
target.activity.mode
|
|
for case in self.cases
|
|
for card in case.coaching_cards
|
|
for target in card.targets
|
|
}
|
|
if modes != set(PRACTICE_MODES):
|
|
raise ValueError("practice benchmark must cover every practice mode")
|
|
required_tags = {
|
|
"reward_hacking",
|
|
"easy_repeat_hacking",
|
|
"memorized_phrase_hacking",
|
|
"unseen_transfer_gate",
|
|
}
|
|
tags = {tag for case in self.cases for tag in case.tags}
|
|
if not required_tags.issubset(tags):
|
|
raise ValueError("practice benchmark lacks required adversarial coverage")
|
|
return self
|
|
|
|
|
|
__all__ = [
|
|
"AttemptOutcome",
|
|
"BeforeAfterComparison",
|
|
"BranchActivity",
|
|
"COMPETENCY_BANDS",
|
|
"ClientPracticeResponse",
|
|
"CoachingCard",
|
|
"CompetencyBand",
|
|
"CompetencyDefinition",
|
|
"CompetencyGraph",
|
|
"CompetencyState",
|
|
"ConstrainedResponseActivity",
|
|
"CriterionObservation",
|
|
"CriterionStatus",
|
|
"CurriculumDecision",
|
|
"DifficultyLadderActivity",
|
|
"DifficultyStep",
|
|
"EvidenceKind",
|
|
"PRACTICE_MODES",
|
|
"PracticeActivity",
|
|
"PracticeAttemptAssessment",
|
|
"PracticeAttemptObservation",
|
|
"PracticeBenchmarkCase",
|
|
"PracticeBenchmarkExpectation",
|
|
"PracticeBenchmarkPack",
|
|
"PracticeEpisodeAssessment",
|
|
"PracticeEpisodeInput",
|
|
"PracticeEvidenceRef",
|
|
"PracticeMode",
|
|
"PracticePrescription",
|
|
"PracticeProgress",
|
|
"PracticeTargetSpec",
|
|
"ReplayActivity",
|
|
"ScenarioNovelty",
|
|
"VoiceRetryActivity",
|
|
]
|