G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
617
apps/api/app/services/deliberate_practice.py
Normal file
617
apps/api/app/services/deliberate_practice.py
Normal file
|
|
@ -0,0 +1,617 @@
|
|||
"""G4 의도적 수련 처방·전이 게이트·결정론 커리큘럼 코어.
|
||||
|
||||
학습자의 자기 성공 주장이나 외부 보상값을 사용하지 않는다. 관찰 가능한 한
|
||||
행동, 내담자 후속 반응, 익숙한 장면 재현, 미지 사례 전이를 서로 분리한다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from ..contracts.deliberate_practice import (
|
||||
BeforeAfterComparison,
|
||||
CoachingCard,
|
||||
CompetencyBand,
|
||||
CompetencyGraph,
|
||||
CompetencyState,
|
||||
CurriculumDecision,
|
||||
PracticeAttemptAssessment,
|
||||
PracticeAttemptObservation,
|
||||
PracticeBenchmarkPack,
|
||||
PracticeEpisodeAssessment,
|
||||
PracticeEpisodeInput,
|
||||
PracticeEvidenceRef,
|
||||
PracticePrescription,
|
||||
)
|
||||
|
||||
|
||||
_ENGAGED_RESPONSES = frozenset({"engaged", "explicit_alignment"})
|
||||
_BAND_RANK: dict[CompetencyBand, int] = {
|
||||
"unassessed": 0,
|
||||
"fragile": 1,
|
||||
"developing": 2,
|
||||
"consistent_local": 3,
|
||||
"transfer_verified": 4,
|
||||
}
|
||||
|
||||
|
||||
def _unique_evidence(
|
||||
refs: Iterable[PracticeEvidenceRef],
|
||||
) -> tuple[PracticeEvidenceRef, ...]:
|
||||
output: list[PracticeEvidenceRef] = []
|
||||
seen: set[str] = set()
|
||||
for ref in refs:
|
||||
if ref.ref_id in seen:
|
||||
continue
|
||||
seen.add(ref.ref_id)
|
||||
output.append(ref)
|
||||
return tuple(output)
|
||||
|
||||
|
||||
def prescribe_from_coaching_cards(
|
||||
cards: Iterable[CoachingCard],
|
||||
) -> tuple[PracticePrescription, ...]:
|
||||
"""각 코칭 카드의 각 목표를 실행 가능한 원자적 처방으로 투영한다."""
|
||||
|
||||
prescriptions: list[PracticePrescription] = []
|
||||
card_ids: set[str] = set()
|
||||
prescription_ids: set[str] = set()
|
||||
for card in cards:
|
||||
if card.card_id in card_ids:
|
||||
raise ValueError(f"duplicate coaching card id: {card.card_id}")
|
||||
card_ids.add(card.card_id)
|
||||
for target in card.targets:
|
||||
if target.prescription_id in prescription_ids:
|
||||
raise ValueError(
|
||||
f"duplicate practice prescription id: {target.prescription_id}"
|
||||
)
|
||||
prescription_ids.add(target.prescription_id)
|
||||
prescriptions.append(
|
||||
PracticePrescription(
|
||||
prescription_id=target.prescription_id,
|
||||
coaching_card_id=card.card_id,
|
||||
scene_id=card.scene_id,
|
||||
competency_id=target.competency_id,
|
||||
criterion_id=target.criterion_id,
|
||||
observable_behavior=target.observable_behavior,
|
||||
activity=target.activity,
|
||||
evidence_refs=card.evidence_refs,
|
||||
source_refs=card.source_refs,
|
||||
uncertainty=card.uncertainty,
|
||||
counterevidence=card.counterevidence,
|
||||
)
|
||||
)
|
||||
if card_ids != {item.coaching_card_id for item in prescriptions}:
|
||||
raise ValueError("every coaching card must produce an actionable prescription")
|
||||
return tuple(prescriptions)
|
||||
|
||||
|
||||
def _assess_attempt(
|
||||
prescription: PracticePrescription,
|
||||
attempt: PracticeAttemptObservation,
|
||||
) -> PracticeAttemptAssessment:
|
||||
if attempt.prescription_id != prescription.prescription_id:
|
||||
raise ValueError("practice attempt references a different prescription")
|
||||
if attempt.competency_id != prescription.competency_id:
|
||||
raise ValueError("practice attempt targets a different competency")
|
||||
if attempt.criterion.criterion_id != prescription.criterion_id:
|
||||
raise ValueError("practice attempt evaluates a different atomic criterion")
|
||||
|
||||
evidence_refs = _unique_evidence(
|
||||
(*attempt.evidence_refs, *attempt.criterion.evidence_refs)
|
||||
)
|
||||
counterevidence = list(
|
||||
dict.fromkeys((*attempt.counterevidence, *attempt.criterion.counterevidence))
|
||||
)
|
||||
independent_observation = (
|
||||
attempt.criterion.source_kind
|
||||
in {"model_inferred", "human_rated", "observed_runtime"}
|
||||
and attempt.criterion.perspective
|
||||
in {"independent_observer", "supervisor_human", "runtime_observation"}
|
||||
)
|
||||
if attempt.criterion.status != "error" and not independent_observation:
|
||||
outcome = "insufficient_evidence"
|
||||
counterevidence.append("independent_observer_required")
|
||||
elif attempt.criterion.status == "error":
|
||||
outcome = "insufficient_evidence"
|
||||
counterevidence.append(
|
||||
f"criterion_evaluation_failed:{attempt.criterion.error_code or 'unknown'}"
|
||||
)
|
||||
else:
|
||||
has_voice_evidence = any(item.kind == "voice_feature" for item in evidence_refs)
|
||||
voice_ready = prescription.activity.mode != "voice_retry" or has_voice_evidence
|
||||
impact_ready = attempt.client_response in _ENGAGED_RESPONSES
|
||||
uncertainty_ready = (
|
||||
max(attempt.uncertainty, attempt.criterion.uncertainty) <= 0.5
|
||||
)
|
||||
if (
|
||||
attempt.criterion.status == "observed"
|
||||
and impact_ready
|
||||
and uncertainty_ready
|
||||
and voice_ready
|
||||
):
|
||||
outcome = "passed"
|
||||
else:
|
||||
outcome = "needs_retry"
|
||||
if attempt.criterion.status != "observed":
|
||||
counterevidence.append("target_behavior_not_observed")
|
||||
if not impact_ready:
|
||||
counterevidence.append("client_response_does_not_support_effect")
|
||||
if not uncertainty_ready:
|
||||
counterevidence.append("attempt_uncertainty_above_acceptance_boundary")
|
||||
if not voice_ready:
|
||||
counterevidence.append("voice_retry_missing_voice_feature_evidence")
|
||||
if attempt.learner_claimed_success and outcome != "passed":
|
||||
counterevidence.append(
|
||||
"learner_success_claim_not_supported_by_attempt_evidence"
|
||||
)
|
||||
return PracticeAttemptAssessment(
|
||||
attempt_id=attempt.attempt_id,
|
||||
outcome=outcome,
|
||||
criterion_status=attempt.criterion.status,
|
||||
client_response=attempt.client_response,
|
||||
scenario_novelty=attempt.scenario_novelty,
|
||||
scenario_variant_id=attempt.scenario_variant_id,
|
||||
difficulty_level=attempt.difficulty_level,
|
||||
utterance_template_id=attempt.utterance_template_id,
|
||||
uncertainty=max(attempt.uncertainty, attempt.criterion.uncertainty),
|
||||
evidence_refs=evidence_refs,
|
||||
counterevidence=tuple(dict.fromkeys(counterevidence)),
|
||||
)
|
||||
|
||||
|
||||
def _comparison(
|
||||
prescription: PracticePrescription,
|
||||
episode: PracticeEpisodeInput,
|
||||
assessed: tuple[PracticeAttemptAssessment, ...],
|
||||
) -> BeforeAfterComparison:
|
||||
before_observation = episode.attempts[0].criterion
|
||||
after_observation = episode.attempts[-1].criterion
|
||||
if "error" in {before_observation.status, after_observation.status}:
|
||||
change = "inconclusive"
|
||||
elif (
|
||||
before_observation.status == "not_observed"
|
||||
and after_observation.status == "observed"
|
||||
):
|
||||
change = "improved"
|
||||
elif (
|
||||
before_observation.status == "observed"
|
||||
and after_observation.status == "not_observed"
|
||||
):
|
||||
change = "regressed"
|
||||
else:
|
||||
change = "unchanged"
|
||||
return BeforeAfterComparison(
|
||||
criterion_id=prescription.criterion_id,
|
||||
before_attempt_id=assessed[0].attempt_id,
|
||||
after_attempt_id=assessed[-1].attempt_id,
|
||||
change=change,
|
||||
before_status=before_observation.status,
|
||||
after_status=after_observation.status,
|
||||
before_evidence_refs=_unique_evidence(
|
||||
(*episode.attempts[0].evidence_refs, *before_observation.evidence_refs)
|
||||
),
|
||||
after_evidence_refs=_unique_evidence(
|
||||
(*episode.attempts[-1].evidence_refs, *after_observation.evidence_refs)
|
||||
),
|
||||
uncertainty=max(assessed[0].uncertainty, assessed[-1].uncertainty),
|
||||
counterevidence=tuple(
|
||||
dict.fromkeys((*assessed[0].counterevidence, *assessed[-1].counterevidence))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def assess_practice_episode(
|
||||
prescription: PracticePrescription,
|
||||
episode: PracticeEpisodeInput,
|
||||
*,
|
||||
prior_state: CompetencyState | None = None,
|
||||
) -> PracticeEpisodeAssessment:
|
||||
"""전후 근거와 전이 조건을 분리해 한 연습 episode를 판정한다."""
|
||||
|
||||
if episode.prescription_id != prescription.prescription_id:
|
||||
raise ValueError("practice episode references a different prescription")
|
||||
if prior_state is not None and prior_state.competency_id != prescription.competency_id:
|
||||
raise ValueError("prior competency state does not match practice prescription")
|
||||
assessed = tuple(_assess_attempt(prescription, item) for item in episode.attempts)
|
||||
familiar_variants = {
|
||||
item.scenario_variant_id
|
||||
for item in assessed
|
||||
if item.scenario_novelty == "familiar"
|
||||
}
|
||||
familiar_templates = {
|
||||
item.utterance_template_id
|
||||
for item in assessed
|
||||
if item.scenario_novelty == "familiar" and item.utterance_template_id
|
||||
}
|
||||
|
||||
gated_attempts: list[PracticeAttemptAssessment] = []
|
||||
for item in assessed:
|
||||
blockers: list[str] = []
|
||||
if item.scenario_novelty == "unseen_transfer" and item.outcome == "passed":
|
||||
if item.scenario_variant_id in familiar_variants:
|
||||
blockers.append("transfer_variant_was_already_familiar")
|
||||
if item.utterance_template_id in familiar_templates:
|
||||
blockers.append("memorized_phrase_reused_in_transfer")
|
||||
if blockers:
|
||||
item = item.model_copy(
|
||||
update={
|
||||
"outcome": "needs_retry",
|
||||
"counterevidence": tuple(
|
||||
dict.fromkeys((*item.counterevidence, *blockers))
|
||||
),
|
||||
}
|
||||
)
|
||||
gated_attempts.append(item)
|
||||
final_attempts = tuple(gated_attempts)
|
||||
|
||||
familiar_passed = any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "familiar"
|
||||
for item in final_attempts
|
||||
)
|
||||
transfer_passed = any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
||||
for item in final_attempts
|
||||
)
|
||||
prior_familiar_demonstrations = (
|
||||
prior_state.familiar_demonstrations if prior_state is not None else 0
|
||||
)
|
||||
familiar_basis_ready = familiar_passed or prior_familiar_demonstrations > 0
|
||||
mastery_blockers: list[str] = []
|
||||
if not familiar_basis_ready:
|
||||
mastery_blockers.append("familiar_rehearsal_not_demonstrated")
|
||||
if not transfer_passed:
|
||||
mastery_blockers.append("unseen_transfer_not_verified")
|
||||
for item in final_attempts:
|
||||
mastery_blockers.extend(
|
||||
reason
|
||||
for reason in item.counterevidence
|
||||
if reason
|
||||
in {
|
||||
"transfer_variant_was_already_familiar",
|
||||
"memorized_phrase_reused_in_transfer",
|
||||
}
|
||||
)
|
||||
mastery_allowed = familiar_basis_ready and transfer_passed
|
||||
if mastery_allowed:
|
||||
progress = "mastered"
|
||||
event_names = (
|
||||
"practice.attempted",
|
||||
"transfer.verified",
|
||||
"practice.mastered",
|
||||
)
|
||||
elif familiar_passed:
|
||||
progress = "transfer_pending"
|
||||
event_names = ("practice.attempted",)
|
||||
else:
|
||||
progress = "practicing"
|
||||
event_names = ("practice.attempted",)
|
||||
|
||||
evidence_refs = _unique_evidence(
|
||||
ref for item in final_attempts for ref in item.evidence_refs
|
||||
)
|
||||
counterevidence = tuple(
|
||||
dict.fromkeys(
|
||||
reason for item in final_attempts for reason in item.counterevidence
|
||||
)
|
||||
)
|
||||
return PracticeEpisodeAssessment(
|
||||
event_names=event_names,
|
||||
episode_id=episode.episode_id,
|
||||
prescription_id=prescription.prescription_id,
|
||||
competency_id=prescription.competency_id,
|
||||
attempts=final_attempts,
|
||||
comparison=_comparison(prescription, episode, final_attempts),
|
||||
prior_familiar_demonstrations=prior_familiar_demonstrations,
|
||||
progress=progress,
|
||||
mastery_allowed=mastery_allowed,
|
||||
mastery_blockers=tuple(dict.fromkeys(mastery_blockers)),
|
||||
uncertainty=max(item.uncertainty for item in final_attempts),
|
||||
evidence_refs=evidence_refs,
|
||||
counterevidence=counterevidence,
|
||||
)
|
||||
|
||||
|
||||
def apply_episode_to_competency_graph(
|
||||
graph: CompetencyGraph,
|
||||
assessment: PracticeEpisodeAssessment,
|
||||
) -> CompetencyGraph:
|
||||
"""append-only attempt evidence를 반영한 새 역량 그래프 snapshot을 만든다."""
|
||||
|
||||
states = {item.competency_id: item for item in graph.states}
|
||||
previous = states.get(assessment.competency_id)
|
||||
if previous is None:
|
||||
raise ValueError("practice episode competency does not exist in graph")
|
||||
familiar_passes = sum(
|
||||
item.outcome == "passed" and item.scenario_novelty == "familiar"
|
||||
for item in assessment.attempts
|
||||
)
|
||||
transfer_passes = sum(
|
||||
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
||||
for item in assessment.attempts
|
||||
)
|
||||
observed_any = any(
|
||||
item.criterion_status == "observed" for item in assessment.attempts
|
||||
)
|
||||
if assessment.progress == "mastered":
|
||||
derived_band: CompetencyBand = "transfer_verified"
|
||||
elif familiar_passes:
|
||||
derived_band = "consistent_local"
|
||||
elif observed_any:
|
||||
derived_band = "developing"
|
||||
else:
|
||||
derived_band = "fragile"
|
||||
band = max(
|
||||
(previous.band, derived_band),
|
||||
key=lambda item: _BAND_RANK[item],
|
||||
)
|
||||
passed_familiar_difficulties = [
|
||||
item.difficulty_level
|
||||
for item in assessment.attempts
|
||||
if item.outcome == "passed" and item.scenario_novelty == "familiar"
|
||||
]
|
||||
highest_difficulty = max(
|
||||
[previous.highest_familiar_difficulty, *passed_familiar_difficulties]
|
||||
)
|
||||
evidence_refs = _unique_evidence(
|
||||
(*previous.evidence_refs, *assessment.evidence_refs)
|
||||
)
|
||||
counterevidence = tuple(
|
||||
dict.fromkeys((*previous.counterevidence, *assessment.counterevidence))
|
||||
)
|
||||
any_passed = familiar_passes + transfer_passes > 0
|
||||
forgetting_risk = (
|
||||
max(0.05, previous.forgetting_risk - 0.25)
|
||||
if any_passed
|
||||
else min(1.0, previous.forgetting_risk + 0.08)
|
||||
)
|
||||
states[assessment.competency_id] = CompetencyState(
|
||||
competency_id=previous.competency_id,
|
||||
band=band,
|
||||
forgetting_risk=round(forgetting_risk, 6),
|
||||
uncertainty=assessment.uncertainty,
|
||||
attempt_count=previous.attempt_count + len(assessment.attempts),
|
||||
familiar_demonstrations=previous.familiar_demonstrations + familiar_passes,
|
||||
unseen_transfer_demonstrations=(
|
||||
previous.unseen_transfer_demonstrations + transfer_passes
|
||||
),
|
||||
highest_familiar_difficulty=highest_difficulty,
|
||||
evidence_refs=evidence_refs,
|
||||
counterevidence=counterevidence,
|
||||
)
|
||||
return graph.model_copy(
|
||||
update={
|
||||
"states": tuple(states[item.competency_id] for item in graph.definitions)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def select_next_practice(
|
||||
graph: CompetencyGraph,
|
||||
prescriptions: Iterable[PracticePrescription],
|
||||
) -> CurriculumDecision:
|
||||
"""가장 약한 band를 먼저, 같은 band 안에서는 망각 위험을 먼저 선택한다."""
|
||||
|
||||
definitions = {item.competency_id: item for item in graph.definitions}
|
||||
states = {item.competency_id: item for item in graph.states}
|
||||
all_prescriptions = tuple(prescriptions)
|
||||
if len({item.prescription_id for item in all_prescriptions}) != len(
|
||||
all_prescriptions
|
||||
):
|
||||
raise ValueError("curriculum candidates require unique prescription ids")
|
||||
|
||||
eligible: list[PracticePrescription] = []
|
||||
blocked: list[str] = []
|
||||
for prescription in all_prescriptions:
|
||||
state = states.get(prescription.competency_id)
|
||||
definition = definitions.get(prescription.competency_id)
|
||||
if state is None or definition is None:
|
||||
blocked.append(f"{prescription.prescription_id}:competency_missing")
|
||||
continue
|
||||
unmet = [
|
||||
prerequisite
|
||||
for prerequisite in definition.prerequisite_ids
|
||||
if states[prerequisite].band != "transfer_verified"
|
||||
]
|
||||
if unmet:
|
||||
blocked.append(
|
||||
f"{prescription.prescription_id}:prerequisite_unverified:{','.join(unmet)}"
|
||||
)
|
||||
continue
|
||||
activity = prescription.activity
|
||||
repeated_easy = (
|
||||
state.familiar_demonstrations >= 2
|
||||
and activity.scenario_novelty == "familiar"
|
||||
and activity.difficulty_level <= state.highest_familiar_difficulty
|
||||
)
|
||||
if repeated_easy:
|
||||
blocked.append(f"{prescription.prescription_id}:easy_repeat_blocked")
|
||||
continue
|
||||
eligible.append(prescription)
|
||||
if not eligible:
|
||||
raise ValueError("no executable practice remains after curriculum gates")
|
||||
|
||||
def candidate_key(item: PracticePrescription) -> tuple[object, ...]:
|
||||
state = states[item.competency_id]
|
||||
transfer_fit = (
|
||||
0
|
||||
if state.band == "consistent_local"
|
||||
and item.activity.scenario_novelty == "unseen_transfer"
|
||||
else 1
|
||||
)
|
||||
return (
|
||||
_BAND_RANK[state.band],
|
||||
-state.forgetting_risk,
|
||||
transfer_fit,
|
||||
-item.activity.difficulty_level
|
||||
if state.familiar_demonstrations >= 1
|
||||
else item.activity.difficulty_level,
|
||||
item.competency_id,
|
||||
item.prescription_id,
|
||||
)
|
||||
|
||||
ordered = sorted(eligible, key=candidate_key)
|
||||
selected = ordered[0]
|
||||
state = states[selected.competency_id]
|
||||
return CurriculumDecision(
|
||||
selected_prescription_id=selected.prescription_id,
|
||||
competency_id=selected.competency_id,
|
||||
competency_band=state.band,
|
||||
forgetting_risk=state.forgetting_risk,
|
||||
mode=selected.activity.mode,
|
||||
selection_basis=(
|
||||
f"weakest_available_band:{state.band}",
|
||||
f"forgetting_risk:{state.forgetting_risk:.3f}",
|
||||
f"uncertainty:{state.uncertainty:.3f}",
|
||||
f"scenario_novelty:{selected.activity.scenario_novelty}",
|
||||
),
|
||||
deferred_prescription_ids=tuple(item.prescription_id for item in ordered[1:]),
|
||||
blocked_prescription_reasons=tuple(blocked),
|
||||
)
|
||||
|
||||
|
||||
def load_practice_benchmark(path: Path) -> PracticeBenchmarkPack:
|
||||
return PracticeBenchmarkPack.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def evaluate_practice_benchmark(pack: PracticeBenchmarkPack) -> dict[str, object]:
|
||||
"""실행 연결, 선택, 전이, 세 가지 보상 해킹 회귀를 따로 보고한다."""
|
||||
|
||||
episode_hits = final_band_hits = selection_hits = 0
|
||||
episode_total = 0
|
||||
prescription_count = target_count = 0
|
||||
reward_hacking_regressions = 0
|
||||
easy_repeat_regressions = 0
|
||||
memorized_phrase_false_mastery = 0
|
||||
premature_mastery_count = 0
|
||||
rows: list[dict[str, object]] = []
|
||||
|
||||
for case in pack.cases:
|
||||
prescriptions = prescribe_from_coaching_cards(case.coaching_cards)
|
||||
target_count += sum(len(card.targets) for card in case.coaching_cards)
|
||||
prescription_count += sum(item.can_launch for item in prescriptions)
|
||||
by_id = {item.prescription_id: item for item in prescriptions}
|
||||
graph = case.graph
|
||||
assessments: list[PracticeEpisodeAssessment] = []
|
||||
for episode in case.episodes:
|
||||
assessment = assess_practice_episode(
|
||||
by_id[episode.prescription_id], episode
|
||||
)
|
||||
assessments.append(assessment)
|
||||
graph = apply_episode_to_competency_graph(graph, assessment)
|
||||
actual_progress = tuple(item.progress for item in assessments)
|
||||
expected_progress = case.expected.episode_progress
|
||||
episode_hits += sum(
|
||||
actual == expected
|
||||
for actual, expected in zip(
|
||||
actual_progress, expected_progress, strict=False
|
||||
)
|
||||
)
|
||||
episode_total += max(len(actual_progress), len(expected_progress))
|
||||
final_state = next(
|
||||
item
|
||||
for item in graph.states
|
||||
if item.competency_id == case.expected.final_competency_id
|
||||
)
|
||||
final_band_match = final_state.band == case.expected.final_band
|
||||
final_band_hits += int(final_band_match)
|
||||
decision = select_next_practice(graph, prescriptions)
|
||||
selection_match = (
|
||||
decision.selected_prescription_id == case.expected.selected_prescription_id
|
||||
)
|
||||
selection_hits += int(selection_match)
|
||||
|
||||
if "reward_hacking" in case.tags:
|
||||
claimed_without_evidence = any(
|
||||
attempt.learner_claimed_success and assessed_attempt.outcome != "passed"
|
||||
for episode, assessment in zip(case.episodes, assessments, strict=False)
|
||||
for attempt, assessed_attempt in zip(
|
||||
episode.attempts, assessment.attempts, strict=False
|
||||
)
|
||||
)
|
||||
if claimed_without_evidence and final_state.band == "transfer_verified":
|
||||
reward_hacking_regressions += 1
|
||||
if "easy_repeat_hacking" in case.tags and not selection_match:
|
||||
easy_repeat_regressions += 1
|
||||
if "memorized_phrase_hacking" in case.tags and any(
|
||||
item.progress == "mastered" for item in assessments
|
||||
):
|
||||
memorized_phrase_false_mastery += 1
|
||||
for assessment in assessments:
|
||||
if assessment.progress == "mastered" and not any(
|
||||
item.outcome == "passed" and item.scenario_novelty == "unseen_transfer"
|
||||
for item in assessment.attempts
|
||||
):
|
||||
premature_mastery_count += 1
|
||||
|
||||
evidence_refs = _unique_evidence(
|
||||
ref for item in assessments for ref in item.evidence_refs
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"case_id": case.case_id,
|
||||
"actual_progress": list(actual_progress),
|
||||
"expected_progress": list(expected_progress),
|
||||
"actual_final_band": final_state.band,
|
||||
"expected_final_band": case.expected.final_band,
|
||||
"selected_prescription_id": decision.selected_prescription_id,
|
||||
"selection_match": selection_match,
|
||||
"uncertainty": max(
|
||||
[final_state.uncertainty]
|
||||
+ [item.uncertainty for item in assessments]
|
||||
),
|
||||
"evidence_refs": [item.ref_id for item in evidence_refs],
|
||||
"counterevidence": list(
|
||||
dict.fromkeys(
|
||||
reason
|
||||
for item in assessments
|
||||
for reason in item.counterevidence
|
||||
)
|
||||
),
|
||||
"blocked_prescription_reasons": list(
|
||||
decision.blocked_prescription_reasons
|
||||
),
|
||||
"tags": list(case.tags),
|
||||
}
|
||||
)
|
||||
|
||||
case_count = len(pack.cases)
|
||||
return {
|
||||
"schema_version": "vignette.deliberate-practice-benchmark-report.v1",
|
||||
"data_classification": "synthetic_educational",
|
||||
"clinical_claim_allowed": False,
|
||||
"benchmark_version": pack.version,
|
||||
"case_count": case_count,
|
||||
"actionable_prescription_coverage": (
|
||||
prescription_count / target_count if target_count else None
|
||||
),
|
||||
"episode_progress_accuracy": (
|
||||
episode_hits / episode_total if episode_total else 1.0
|
||||
),
|
||||
"final_band_accuracy": final_band_hits / case_count if case_count else None,
|
||||
"curriculum_selection_accuracy": selection_hits / case_count
|
||||
if case_count
|
||||
else None,
|
||||
"reward_hacking_regressions": reward_hacking_regressions,
|
||||
"easy_repeat_regressions": easy_repeat_regressions,
|
||||
"memorized_phrase_false_mastery": memorized_phrase_false_mastery,
|
||||
"premature_mastery_count": premature_mastery_count,
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
def render_practice_benchmark_report(report: dict[str, object]) -> str:
|
||||
return json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"apply_episode_to_competency_graph",
|
||||
"assess_practice_episode",
|
||||
"evaluate_practice_benchmark",
|
||||
"load_practice_benchmark",
|
||||
"prescribe_from_coaching_cards",
|
||||
"render_practice_benchmark_report",
|
||||
"select_next_practice",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue