"""G4 의도적 수련 처방·전이 게이트·결정론 커리큘럼 코어. 학습자의 자기 성공 주장이나 외부 보상값을 사용하지 않는다. 관찰 가능한 한 행동, 내담자 후속 반응, 익숙한 장면 재현, 미지 사례 전이를 서로 분리한다. """ from __future__ import annotations import json from pathlib import Path from typing import Iterable from ..contracts.deliberate_practice import ( BeforeAfterComparison, CoachingCard, CompetencyBand, CompetencyGraph, CompetencyState, CurriculumDecision, PracticeAttemptAssessment, PracticeAttemptObservation, PracticeBenchmarkPack, PracticeEpisodeAssessment, PracticeEpisodeInput, PracticeEvidenceRef, PracticePrescription, ) _ENGAGED_RESPONSES = frozenset({"engaged", "explicit_alignment"}) _BAND_RANK: dict[CompetencyBand, int] = { "unassessed": 0, "fragile": 1, "developing": 2, "consistent_local": 3, "transfer_verified": 4, } def _unique_evidence( refs: Iterable[PracticeEvidenceRef], ) -> tuple[PracticeEvidenceRef, ...]: output: list[PracticeEvidenceRef] = [] seen: set[str] = set() for ref in refs: if ref.ref_id in seen: continue seen.add(ref.ref_id) output.append(ref) return tuple(output) def prescribe_from_coaching_cards( cards: Iterable[CoachingCard], ) -> tuple[PracticePrescription, ...]: """각 코칭 카드의 각 목표를 실행 가능한 원자적 처방으로 투영한다.""" prescriptions: list[PracticePrescription] = [] card_ids: set[str] = set() prescription_ids: set[str] = set() for card in cards: if card.card_id in card_ids: raise ValueError(f"duplicate coaching card id: {card.card_id}") card_ids.add(card.card_id) for target in card.targets: if target.prescription_id in prescription_ids: raise ValueError( f"duplicate practice prescription id: {target.prescription_id}" ) prescription_ids.add(target.prescription_id) prescriptions.append( PracticePrescription( prescription_id=target.prescription_id, coaching_card_id=card.card_id, scene_id=card.scene_id, competency_id=target.competency_id, criterion_id=target.criterion_id, observable_behavior=target.observable_behavior, activity=target.activity, evidence_refs=card.evidence_refs, source_refs=card.source_refs, uncertainty=card.uncertainty, counterevidence=card.counterevidence, ) ) if card_ids != {item.coaching_card_id for item in prescriptions}: raise ValueError("every coaching card must produce an actionable prescription") return tuple(prescriptions) def _assess_attempt( prescription: PracticePrescription, attempt: PracticeAttemptObservation, ) -> PracticeAttemptAssessment: if attempt.prescription_id != prescription.prescription_id: raise ValueError("practice attempt references a different prescription") if attempt.competency_id != prescription.competency_id: raise ValueError("practice attempt targets a different competency") if attempt.criterion.criterion_id != prescription.criterion_id: raise ValueError("practice attempt evaluates a different atomic criterion") evidence_refs = _unique_evidence( (*attempt.evidence_refs, *attempt.criterion.evidence_refs) ) counterevidence = list( dict.fromkeys((*attempt.counterevidence, *attempt.criterion.counterevidence)) ) independent_observation = ( attempt.criterion.source_kind in {"model_inferred", "human_rated", "observed_runtime"} and attempt.criterion.perspective in {"independent_observer", "supervisor_human", "runtime_observation"} ) if attempt.criterion.status != "error" and not independent_observation: outcome = "insufficient_evidence" counterevidence.append("independent_observer_required") elif attempt.criterion.status == "error": outcome = "insufficient_evidence" counterevidence.append( f"criterion_evaluation_failed:{attempt.criterion.error_code or 'unknown'}" ) else: has_voice_evidence = any(item.kind == "voice_feature" for item in evidence_refs) voice_ready = prescription.activity.mode != "voice_retry" or has_voice_evidence impact_ready = attempt.client_response in _ENGAGED_RESPONSES uncertainty_ready = ( max(attempt.uncertainty, attempt.criterion.uncertainty) <= 0.5 ) if ( attempt.criterion.status == "observed" and impact_ready and uncertainty_ready and voice_ready ): outcome = "passed" else: outcome = "needs_retry" if attempt.criterion.status != "observed": counterevidence.append("target_behavior_not_observed") if not impact_ready: counterevidence.append("client_response_does_not_support_effect") if not uncertainty_ready: counterevidence.append("attempt_uncertainty_above_acceptance_boundary") if not voice_ready: counterevidence.append("voice_retry_missing_voice_feature_evidence") if attempt.learner_claimed_success and outcome != "passed": counterevidence.append( "learner_success_claim_not_supported_by_attempt_evidence" ) return PracticeAttemptAssessment( attempt_id=attempt.attempt_id, outcome=outcome, criterion_status=attempt.criterion.status, client_response=attempt.client_response, scenario_novelty=attempt.scenario_novelty, scenario_variant_id=attempt.scenario_variant_id, difficulty_level=attempt.difficulty_level, utterance_template_id=attempt.utterance_template_id, uncertainty=max(attempt.uncertainty, attempt.criterion.uncertainty), evidence_refs=evidence_refs, counterevidence=tuple(dict.fromkeys(counterevidence)), ) def _comparison( prescription: PracticePrescription, episode: PracticeEpisodeInput, assessed: tuple[PracticeAttemptAssessment, ...], ) -> BeforeAfterComparison: before_observation = episode.attempts[0].criterion after_observation = episode.attempts[-1].criterion if "error" in {before_observation.status, after_observation.status}: change = "inconclusive" elif ( before_observation.status == "not_observed" and after_observation.status == "observed" ): change = "improved" elif ( before_observation.status == "observed" and after_observation.status == "not_observed" ): change = "regressed" else: change = "unchanged" return BeforeAfterComparison( criterion_id=prescription.criterion_id, before_attempt_id=assessed[0].attempt_id, after_attempt_id=assessed[-1].attempt_id, change=change, before_status=before_observation.status, after_status=after_observation.status, before_evidence_refs=_unique_evidence( (*episode.attempts[0].evidence_refs, *before_observation.evidence_refs) ), after_evidence_refs=_unique_evidence( (*episode.attempts[-1].evidence_refs, *after_observation.evidence_refs) ), uncertainty=max(assessed[0].uncertainty, assessed[-1].uncertainty), counterevidence=tuple( dict.fromkeys((*assessed[0].counterevidence, *assessed[-1].counterevidence)) ), ) def assess_practice_episode( prescription: PracticePrescription, episode: PracticeEpisodeInput, *, prior_state: CompetencyState | None = None, ) -> PracticeEpisodeAssessment: """전후 근거와 전이 조건을 분리해 한 연습 episode를 판정한다.""" if episode.prescription_id != prescription.prescription_id: raise ValueError("practice episode references a different prescription") if prior_state is not None and prior_state.competency_id != prescription.competency_id: raise ValueError("prior competency state does not match practice prescription") assessed = tuple(_assess_attempt(prescription, item) for item in episode.attempts) familiar_variants = { item.scenario_variant_id for item in assessed if item.scenario_novelty == "familiar" } familiar_templates = { item.utterance_template_id for item in assessed if item.scenario_novelty == "familiar" and item.utterance_template_id } gated_attempts: list[PracticeAttemptAssessment] = [] for item in assessed: blockers: list[str] = [] if item.scenario_novelty == "unseen_transfer" and item.outcome == "passed": if item.scenario_variant_id in familiar_variants: blockers.append("transfer_variant_was_already_familiar") if item.utterance_template_id in familiar_templates: blockers.append("memorized_phrase_reused_in_transfer") if blockers: item = item.model_copy( update={ "outcome": "needs_retry", "counterevidence": tuple( dict.fromkeys((*item.counterevidence, *blockers)) ), } ) gated_attempts.append(item) final_attempts = tuple(gated_attempts) familiar_passed = any( item.outcome == "passed" and item.scenario_novelty == "familiar" for item in final_attempts ) transfer_passed = any( item.outcome == "passed" and item.scenario_novelty == "unseen_transfer" for item in final_attempts ) prior_familiar_demonstrations = ( prior_state.familiar_demonstrations if prior_state is not None else 0 ) familiar_basis_ready = familiar_passed or prior_familiar_demonstrations > 0 mastery_blockers: list[str] = [] if not familiar_basis_ready: mastery_blockers.append("familiar_rehearsal_not_demonstrated") if not transfer_passed: mastery_blockers.append("unseen_transfer_not_verified") for item in final_attempts: mastery_blockers.extend( reason for reason in item.counterevidence if reason in { "transfer_variant_was_already_familiar", "memorized_phrase_reused_in_transfer", } ) mastery_allowed = familiar_basis_ready and transfer_passed if mastery_allowed: progress = "mastered" event_names = ( "practice.attempted", "transfer.verified", "practice.mastered", ) elif familiar_passed: progress = "transfer_pending" event_names = ("practice.attempted",) else: progress = "practicing" event_names = ("practice.attempted",) evidence_refs = _unique_evidence( ref for item in final_attempts for ref in item.evidence_refs ) counterevidence = tuple( dict.fromkeys( reason for item in final_attempts for reason in item.counterevidence ) ) return PracticeEpisodeAssessment( event_names=event_names, episode_id=episode.episode_id, prescription_id=prescription.prescription_id, competency_id=prescription.competency_id, attempts=final_attempts, comparison=_comparison(prescription, episode, final_attempts), prior_familiar_demonstrations=prior_familiar_demonstrations, progress=progress, mastery_allowed=mastery_allowed, mastery_blockers=tuple(dict.fromkeys(mastery_blockers)), uncertainty=max(item.uncertainty for item in final_attempts), evidence_refs=evidence_refs, counterevidence=counterevidence, ) def apply_episode_to_competency_graph( graph: CompetencyGraph, assessment: PracticeEpisodeAssessment, ) -> CompetencyGraph: """append-only attempt evidence를 반영한 새 역량 그래프 snapshot을 만든다.""" states = {item.competency_id: item for item in graph.states} previous = states.get(assessment.competency_id) if previous is None: raise ValueError("practice episode competency does not exist in graph") familiar_passes = sum( item.outcome == "passed" and item.scenario_novelty == "familiar" for item in assessment.attempts ) transfer_passes = sum( item.outcome == "passed" and item.scenario_novelty == "unseen_transfer" for item in assessment.attempts ) observed_any = any( item.criterion_status == "observed" for item in assessment.attempts ) if assessment.progress == "mastered": derived_band: CompetencyBand = "transfer_verified" elif familiar_passes: derived_band = "consistent_local" elif observed_any: derived_band = "developing" else: derived_band = "fragile" band = max( (previous.band, derived_band), key=lambda item: _BAND_RANK[item], ) passed_familiar_difficulties = [ item.difficulty_level for item in assessment.attempts if item.outcome == "passed" and item.scenario_novelty == "familiar" ] highest_difficulty = max( [previous.highest_familiar_difficulty, *passed_familiar_difficulties] ) evidence_refs = _unique_evidence( (*previous.evidence_refs, *assessment.evidence_refs) ) counterevidence = tuple( dict.fromkeys((*previous.counterevidence, *assessment.counterevidence)) ) any_passed = familiar_passes + transfer_passes > 0 forgetting_risk = ( max(0.05, previous.forgetting_risk - 0.25) if any_passed else min(1.0, previous.forgetting_risk + 0.08) ) states[assessment.competency_id] = CompetencyState( competency_id=previous.competency_id, band=band, forgetting_risk=round(forgetting_risk, 6), uncertainty=assessment.uncertainty, attempt_count=previous.attempt_count + len(assessment.attempts), familiar_demonstrations=previous.familiar_demonstrations + familiar_passes, unseen_transfer_demonstrations=( previous.unseen_transfer_demonstrations + transfer_passes ), highest_familiar_difficulty=highest_difficulty, evidence_refs=evidence_refs, counterevidence=counterevidence, ) return graph.model_copy( update={ "states": tuple(states[item.competency_id] for item in graph.definitions) } ) def select_next_practice( graph: CompetencyGraph, prescriptions: Iterable[PracticePrescription], ) -> CurriculumDecision: """가장 약한 band를 먼저, 같은 band 안에서는 망각 위험을 먼저 선택한다.""" definitions = {item.competency_id: item for item in graph.definitions} states = {item.competency_id: item for item in graph.states} all_prescriptions = tuple(prescriptions) if len({item.prescription_id for item in all_prescriptions}) != len( all_prescriptions ): raise ValueError("curriculum candidates require unique prescription ids") eligible: list[PracticePrescription] = [] blocked: list[str] = [] for prescription in all_prescriptions: state = states.get(prescription.competency_id) definition = definitions.get(prescription.competency_id) if state is None or definition is None: blocked.append(f"{prescription.prescription_id}:competency_missing") continue unmet = [ prerequisite for prerequisite in definition.prerequisite_ids if states[prerequisite].band != "transfer_verified" ] if unmet: blocked.append( f"{prescription.prescription_id}:prerequisite_unverified:{','.join(unmet)}" ) continue activity = prescription.activity repeated_easy = ( state.familiar_demonstrations >= 2 and activity.scenario_novelty == "familiar" and activity.difficulty_level <= state.highest_familiar_difficulty ) if repeated_easy: blocked.append(f"{prescription.prescription_id}:easy_repeat_blocked") continue eligible.append(prescription) if not eligible: raise ValueError("no executable practice remains after curriculum gates") def candidate_key(item: PracticePrescription) -> tuple[object, ...]: state = states[item.competency_id] transfer_fit = ( 0 if state.band == "consistent_local" and item.activity.scenario_novelty == "unseen_transfer" else 1 ) return ( _BAND_RANK[state.band], -state.forgetting_risk, transfer_fit, -item.activity.difficulty_level if state.familiar_demonstrations >= 1 else item.activity.difficulty_level, item.competency_id, item.prescription_id, ) ordered = sorted(eligible, key=candidate_key) selected = ordered[0] state = states[selected.competency_id] return CurriculumDecision( selected_prescription_id=selected.prescription_id, competency_id=selected.competency_id, competency_band=state.band, forgetting_risk=state.forgetting_risk, mode=selected.activity.mode, selection_basis=( f"weakest_available_band:{state.band}", f"forgetting_risk:{state.forgetting_risk:.3f}", f"uncertainty:{state.uncertainty:.3f}", f"scenario_novelty:{selected.activity.scenario_novelty}", ), deferred_prescription_ids=tuple(item.prescription_id for item in ordered[1:]), blocked_prescription_reasons=tuple(blocked), ) def load_practice_benchmark(path: Path) -> PracticeBenchmarkPack: return PracticeBenchmarkPack.model_validate_json(path.read_text(encoding="utf-8")) def evaluate_practice_benchmark(pack: PracticeBenchmarkPack) -> dict[str, object]: """실행 연결, 선택, 전이, 세 가지 보상 해킹 회귀를 따로 보고한다.""" episode_hits = final_band_hits = selection_hits = 0 episode_total = 0 prescription_count = target_count = 0 reward_hacking_regressions = 0 easy_repeat_regressions = 0 memorized_phrase_false_mastery = 0 premature_mastery_count = 0 rows: list[dict[str, object]] = [] for case in pack.cases: prescriptions = prescribe_from_coaching_cards(case.coaching_cards) target_count += sum(len(card.targets) for card in case.coaching_cards) prescription_count += sum(item.can_launch for item in prescriptions) by_id = {item.prescription_id: item for item in prescriptions} graph = case.graph assessments: list[PracticeEpisodeAssessment] = [] for episode in case.episodes: assessment = assess_practice_episode( by_id[episode.prescription_id], episode ) assessments.append(assessment) graph = apply_episode_to_competency_graph(graph, assessment) actual_progress = tuple(item.progress for item in assessments) expected_progress = case.expected.episode_progress episode_hits += sum( actual == expected for actual, expected in zip( actual_progress, expected_progress, strict=False ) ) episode_total += max(len(actual_progress), len(expected_progress)) final_state = next( item for item in graph.states if item.competency_id == case.expected.final_competency_id ) final_band_match = final_state.band == case.expected.final_band final_band_hits += int(final_band_match) decision = select_next_practice(graph, prescriptions) selection_match = ( decision.selected_prescription_id == case.expected.selected_prescription_id ) selection_hits += int(selection_match) if "reward_hacking" in case.tags: claimed_without_evidence = any( attempt.learner_claimed_success and assessed_attempt.outcome != "passed" for episode, assessment in zip(case.episodes, assessments, strict=False) for attempt, assessed_attempt in zip( episode.attempts, assessment.attempts, strict=False ) ) if claimed_without_evidence and final_state.band == "transfer_verified": reward_hacking_regressions += 1 if "easy_repeat_hacking" in case.tags and not selection_match: easy_repeat_regressions += 1 if "memorized_phrase_hacking" in case.tags and any( item.progress == "mastered" for item in assessments ): memorized_phrase_false_mastery += 1 for assessment in assessments: if assessment.progress == "mastered" and not any( item.outcome == "passed" and item.scenario_novelty == "unseen_transfer" for item in assessment.attempts ): premature_mastery_count += 1 evidence_refs = _unique_evidence( ref for item in assessments for ref in item.evidence_refs ) rows.append( { "case_id": case.case_id, "actual_progress": list(actual_progress), "expected_progress": list(expected_progress), "actual_final_band": final_state.band, "expected_final_band": case.expected.final_band, "selected_prescription_id": decision.selected_prescription_id, "selection_match": selection_match, "uncertainty": max( [final_state.uncertainty] + [item.uncertainty for item in assessments] ), "evidence_refs": [item.ref_id for item in evidence_refs], "counterevidence": list( dict.fromkeys( reason for item in assessments for reason in item.counterevidence ) ), "blocked_prescription_reasons": list( decision.blocked_prescription_reasons ), "tags": list(case.tags), } ) case_count = len(pack.cases) return { "schema_version": "vignette.deliberate-practice-benchmark-report.v1", "data_classification": "synthetic_educational", "clinical_claim_allowed": False, "benchmark_version": pack.version, "case_count": case_count, "actionable_prescription_coverage": ( prescription_count / target_count if target_count else None ), "episode_progress_accuracy": ( episode_hits / episode_total if episode_total else 1.0 ), "final_band_accuracy": final_band_hits / case_count if case_count else None, "curriculum_selection_accuracy": selection_hits / case_count if case_count else None, "reward_hacking_regressions": reward_hacking_regressions, "easy_repeat_regressions": easy_repeat_regressions, "memorized_phrase_false_mastery": memorized_phrase_false_mastery, "premature_mastery_count": premature_mastery_count, "rows": rows, } def render_practice_benchmark_report(report: dict[str, object]) -> str: return json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) __all__ = [ "apply_episode_to_competency_graph", "assess_practice_episode", "evaluate_practice_benchmark", "load_practice_benchmark", "prescribe_from_coaching_cards", "render_practice_benchmark_report", "select_next_practice", ]