{ "schema_version": "vignette.calibration-transfer-benchmark.v1", "version": "1.0.0", "data_classification": "synthetic_educational", "clinical_claim_allowed": false, "cases": [ { "case_id": "oas-g5-bench-001", "title_ko": "외부평가 전 잠금과 반복 블록 자기보정", "tags": ["post_reveal_contamination", "calibration_improvement"], "calibration_blocks": [ { "block_sequence": 1, "prediction_history": {"lock_id":"oas-g5-lock-a1","revisions":[{"prediction_id":"oas-g5-prediction-a1","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a1","scenario_variant_id":"cal-a1","phrase_family_id":"cal-phrase-a1","predicted_success_probability":0.95,"confidence":0.95,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":1,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":1,"external_reveal_sequence":2}, "observation": {"observation_id":"oas-g5-observation-a1","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a1","scenario_variant_id":"cal-a1","phrase_family_id":"cal-phrase-a1","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a1-turn"],"counterevidence":["impact_check_missing"],"revealed_sequence":2} }, { "block_sequence": 2, "prediction_history": {"lock_id":"oas-g5-lock-a2","revisions":[{"prediction_id":"oas-g5-prediction-a2","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a2","scenario_variant_id":"cal-a2","phrase_family_id":"cal-phrase-a2","predicted_success_probability":0.85,"confidence":0.9,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":3,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":3,"external_reveal_sequence":4}, "observation": {"observation_id":"oas-g5-observation-a2","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a2","scenario_variant_id":"cal-a2","phrase_family_id":"cal-phrase-a2","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a2-turn"],"counterevidence":["client_withdrew"],"revealed_sequence":4} }, { "block_sequence": 3, "prediction_history": {"lock_id":"oas-g5-lock-a3","revisions":[{"prediction_id":"oas-g5-prediction-a3","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a3","scenario_variant_id":"cal-a3","phrase_family_id":"cal-phrase-a3","predicted_success_probability":0.8,"confidence":0.85,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":5,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":5,"external_reveal_sequence":6}, "observation": {"observation_id":"oas-g5-observation-a3","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a3","scenario_variant_id":"cal-a3","phrase_family_id":"cal-phrase-a3","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a3-turn"],"counterevidence":[],"revealed_sequence":6} }, { "block_sequence": 4, "prediction_history": {"lock_id":"oas-g5-lock-a4","revisions":[{"prediction_id":"oas-g5-prediction-a4","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a4","scenario_variant_id":"cal-a4","phrase_family_id":"cal-phrase-a4","predicted_success_probability":0.35,"confidence":0.7,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":7,"revision_reason":"반대근거를 반영한 사전 예측"}],"locked_sequence":7,"external_reveal_sequence":8}, "observation": {"observation_id":"oas-g5-observation-a4","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a4","scenario_variant_id":"cal-a4","phrase_family_id":"cal-phrase-a4","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a4-turn"],"counterevidence":["premature_advice"],"revealed_sequence":8} }, { "block_sequence": 5, "prediction_history": {"lock_id":"oas-g5-lock-a5","revisions":[{"prediction_id":"oas-g5-prediction-a5","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a5","scenario_variant_id":"cal-a5","phrase_family_id":"cal-phrase-a5","predicted_success_probability":0.75,"confidence":0.75,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":9,"revision_reason":"행동 근거를 반영한 사전 예측"}],"locked_sequence":9,"external_reveal_sequence":10}, "observation": {"observation_id":"oas-g5-observation-a5","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a5","scenario_variant_id":"cal-a5","phrase_family_id":"cal-phrase-a5","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a5-turn"],"counterevidence":[],"revealed_sequence":10} }, { "block_sequence": 6, "prediction_history": {"lock_id":"oas-g5-lock-a6","revisions":[{"prediction_id":"oas-g5-prediction-a6","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a6","scenario_variant_id":"cal-a6","phrase_family_id":"cal-phrase-a6","predicted_success_probability":0.8,"confidence":0.8,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":11,"revision_reason":"행동과 반응 근거를 반영한 사전 예측"}],"locked_sequence":11,"external_reveal_sequence":12}, "observation": {"observation_id":"oas-g5-observation-a6","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a6","scenario_variant_id":"cal-a6","phrase_family_id":"cal-phrase-a6","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a6-turn"],"counterevidence":[],"revealed_sequence":12} } ], "expected": {"calibration_improved":{"competency.empathic-check":true},"transfer_verified":{},"drift_status":{}} }, { "case_id": "oas-g5-bench-002", "title_ko": "새 문구와 새 관계 맥락의 전이 성공", "tags": ["unseen_transfer_verified"], "transfer_suite": { "suite_id": "oas-g5-suite-true-transfer", "training_phrase_family_ids": ["training-empathy-phrase"], "trials": [ {"trial_id":"oas-g5-transfer-true-1","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-1","variation":{"context_variant":"학업 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"간접 거절","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"novel-1"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-1-behavior","true-1-response"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-true-2","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-2","variation":{"context_variant":"가족 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"직접 항의","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"novel-2"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-2-behavior","true-2-response"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-true-3","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-3","variation":{"context_variant":"학업 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"침묵 뒤 항의","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"novel-3"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-3-behavior","true-3-response"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-true-4","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-4","variation":{"context_variant":"가족 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"주제 전환","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"novel-4"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-4-behavior","true-4-response"],"counterevidence":[]} ] }, "expected": {"calibration_improved":{},"transfer_verified":{"competency.rupture-repair":true},"drift_status":{}} }, { "case_id": "oas-g5-bench-003", "title_ko": "성공처럼 보이는 문장 암기 전이 차단", "tags": ["memorized_phrase_transfer"], "transfer_suite": { "suite_id": "oas-g5-suite-memorized-transfer", "training_phrase_family_ids": ["memorized-repair-phrase"], "trials": [ {"trial_id":"oas-g5-transfer-mem-1","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-1","variation":{"context_variant":"학업 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"간접 거절","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-1"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-mem-2","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-2","variation":{"context_variant":"가족 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"직접 항의","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-2"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-mem-3","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-3","variation":{"context_variant":"학업 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"침묵 뒤 항의","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-3"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-mem-4","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-4","variation":{"context_variant":"가족 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"주제 전환","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-4"],"counterevidence":[]} ] }, "expected": {"calibration_improved":{},"transfer_verified":{"competency.rupture-repair":false},"drift_status":{}} }, { "case_id": "oas-g5-bench-004", "title_ko": "충분한 합성 subgroup 표본의 평가 드리프트 신호", "tags": ["synthetic_subgroup_drift"], "transfer_suite": { "suite_id": "oas-g5-suite-subgroup-drift", "training_phrase_family_ids": ["training-open-question"], "trials": [ {"trial_id":"oas-g5-transfer-drift-a1","competency_id":"competency.open-question","scenario_variant_id":"drift-a1","variation":{"context_variant":"학업","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"서술형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"drift-a1"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a1"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-drift-a2","competency_id":"competency.open-question","scenario_variant_id":"drift-a2","variation":{"context_variant":"가족","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"우회형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-home","phrase_family_id":"drift-a2"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a2"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-drift-a3","competency_id":"competency.open-question","scenario_variant_id":"drift-a3","variation":{"context_variant":"학업","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"단답형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"drift-a3"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a3"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-drift-b1","competency_id":"competency.open-question","scenario_variant_id":"drift-b1","variation":{"context_variant":"가족","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"서술형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"drift-b1"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-b1"],"counterevidence":[]}, {"trial_id":"oas-g5-transfer-drift-b2","competency_id":"competency.open-question","scenario_variant_id":"drift-b2","variation":{"context_variant":"학업","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"우회형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-school","phrase_family_id":"drift-b2"},"status":"failed","uncertainty":0.2,"evidence_refs":["drift-b2"],"counterevidence":["closed_question" ]}, {"trial_id":"oas-g5-transfer-drift-b3","competency_id":"competency.open-question","scenario_variant_id":"drift-b3","variation":{"context_variant":"가족","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"단답형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"drift-b3"},"status":"failed","uncertainty":0.2,"evidence_refs":["drift-b3"],"counterevidence":["leading_question"]} ] }, "expected": {"calibration_improved":{},"transfer_verified":{"competency.open-question":false},"drift_status":{"competency.open-question":"drift_flagged"}} } ] }