yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
vignette/apps/api/app/data/calibration_transfer_benchmark_g5.v1.json
Yun Chan 16e791e044 G0~G8 성과·동맹 측정 OS 작업 일괄 고정 
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본
폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을
git 이력으로 고정하는 것이 목적이다.

- contracts/routes/services: measurement, outcome_trajectory, rupture_repair,
  deliberate_practice, calibration_transfer, supervision_research,
  multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트
- infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행)
- apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가
- docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG
- scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트

engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
2026-08-08 01:30:53 +09:00

96 lines
14 KiB
JSON
Raw Blame History

{
"schema_version": "vignette.calibration-transfer-benchmark.v1",
"version": "1.0.0",
"data_classification": "synthetic_educational",
"clinical_claim_allowed": false,
"cases": [
{
"case_id": "oas-g5-bench-001",
"title_ko": "외부평가 전 잠금과 반복 블록 자기보정",
"tags": ["post_reveal_contamination", "calibration_improvement"],
"calibration_blocks": [
{
"block_sequence": 1,
"prediction_history": {"lock_id":"oas-g5-lock-a1","revisions":[{"prediction_id":"oas-g5-prediction-a1","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a1","scenario_variant_id":"cal-a1","phrase_family_id":"cal-phrase-a1","predicted_success_probability":0.95,"confidence":0.95,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":1,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":1,"external_reveal_sequence":2},
"observation": {"observation_id":"oas-g5-observation-a1","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a1","scenario_variant_id":"cal-a1","phrase_family_id":"cal-phrase-a1","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a1-turn"],"counterevidence":["impact_check_missing"],"revealed_sequence":2}
},
{
"block_sequence": 2,
"prediction_history": {"lock_id":"oas-g5-lock-a2","revisions":[{"prediction_id":"oas-g5-prediction-a2","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a2","scenario_variant_id":"cal-a2","phrase_family_id":"cal-phrase-a2","predicted_success_probability":0.85,"confidence":0.9,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":3,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":3,"external_reveal_sequence":4},
"observation": {"observation_id":"oas-g5-observation-a2","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a2","scenario_variant_id":"cal-a2","phrase_family_id":"cal-phrase-a2","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a2-turn"],"counterevidence":["client_withdrew"],"revealed_sequence":4}
},
{
"block_sequence": 3,
"prediction_history": {"lock_id":"oas-g5-lock-a3","revisions":[{"prediction_id":"oas-g5-prediction-a3","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a3","scenario_variant_id":"cal-a3","phrase_family_id":"cal-phrase-a3","predicted_success_probability":0.8,"confidence":0.85,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":5,"revision_reason":"외부평가 전 최초 예측"}],"locked_sequence":5,"external_reveal_sequence":6},
"observation": {"observation_id":"oas-g5-observation-a3","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a3","scenario_variant_id":"cal-a3","phrase_family_id":"cal-phrase-a3","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a3-turn"],"counterevidence":[],"revealed_sequence":6}
},
{
"block_sequence": 4,
"prediction_history": {"lock_id":"oas-g5-lock-a4","revisions":[{"prediction_id":"oas-g5-prediction-a4","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a4","scenario_variant_id":"cal-a4","phrase_family_id":"cal-phrase-a4","predicted_success_probability":0.35,"confidence":0.7,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":7,"revision_reason":"반대근거를 반영한 사전 예측"}],"locked_sequence":7,"external_reveal_sequence":8},
"observation": {"observation_id":"oas-g5-observation-a4","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a4","scenario_variant_id":"cal-a4","phrase_family_id":"cal-phrase-a4","status":"failed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a4-turn"],"counterevidence":["premature_advice"],"revealed_sequence":8}
},
{
"block_sequence": 5,
"prediction_history": {"lock_id":"oas-g5-lock-a5","revisions":[{"prediction_id":"oas-g5-prediction-a5","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a5","scenario_variant_id":"cal-a5","phrase_family_id":"cal-phrase-a5","predicted_success_probability":0.75,"confidence":0.75,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":9,"revision_reason":"행동 근거를 반영한 사전 예측"}],"locked_sequence":9,"external_reveal_sequence":10},
"observation": {"observation_id":"oas-g5-observation-a5","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a5","scenario_variant_id":"cal-a5","phrase_family_id":"cal-phrase-a5","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a5-turn"],"counterevidence":[],"revealed_sequence":10}
},
{
"block_sequence": 6,
"prediction_history": {"lock_id":"oas-g5-lock-a6","revisions":[{"prediction_id":"oas-g5-prediction-a6","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a6","scenario_variant_id":"cal-a6","phrase_family_id":"cal-phrase-a6","predicted_success_probability":0.8,"confidence":0.8,"revision_no":1,"supersedes_prediction_id":null,"recorded_sequence":11,"revision_reason":"행동과 반응 근거를 반영한 사전 예측"}],"locked_sequence":11,"external_reveal_sequence":12},
"observation": {"observation_id":"oas-g5-observation-a6","competency_id":"competency.empathic-check","practice_block_id":"oas-g5-block-a6","scenario_variant_id":"cal-a6","phrase_family_id":"cal-phrase-a6","status":"passed","source_kind":"human_rated","perspective":"supervisor_human","model_run_id":null,"uncertainty":0.1,"evidence_refs":["cal-a6-turn"],"counterevidence":[],"revealed_sequence":12}
}
],
"expected": {"calibration_improved":{"competency.empathic-check":true},"transfer_verified":{},"drift_status":{}}
},
{
"case_id": "oas-g5-bench-002",
"title_ko": "새 문구와 새 관계 맥락의 전이 성공",
"tags": ["unseen_transfer_verified"],
"transfer_suite": {
"suite_id": "oas-g5-suite-true-transfer",
"training_phrase_family_ids": ["training-empathy-phrase"],
"trials": [
{"trial_id":"oas-g5-transfer-true-1","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-1","variation":{"context_variant":"학업 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"간접 거절","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"novel-1"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-1-behavior","true-1-response"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-true-2","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-2","variation":{"context_variant":"가족 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"직접 항의","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"novel-2"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-2-behavior","true-2-response"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-true-3","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-3","variation":{"context_variant":"학업 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"침묵 뒤 항의","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"novel-3"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-3-behavior","true-3-response"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-true-4","competency_id":"competency.rupture-repair","scenario_variant_id":"unseen-4","variation":{"context_variant":"가족 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"주제 전환","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"novel-4"},"status":"passed","uncertainty":0.2,"evidence_refs":["true-4-behavior","true-4-response"],"counterevidence":[]}
]
},
"expected": {"calibration_improved":{},"transfer_verified":{"competency.rupture-repair":true},"drift_status":{}}
},
{
"case_id": "oas-g5-bench-003",
"title_ko": "성공처럼 보이는 문장 암기 전이 차단",
"tags": ["memorized_phrase_transfer"],
"transfer_suite": {
"suite_id": "oas-g5-suite-memorized-transfer",
"training_phrase_family_ids": ["memorized-repair-phrase"],
"trials": [
{"trial_id":"oas-g5-transfer-mem-1","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-1","variation":{"context_variant":"학업 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"간접 거절","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-1"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-mem-2","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-2","variation":{"context_variant":"가족 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"직접 항의","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-2"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-mem-3","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-3","variation":{"context_variant":"학업 갈등","relationship_style":"confrontational","difficulty_level":4,"expression_variant":"침묵 뒤 항의","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-3"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-mem-4","competency_id":"competency.rupture-repair","scenario_variant_id":"mem-4","variation":{"context_variant":"가족 갈등","relationship_style":"withdrawn","difficulty_level":3,"expression_variant":"주제 전환","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"memorized-repair-phrase"},"status":"passed","uncertainty":0.2,"evidence_refs":["mem-4"],"counterevidence":[]}
]
},
"expected": {"calibration_improved":{},"transfer_verified":{"competency.rupture-repair":false},"drift_status":{}}
},
{
"case_id": "oas-g5-bench-004",
"title_ko": "충분한 합성 subgroup 표본의 평가 드리프트 신호",
"tags": ["synthetic_subgroup_drift"],
"transfer_suite": {
"suite_id": "oas-g5-suite-subgroup-drift",
"training_phrase_family_ids": ["training-open-question"],
"trials": [
{"trial_id":"oas-g5-transfer-drift-a1","competency_id":"competency.open-question","scenario_variant_id":"drift-a1","variation":{"context_variant":"학업","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"서술형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"drift-a1"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a1"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-drift-a2","competency_id":"competency.open-question","scenario_variant_id":"drift-a2","variation":{"context_variant":"가족","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"우회형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-home","phrase_family_id":"drift-a2"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a2"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-drift-a3","competency_id":"competency.open-question","scenario_variant_id":"drift-a3","variation":{"context_variant":"학업","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"단답형","synthetic_subgroup":"synthetic-a","scenario_family_id":"family-school","phrase_family_id":"drift-a3"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-a3"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-drift-b1","competency_id":"competency.open-question","scenario_variant_id":"drift-b1","variation":{"context_variant":"가족","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"서술형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"drift-b1"},"status":"passed","uncertainty":0.2,"evidence_refs":["drift-b1"],"counterevidence":[]},
{"trial_id":"oas-g5-transfer-drift-b2","competency_id":"competency.open-question","scenario_variant_id":"drift-b2","variation":{"context_variant":"학업","relationship_style":"ambivalent","difficulty_level":4,"expression_variant":"우회형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-school","phrase_family_id":"drift-b2"},"status":"failed","uncertainty":0.2,"evidence_refs":["drift-b2"],"counterevidence":["closed_question" ]},
{"trial_id":"oas-g5-transfer-drift-b3","competency_id":"competency.open-question","scenario_variant_id":"drift-b3","variation":{"context_variant":"가족","relationship_style":"collaborative","difficulty_level":2,"expression_variant":"단답형 변주","synthetic_subgroup":"synthetic-b","scenario_family_id":"family-home","phrase_family_id":"drift-b3"},"status":"failed","uncertainty":0.2,"evidence_refs":["drift-b3"],"counterevidence":["leading_question"]}
]
},
"expected": {"calibration_improved":{},"transfer_verified":{"competency.open-question":false},"drift_status":{"competency.open-question":"drift_flagged"}}
}
]
}
Reference in a new issue View git blame Copy permalink