16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
390 lines
15 KiB
Python
390 lines
15 KiB
Python
"""Alliance measurement benchmark calibration and drift comparison.
|
|
|
|
This module deliberately compares individual dimensions. It never creates an
|
|
alliance total, and missing/error predictions count as missing evidence rather
|
|
than a neutral score.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Iterable, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
|
from ..contracts.measurement import (
|
|
AllianceDimension,
|
|
BenchmarkCase,
|
|
)
|
|
|
|
|
|
AllianceCalibrationPerspective = Literal[
|
|
"client_agent_report",
|
|
"independent_observer",
|
|
]
|
|
|
|
|
|
def _utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class AllianceCalibrationPrediction(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
case_id: str
|
|
perspective: AllianceCalibrationPerspective
|
|
dimension: AllianceDimension
|
|
value: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
evidence_turn_indices: tuple[int, ...] = ()
|
|
status: Literal["ready", "degraded", "error"] = "ready"
|
|
error_code: str | None = None
|
|
provider: str | None = Field(default=None, max_length=120)
|
|
model: str | None = Field(default=None, max_length=240)
|
|
prompt_bundle_version: str | None = Field(default=None, max_length=40)
|
|
model_run_id: str | None = Field(default=None, max_length=80)
|
|
attempt_count: int = Field(default=1, ge=1, le=8)
|
|
prior_model_run_ids: tuple[str, ...] = ()
|
|
prior_error_codes: tuple[str, ...] = ()
|
|
prompt_bundle_hash: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
|
input_evidence_hash: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
|
|
|
@model_validator(mode="after")
|
|
def keep_failures_scoreless(self) -> "AllianceCalibrationPrediction":
|
|
if self.status == "ready" and self.value is None:
|
|
raise ValueError("ready calibration predictions require a value")
|
|
if self.status != "ready" and self.value is not None:
|
|
raise ValueError("failed calibration predictions cannot carry a value")
|
|
if self.status == "error" and not self.error_code:
|
|
raise ValueError("error calibration predictions require error_code")
|
|
if len(set(self.evidence_turn_indices)) != len(self.evidence_turn_indices):
|
|
raise ValueError("calibration evidence indices must be unique")
|
|
if self.evidence_turn_indices and min(self.evidence_turn_indices) < 0:
|
|
raise ValueError("calibration evidence indices must be non-negative")
|
|
return self
|
|
|
|
|
|
class AllianceCalibrationSnapshot(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
schema_version: Literal["alliance-calibration-snapshot.v1"] = (
|
|
"alliance-calibration-snapshot.v1"
|
|
)
|
|
run_id: str = Field(min_length=1, max_length=160)
|
|
provider: str = Field(min_length=1, max_length=120)
|
|
model: str = Field(min_length=1, max_length=240)
|
|
prompt_bundle_version: str = Field(min_length=1, max_length=40)
|
|
benchmark_schema: str = "vignette.outcome_alliance_benchmark.v1"
|
|
generated_at: datetime = Field(default_factory=_utc_now)
|
|
predictions: tuple[AllianceCalibrationPrediction, ...]
|
|
|
|
@model_validator(mode="after")
|
|
def keep_prediction_keys_unique(self) -> "AllianceCalibrationSnapshot":
|
|
keys = [prediction_key(item) for item in self.predictions]
|
|
if len(set(keys)) != len(keys):
|
|
raise ValueError("calibration prediction keys must be unique")
|
|
return self
|
|
|
|
|
|
def prediction_key(
|
|
prediction: AllianceCalibrationPrediction,
|
|
) -> tuple[str, str, str]:
|
|
return (prediction.case_id, prediction.perspective, prediction.dimension)
|
|
|
|
|
|
def _expected_rows(
|
|
cases: Iterable[BenchmarkCase],
|
|
) -> dict[tuple[str, str, str], tuple[str, frozenset[int]]]:
|
|
rows: dict[tuple[str, str, str], tuple[str, frozenset[int]]] = {}
|
|
for case in cases:
|
|
for expectation in case.expected:
|
|
if expectation.construct_key != "working_alliance":
|
|
continue
|
|
if expectation.dimension not in {"goal", "task", "bond"}:
|
|
continue
|
|
if expectation.perspective not in {
|
|
"client_agent_report",
|
|
"independent_observer",
|
|
}:
|
|
continue
|
|
key = (case.case_id, expectation.perspective, expectation.dimension)
|
|
rows[key] = (
|
|
expectation.direction,
|
|
frozenset(expectation.evidence_turn_indices),
|
|
)
|
|
return rows
|
|
|
|
|
|
def direction_matches(direction: str, value: float) -> bool:
|
|
"""Map the G0 directional gold contract onto fixed, auditable bands.
|
|
|
|
``drop`` and ``rise`` describe the endpoint of the provided scene; they are
|
|
therefore evaluated against the same low/high endpoint bands. True
|
|
longitudinal deltas belong to G2 and are not fabricated here.
|
|
"""
|
|
|
|
if direction in {"low", "drop", "not_detected"}:
|
|
return value <= 0.4
|
|
if direction == "mid":
|
|
return 0.4 < value < 0.7
|
|
if direction in {"high", "rise", "detected"}:
|
|
return value >= 0.7
|
|
raise ValueError(f"unsupported benchmark direction: {direction}")
|
|
|
|
|
|
def gold_reference_snapshot(
|
|
cases: Iterable[BenchmarkCase],
|
|
) -> AllianceCalibrationSnapshot:
|
|
"""Build the explicit human-authored directional reference, not a model run."""
|
|
|
|
predictions: list[AllianceCalibrationPrediction] = []
|
|
for key, (direction, evidence) in sorted(_expected_rows(cases).items()):
|
|
if direction in {"low", "drop", "not_detected"}:
|
|
value = 0.2
|
|
elif direction == "mid":
|
|
value = 0.55
|
|
else:
|
|
value = 0.8
|
|
predictions.append(
|
|
AllianceCalibrationPrediction(
|
|
case_id=key[0],
|
|
perspective=key[1],
|
|
dimension=key[2],
|
|
value=value,
|
|
confidence=1.0,
|
|
evidence_turn_indices=tuple(sorted(evidence)),
|
|
provider="human_gold",
|
|
model="oas-g0-directional-reference",
|
|
prompt_bundle_version="not-applicable",
|
|
)
|
|
)
|
|
return AllianceCalibrationSnapshot(
|
|
run_id="oas-g0-human-gold-v1",
|
|
provider="human_gold",
|
|
model="oas-g0-directional-reference",
|
|
prompt_bundle_version="not-applicable",
|
|
predictions=tuple(predictions),
|
|
)
|
|
|
|
|
|
def evaluate_alliance_snapshot(
|
|
snapshot: AllianceCalibrationSnapshot,
|
|
cases: Iterable[BenchmarkCase],
|
|
) -> dict[str, Any]:
|
|
expected = _expected_rows(cases)
|
|
predicted = {prediction_key(item): item for item in snapshot.predictions}
|
|
direction_hits = 0
|
|
ready_count = 0
|
|
missing_count = 0
|
|
error_count = 0
|
|
recovered_after_retry_count = 0
|
|
evidence_hits = 0
|
|
evidence_expected = 0
|
|
evidence_predicted = 0
|
|
dimension_values: dict[str, list[float]] = defaultdict(list)
|
|
rows: list[dict[str, Any]] = []
|
|
|
|
for key, (direction, gold_evidence) in sorted(expected.items()):
|
|
prediction = predicted.get(key)
|
|
if prediction is None:
|
|
missing_count += 1
|
|
rows.append(
|
|
{
|
|
"case_id": key[0],
|
|
"perspective": key[1],
|
|
"dimension": key[2],
|
|
"expected_direction": direction,
|
|
"status": "missing",
|
|
"direction_match": False,
|
|
"value": None,
|
|
"evidence_precision": 0.0,
|
|
"evidence_recall": 0.0,
|
|
}
|
|
)
|
|
evidence_expected += len(gold_evidence)
|
|
continue
|
|
|
|
candidate_evidence = frozenset(prediction.evidence_turn_indices)
|
|
overlap = len(gold_evidence & candidate_evidence)
|
|
evidence_hits += overlap
|
|
evidence_expected += len(gold_evidence)
|
|
evidence_predicted += len(candidate_evidence)
|
|
if prediction.status == "error":
|
|
error_count += 1
|
|
if prediction.status == "ready" and prediction.attempt_count > 1:
|
|
recovered_after_retry_count += 1
|
|
if prediction.status != "ready" or prediction.value is None:
|
|
missing_count += 1
|
|
match = False
|
|
else:
|
|
ready_count += 1
|
|
dimension_values[prediction.dimension].append(prediction.value)
|
|
match = direction_matches(direction, prediction.value)
|
|
direction_hits += int(match)
|
|
rows.append(
|
|
{
|
|
"case_id": key[0],
|
|
"perspective": key[1],
|
|
"dimension": key[2],
|
|
"expected_direction": direction,
|
|
"status": prediction.status,
|
|
"error_code": prediction.error_code,
|
|
"direction_match": match,
|
|
"value": prediction.value,
|
|
"confidence": prediction.confidence,
|
|
"evidence_precision": (
|
|
overlap / len(candidate_evidence) if candidate_evidence else 0.0
|
|
),
|
|
"evidence_recall": overlap / len(gold_evidence),
|
|
}
|
|
)
|
|
|
|
expected_count = len(expected)
|
|
return {
|
|
"run": {
|
|
"run_id": snapshot.run_id,
|
|
"provider": snapshot.provider,
|
|
"model": snapshot.model,
|
|
"prompt_bundle_version": snapshot.prompt_bundle_version,
|
|
"generated_at": snapshot.generated_at,
|
|
},
|
|
"expected_count": expected_count,
|
|
"ready_count": ready_count,
|
|
"missing_or_failed_count": missing_count,
|
|
"error_count": error_count,
|
|
"recovered_after_retry_count": recovered_after_retry_count,
|
|
"direction_accuracy": direction_hits / expected_count if expected_count else 0.0,
|
|
"evidence_precision": (
|
|
evidence_hits / evidence_predicted if evidence_predicted else 0.0
|
|
),
|
|
"evidence_recall": evidence_hits / evidence_expected if evidence_expected else 0.0,
|
|
"dimension_means": {
|
|
dimension: sum(values) / len(values)
|
|
for dimension, values in sorted(dimension_values.items())
|
|
},
|
|
"rows": rows,
|
|
}
|
|
|
|
|
|
def compare_alliance_snapshots(
|
|
baseline: AllianceCalibrationSnapshot,
|
|
candidate: AllianceCalibrationSnapshot,
|
|
cases: Iterable[BenchmarkCase],
|
|
) -> dict[str, Any]:
|
|
case_tuple = tuple(cases)
|
|
baseline_report = evaluate_alliance_snapshot(baseline, case_tuple)
|
|
candidate_report = evaluate_alliance_snapshot(candidate, case_tuple)
|
|
baseline_rows = {prediction_key(item): item for item in baseline.predictions}
|
|
candidate_rows = {prediction_key(item): item for item in candidate.predictions}
|
|
score_deltas: dict[str, list[float]] = defaultdict(list)
|
|
comparable = 0
|
|
for key in sorted(set(baseline_rows) & set(candidate_rows)):
|
|
before = baseline_rows[key]
|
|
after = candidate_rows[key]
|
|
if before.value is None or after.value is None:
|
|
continue
|
|
comparable += 1
|
|
score_deltas[key[2]].append(after.value - before.value)
|
|
|
|
abs_deltas = [abs(value) for values in score_deltas.values() for value in values]
|
|
return {
|
|
"schema_version": "alliance-calibration-comparison.v1",
|
|
"generated_at": _utc_now(),
|
|
"baseline": baseline_report,
|
|
"candidate": candidate_report,
|
|
"drift": {
|
|
"comparable_predictions": comparable,
|
|
"mean_absolute_score_delta": (
|
|
sum(abs_deltas) / len(abs_deltas) if abs_deltas else None
|
|
),
|
|
"dimension_mean_delta": {
|
|
dimension: sum(values) / len(values)
|
|
for dimension, values in sorted(score_deltas.items())
|
|
},
|
|
"direction_accuracy_delta": (
|
|
candidate_report["direction_accuracy"]
|
|
- baseline_report["direction_accuracy"]
|
|
),
|
|
"evidence_recall_delta": (
|
|
candidate_report["evidence_recall"]
|
|
- baseline_report["evidence_recall"]
|
|
),
|
|
"evidence_precision_delta": (
|
|
candidate_report["evidence_precision"]
|
|
- baseline_report["evidence_precision"]
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
def render_alliance_comparison_markdown(report: dict[str, Any]) -> str:
|
|
baseline = report["baseline"]
|
|
candidate = report["candidate"]
|
|
drift = report["drift"]
|
|
|
|
def percent(value: float) -> str:
|
|
return f"{value * 100:.1f}%"
|
|
|
|
lines = [
|
|
"# Alliance measurement calibration comparison",
|
|
"",
|
|
"> 교육용 합성 장면에 대한 모델 측정 보정 보고서다. 실제 내담자의 임상 결과를 뜻하지 않는다.",
|
|
"",
|
|
"| 항목 | 기준 버전 | 후보 버전 | 변화 |",
|
|
"|---|---:|---:|---:|",
|
|
(
|
|
f"| 방향 정확도 | {percent(baseline['direction_accuracy'])} | "
|
|
f"{percent(candidate['direction_accuracy'])} | "
|
|
f"{percent(drift['direction_accuracy_delta'])} |"
|
|
),
|
|
(
|
|
f"| 근거 재현율 | {percent(baseline['evidence_recall'])} | "
|
|
f"{percent(candidate['evidence_recall'])} | "
|
|
f"{percent(drift['evidence_recall_delta'])} |"
|
|
),
|
|
(
|
|
f"| 근거 정밀도 | {percent(baseline['evidence_precision'])} | "
|
|
f"{percent(candidate['evidence_precision'])} | "
|
|
f"{percent(drift['evidence_precision_delta'])} |"
|
|
),
|
|
(
|
|
f"| 무점수/실패 | {baseline['missing_or_failed_count']} | "
|
|
f"{candidate['missing_or_failed_count']} | "
|
|
f"{candidate['missing_or_failed_count'] - baseline['missing_or_failed_count']:+d} |"
|
|
),
|
|
"",
|
|
"## 실행 식별자",
|
|
"",
|
|
f"- 기준: `{baseline['run']['provider']}/{baseline['run']['model']}` · prompt `{baseline['run']['prompt_bundle_version']}` · run `{baseline['run']['run_id']}`",
|
|
f"- 후보: `{candidate['run']['provider']}/{candidate['run']['model']}` · prompt `{candidate['run']['prompt_bundle_version']}` · run `{candidate['run']['run_id']}`",
|
|
f"- 비교 가능 예측: {drift['comparable_predictions']}개",
|
|
f"- 첫 실패 뒤 재시도 회복: {candidate['recovered_after_retry_count']}개",
|
|
f"- 평균 절대 점수 이동: {drift['mean_absolute_score_delta'] if drift['mean_absolute_score_delta'] is not None else 'N/A'}",
|
|
"",
|
|
"## 후보 버전 장면별 결과",
|
|
"",
|
|
"| 장면 | 관점 | 축 | 기대 | 값 | 상태 | 방향 | 근거 recall |",
|
|
"|---|---|---|---|---:|---|---|---:|",
|
|
]
|
|
for row in candidate["rows"]:
|
|
value = "-" if row["value"] is None else f"{row['value']:.3f}"
|
|
lines.append(
|
|
f"| {row['case_id']} | {row['perspective']} | {row['dimension']} | "
|
|
f"{row['expected_direction']} | {value} | {row['status']} | "
|
|
f"{'통과' if row['direction_match'] else '실패'} | "
|
|
f"{percent(row['evidence_recall'])} |"
|
|
)
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
__all__ = [
|
|
"AllianceCalibrationPrediction",
|
|
"AllianceCalibrationSnapshot",
|
|
"compare_alliance_snapshots",
|
|
"direction_matches",
|
|
"evaluate_alliance_snapshot",
|
|
"gold_reference_snapshot",
|
|
"render_alliance_comparison_markdown",
|
|
]
|