"""Alliance measurement benchmark calibration and drift comparison. This module deliberately compares individual dimensions. It never creates an alliance total, and missing/error predictions count as missing evidence rather than a neutral score. """ from __future__ import annotations from collections import defaultdict from datetime import datetime, timezone from typing import Any, Iterable, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator from ..contracts.measurement import ( AllianceDimension, BenchmarkCase, ) AllianceCalibrationPerspective = Literal[ "client_agent_report", "independent_observer", ] def _utc_now() -> datetime: return datetime.now(timezone.utc) class AllianceCalibrationPrediction(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) case_id: str perspective: AllianceCalibrationPerspective dimension: AllianceDimension value: float | None = Field(default=None, ge=0.0, le=1.0) confidence: float | None = Field(default=None, ge=0.0, le=1.0) evidence_turn_indices: tuple[int, ...] = () status: Literal["ready", "degraded", "error"] = "ready" error_code: str | None = None provider: str | None = Field(default=None, max_length=120) model: str | None = Field(default=None, max_length=240) prompt_bundle_version: str | None = Field(default=None, max_length=40) model_run_id: str | None = Field(default=None, max_length=80) attempt_count: int = Field(default=1, ge=1, le=8) prior_model_run_ids: tuple[str, ...] = () prior_error_codes: tuple[str, ...] = () prompt_bundle_hash: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$") input_evidence_hash: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$") @model_validator(mode="after") def keep_failures_scoreless(self) -> "AllianceCalibrationPrediction": if self.status == "ready" and self.value is None: raise ValueError("ready calibration predictions require a value") if self.status != "ready" and self.value is not None: raise ValueError("failed calibration predictions cannot carry a value") if self.status == "error" and not self.error_code: raise ValueError("error calibration predictions require error_code") if len(set(self.evidence_turn_indices)) != len(self.evidence_turn_indices): raise ValueError("calibration evidence indices must be unique") if self.evidence_turn_indices and min(self.evidence_turn_indices) < 0: raise ValueError("calibration evidence indices must be non-negative") return self class AllianceCalibrationSnapshot(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=()) schema_version: Literal["alliance-calibration-snapshot.v1"] = ( "alliance-calibration-snapshot.v1" ) run_id: str = Field(min_length=1, max_length=160) provider: str = Field(min_length=1, max_length=120) model: str = Field(min_length=1, max_length=240) prompt_bundle_version: str = Field(min_length=1, max_length=40) benchmark_schema: str = "vignette.outcome_alliance_benchmark.v1" generated_at: datetime = Field(default_factory=_utc_now) predictions: tuple[AllianceCalibrationPrediction, ...] @model_validator(mode="after") def keep_prediction_keys_unique(self) -> "AllianceCalibrationSnapshot": keys = [prediction_key(item) for item in self.predictions] if len(set(keys)) != len(keys): raise ValueError("calibration prediction keys must be unique") return self def prediction_key( prediction: AllianceCalibrationPrediction, ) -> tuple[str, str, str]: return (prediction.case_id, prediction.perspective, prediction.dimension) def _expected_rows( cases: Iterable[BenchmarkCase], ) -> dict[tuple[str, str, str], tuple[str, frozenset[int]]]: rows: dict[tuple[str, str, str], tuple[str, frozenset[int]]] = {} for case in cases: for expectation in case.expected: if expectation.construct_key != "working_alliance": continue if expectation.dimension not in {"goal", "task", "bond"}: continue if expectation.perspective not in { "client_agent_report", "independent_observer", }: continue key = (case.case_id, expectation.perspective, expectation.dimension) rows[key] = ( expectation.direction, frozenset(expectation.evidence_turn_indices), ) return rows def direction_matches(direction: str, value: float) -> bool: """Map the G0 directional gold contract onto fixed, auditable bands. ``drop`` and ``rise`` describe the endpoint of the provided scene; they are therefore evaluated against the same low/high endpoint bands. True longitudinal deltas belong to G2 and are not fabricated here. """ if direction in {"low", "drop", "not_detected"}: return value <= 0.4 if direction == "mid": return 0.4 < value < 0.7 if direction in {"high", "rise", "detected"}: return value >= 0.7 raise ValueError(f"unsupported benchmark direction: {direction}") def gold_reference_snapshot( cases: Iterable[BenchmarkCase], ) -> AllianceCalibrationSnapshot: """Build the explicit human-authored directional reference, not a model run.""" predictions: list[AllianceCalibrationPrediction] = [] for key, (direction, evidence) in sorted(_expected_rows(cases).items()): if direction in {"low", "drop", "not_detected"}: value = 0.2 elif direction == "mid": value = 0.55 else: value = 0.8 predictions.append( AllianceCalibrationPrediction( case_id=key[0], perspective=key[1], dimension=key[2], value=value, confidence=1.0, evidence_turn_indices=tuple(sorted(evidence)), provider="human_gold", model="oas-g0-directional-reference", prompt_bundle_version="not-applicable", ) ) return AllianceCalibrationSnapshot( run_id="oas-g0-human-gold-v1", provider="human_gold", model="oas-g0-directional-reference", prompt_bundle_version="not-applicable", predictions=tuple(predictions), ) def evaluate_alliance_snapshot( snapshot: AllianceCalibrationSnapshot, cases: Iterable[BenchmarkCase], ) -> dict[str, Any]: expected = _expected_rows(cases) predicted = {prediction_key(item): item for item in snapshot.predictions} direction_hits = 0 ready_count = 0 missing_count = 0 error_count = 0 recovered_after_retry_count = 0 evidence_hits = 0 evidence_expected = 0 evidence_predicted = 0 dimension_values: dict[str, list[float]] = defaultdict(list) rows: list[dict[str, Any]] = [] for key, (direction, gold_evidence) in sorted(expected.items()): prediction = predicted.get(key) if prediction is None: missing_count += 1 rows.append( { "case_id": key[0], "perspective": key[1], "dimension": key[2], "expected_direction": direction, "status": "missing", "direction_match": False, "value": None, "evidence_precision": 0.0, "evidence_recall": 0.0, } ) evidence_expected += len(gold_evidence) continue candidate_evidence = frozenset(prediction.evidence_turn_indices) overlap = len(gold_evidence & candidate_evidence) evidence_hits += overlap evidence_expected += len(gold_evidence) evidence_predicted += len(candidate_evidence) if prediction.status == "error": error_count += 1 if prediction.status == "ready" and prediction.attempt_count > 1: recovered_after_retry_count += 1 if prediction.status != "ready" or prediction.value is None: missing_count += 1 match = False else: ready_count += 1 dimension_values[prediction.dimension].append(prediction.value) match = direction_matches(direction, prediction.value) direction_hits += int(match) rows.append( { "case_id": key[0], "perspective": key[1], "dimension": key[2], "expected_direction": direction, "status": prediction.status, "error_code": prediction.error_code, "direction_match": match, "value": prediction.value, "confidence": prediction.confidence, "evidence_precision": ( overlap / len(candidate_evidence) if candidate_evidence else 0.0 ), "evidence_recall": overlap / len(gold_evidence), } ) expected_count = len(expected) return { "run": { "run_id": snapshot.run_id, "provider": snapshot.provider, "model": snapshot.model, "prompt_bundle_version": snapshot.prompt_bundle_version, "generated_at": snapshot.generated_at, }, "expected_count": expected_count, "ready_count": ready_count, "missing_or_failed_count": missing_count, "error_count": error_count, "recovered_after_retry_count": recovered_after_retry_count, "direction_accuracy": direction_hits / expected_count if expected_count else 0.0, "evidence_precision": ( evidence_hits / evidence_predicted if evidence_predicted else 0.0 ), "evidence_recall": evidence_hits / evidence_expected if evidence_expected else 0.0, "dimension_means": { dimension: sum(values) / len(values) for dimension, values in sorted(dimension_values.items()) }, "rows": rows, } def compare_alliance_snapshots( baseline: AllianceCalibrationSnapshot, candidate: AllianceCalibrationSnapshot, cases: Iterable[BenchmarkCase], ) -> dict[str, Any]: case_tuple = tuple(cases) baseline_report = evaluate_alliance_snapshot(baseline, case_tuple) candidate_report = evaluate_alliance_snapshot(candidate, case_tuple) baseline_rows = {prediction_key(item): item for item in baseline.predictions} candidate_rows = {prediction_key(item): item for item in candidate.predictions} score_deltas: dict[str, list[float]] = defaultdict(list) comparable = 0 for key in sorted(set(baseline_rows) & set(candidate_rows)): before = baseline_rows[key] after = candidate_rows[key] if before.value is None or after.value is None: continue comparable += 1 score_deltas[key[2]].append(after.value - before.value) abs_deltas = [abs(value) for values in score_deltas.values() for value in values] return { "schema_version": "alliance-calibration-comparison.v1", "generated_at": _utc_now(), "baseline": baseline_report, "candidate": candidate_report, "drift": { "comparable_predictions": comparable, "mean_absolute_score_delta": ( sum(abs_deltas) / len(abs_deltas) if abs_deltas else None ), "dimension_mean_delta": { dimension: sum(values) / len(values) for dimension, values in sorted(score_deltas.items()) }, "direction_accuracy_delta": ( candidate_report["direction_accuracy"] - baseline_report["direction_accuracy"] ), "evidence_recall_delta": ( candidate_report["evidence_recall"] - baseline_report["evidence_recall"] ), "evidence_precision_delta": ( candidate_report["evidence_precision"] - baseline_report["evidence_precision"] ), }, } def render_alliance_comparison_markdown(report: dict[str, Any]) -> str: baseline = report["baseline"] candidate = report["candidate"] drift = report["drift"] def percent(value: float) -> str: return f"{value * 100:.1f}%" lines = [ "# Alliance measurement calibration comparison", "", "> 교육용 합성 장면에 대한 모델 측정 보정 보고서다. 실제 내담자의 임상 결과를 뜻하지 않는다.", "", "| 항목 | 기준 버전 | 후보 버전 | 변화 |", "|---|---:|---:|---:|", ( f"| 방향 정확도 | {percent(baseline['direction_accuracy'])} | " f"{percent(candidate['direction_accuracy'])} | " f"{percent(drift['direction_accuracy_delta'])} |" ), ( f"| 근거 재현율 | {percent(baseline['evidence_recall'])} | " f"{percent(candidate['evidence_recall'])} | " f"{percent(drift['evidence_recall_delta'])} |" ), ( f"| 근거 정밀도 | {percent(baseline['evidence_precision'])} | " f"{percent(candidate['evidence_precision'])} | " f"{percent(drift['evidence_precision_delta'])} |" ), ( f"| 무점수/실패 | {baseline['missing_or_failed_count']} | " f"{candidate['missing_or_failed_count']} | " f"{candidate['missing_or_failed_count'] - baseline['missing_or_failed_count']:+d} |" ), "", "## 실행 식별자", "", f"- 기준: `{baseline['run']['provider']}/{baseline['run']['model']}` · prompt `{baseline['run']['prompt_bundle_version']}` · run `{baseline['run']['run_id']}`", f"- 후보: `{candidate['run']['provider']}/{candidate['run']['model']}` · prompt `{candidate['run']['prompt_bundle_version']}` · run `{candidate['run']['run_id']}`", f"- 비교 가능 예측: {drift['comparable_predictions']}개", f"- 첫 실패 뒤 재시도 회복: {candidate['recovered_after_retry_count']}개", f"- 평균 절대 점수 이동: {drift['mean_absolute_score_delta'] if drift['mean_absolute_score_delta'] is not None else 'N/A'}", "", "## 후보 버전 장면별 결과", "", "| 장면 | 관점 | 축 | 기대 | 값 | 상태 | 방향 | 근거 recall |", "|---|---|---|---|---:|---|---|---:|", ] for row in candidate["rows"]: value = "-" if row["value"] is None else f"{row['value']:.3f}" lines.append( f"| {row['case_id']} | {row['perspective']} | {row['dimension']} | " f"{row['expected_direction']} | {value} | {row['status']} | " f"{'통과' if row['direction_match'] else '실패'} | " f"{percent(row['evidence_recall'])} |" ) lines.append("") return "\n".join(lines) __all__ = [ "AllianceCalibrationPrediction", "AllianceCalibrationSnapshot", "compare_alliance_snapshots", "direction_matches", "evaluate_alliance_snapshot", "gold_reference_snapshot", "render_alliance_comparison_markdown", ]