"""G3 균열·복구 상태기계와 결정론 benchmark 코어. 문장 표면형이나 균열 개수를 점수화하지 않는다. 관찰된 복구 행동과 그 다음 내담자 반응이 함께 있어야 resolved가 되며, safety 원장은 판정과 분리해 전달한다. """ from __future__ import annotations import json from pathlib import Path from typing import Iterable from ..contracts.rupture_repair import ( RUPTURE_TYPES, FastDeepReconciliation, RepairAttemptAssessment, RepairAttemptObservation, RepairBehavior, RuptureBenchmarkPack, RuptureDetectionSignal, RuptureEpisodeAssessment, RuptureEpisodeInput, RuptureEvidenceRef, RuptureLedgerEntry, RuptureLifecycleState, RuptureType, ) _REQUIRED_BEHAVIORS: dict[RuptureType, tuple[RepairBehavior, ...]] = { "withdrawal": ("curiosity", "impact_acknowledgement", "follow_up_check"), "confrontation": ("curiosity", "impact_acknowledgement", "follow_up_check"), "goal_mismatch": ("curiosity", "goal_reagreement", "follow_up_check"), "task_mismatch": ("curiosity", "task_reagreement", "follow_up_check"), "empathic_miss": ("curiosity", "impact_acknowledgement", "follow_up_check"), "cultural_miss": ("curiosity", "impact_acknowledgement", "follow_up_check"), "boundary_tension": ("naming", "impact_acknowledgement", "follow_up_check"), "premature_advice": ("curiosity", "impact_acknowledgement", "follow_up_check"), "over_disclosure": ("curiosity", "impact_acknowledgement", "follow_up_check"), } _ENGAGED_RESPONSES = frozenset({"engaged", "explicit_alignment"}) _PARTIAL_RESPONSES = frozenset({"mixed", "engaged", "explicit_alignment"}) def _unique_refs( refs: Iterable[RuptureEvidenceRef], ) -> tuple[RuptureEvidenceRef, ...]: result: list[RuptureEvidenceRef] = [] seen: set[str] = set() for ref in refs: if ref.ref_id in seen: continue seen.add(ref.ref_id) result.append(ref) return tuple(result) def _select_deep_detection( signals: tuple[RuptureDetectionSignal, ...], ) -> RuptureDetectionSignal: """완료된 deep 판정이 fast 경고보다 우선하고, deep 오류는 fast를 지우지 않는다.""" deep_ready = [ signal for signal in signals if signal.loop == "deep" and signal.status != "error" ] if deep_ready: return max(deep_ready, key=lambda item: (item.observed_at_turn, item.signal_id)) fast_ready = [signal for signal in signals if signal.status != "error"] if fast_ready: return max(fast_ready, key=lambda item: (item.observed_at_turn, item.signal_id)) return max(signals, key=lambda item: (item.observed_at_turn, item.signal_id)) def _assess_attempt( attempt: RepairAttemptObservation, rupture_type: RuptureType, ) -> RepairAttemptAssessment: required = _REQUIRED_BEHAVIORS[rupture_type] observed = set(attempt.behaviors) present_required = tuple(item for item in required if item in observed) missing = tuple(item for item in required if item not in observed) if not missing and attempt.client_response in _ENGAGED_RESPONSES: outcome = "resolved" derived_counterevidence: tuple[str, ...] = () elif ( len(present_required) >= 2 and attempt.client_response in _PARTIAL_RESPONSES ): outcome = "partial" derived_counterevidence = ( "required_repair_behavior_incomplete" if missing else "client_response_not_yet_explicitly_engaged", ) else: outcome = "missed" derived_counterevidence = ( "formulaic_language_without_observed_repair_impact", "client_response_does_not_support_resolution", ) return RepairAttemptAssessment( attempt_id=attempt.attempt_id, outcome=outcome, observed_behaviors=attempt.behaviors, required_behaviors=required, missing_behaviors=missing, client_response=attempt.client_response, evidence_refs=_unique_refs( (*attempt.evidence_refs, *attempt.response_evidence_refs) ), counterevidence=tuple( dict.fromkeys((*attempt.counterevidence, *derived_counterevidence)) ), uncertainty=attempt.uncertainty, ) def _reconcile( episode: RuptureEpisodeInput, *, final_status: str, evidence_refs: tuple[RuptureEvidenceRef, ...], ) -> FastDeepReconciliation: warning = episode.fast_warning if warning is None: return FastDeepReconciliation( disposition="not_applicable", deep_status=final_status, evidence_refs=evidence_refs, reason="fast-loop warning이 없어 deep 판정을 독립 기록했다.", ) if final_status == "not_applicable": disposition = "dismissed" reason = "deep-loop의 전체 장면 검토에서 균열 근거가 유지되지 않아 fast 경고를 기각했다." elif final_status == warning.provisional_status: disposition = "confirmed" reason = "deep-loop의 후속 장면 검토가 fast 경고 상태를 확인했다." elif final_status == "resolved": disposition = "superseded_resolved" reason = "후속 발화의 복구 행동과 내담자 반응이 확인되어 fast 경고를 resolved로 대체했다." elif final_status == "partial": disposition = "superseded_partial" reason = "후속 발화에서 일부 복구가 확인되어 fast 경고를 partial로 대체했다." else: disposition = "confirmed" reason = "deep-loop에서 충분한 복구 근거가 확인되지 않아 unresolved 경고를 유지했다." return FastDeepReconciliation( warning_id=warning.warning_id, disposition=disposition, provisional_status=warning.provisional_status, deep_status=final_status, evidence_refs=evidence_refs, reason=reason, ) def assess_rupture_episode(episode: RuptureEpisodeInput) -> RuptureEpisodeAssessment: """한 균열 episode를 append-only 상태 전이로 판정한다.""" selected = _select_deep_detection(episode.detection_signals) if selected.status == "error": return RuptureEpisodeAssessment( episode_id=episode.episode_id, assessment_status="error", detected=False, rupture_type=None, final_status="insufficient_evidence", confidence=None, uncertainty=1.0, evidence_refs=(), counterevidence=( f"all_detection_signals_failed:{selected.error_code or 'unknown'}", ), repair_attempts=(), ledger=(), reconciliation=FastDeepReconciliation( disposition="not_applicable", deep_status="insufficient_evidence", reason="탐지 신호가 모두 실패해 균열 부재나 복구 상태를 추정하지 않았다.", ), safety_signals=episode.safety_signals, ) if selected.status != "detected": counterevidence = tuple( dict.fromkeys( reason for signal in episode.detection_signals for reason in signal.counterevidence ) ) reconciliation = _reconcile( episode, final_status="not_applicable", evidence_refs=selected.evidence_refs, ) return RuptureEpisodeAssessment( episode_id=episode.episode_id, detected=False, rupture_type=None, final_status="not_applicable", confidence=None, uncertainty=selected.uncertainty, evidence_refs=selected.evidence_refs, counterevidence=counterevidence, repair_attempts=(), ledger=(), reconciliation=reconciliation, safety_signals=episode.safety_signals, ) assert selected.rupture_type is not None assert selected.confidence is not None rupture_type = selected.rupture_type ledger: list[RuptureLedgerEntry] = [] state: RuptureLifecycleState = "onset" def append_entry( *, event_name: str, from_state: RuptureLifecycleState | None, to_state: RuptureLifecycleState, evidence_refs: tuple[RuptureEvidenceRef, ...], counterevidence: tuple[str, ...], uncertainty: float, source_ref_id: str, reconciles_event_id: str | None = None, ) -> None: ledger.append( RuptureLedgerEntry( sequence_no=len(ledger) + 1, event_name=event_name, from_state=from_state, to_state=to_state, evidence_refs=evidence_refs, counterevidence=counterevidence, uncertainty=uncertainty, source_ref_id=source_ref_id, reconciles_event_id=reconciles_event_id, ) ) append_entry( event_name="rupture.detected", from_state=None, to_state="onset", evidence_refs=selected.evidence_refs, counterevidence=selected.counterevidence, uncertainty=selected.uncertainty, source_ref_id=selected.signal_id, ) attempt_results: list[RepairAttemptAssessment] = [] if episode.recognized_at_turn is None: append_entry( event_name="rupture.missed", from_state=state, to_state="missed", evidence_refs=selected.evidence_refs, counterevidence=("no_recognition_evidence",), uncertainty=selected.uncertainty, source_ref_id=episode.episode_id, ) state = "missed" else: append_entry( event_name="rupture.recognized", from_state=state, to_state="recognized", evidence_refs=episode.recognition_evidence_refs, counterevidence=(), uncertainty=selected.uncertainty, source_ref_id=episode.episode_id, ) state = "recognized" for attempt in episode.repair_attempts: append_entry( event_name="repair.attempted", from_state=state, to_state="repair_attempted", evidence_refs=attempt.evidence_refs, counterevidence=attempt.counterevidence, uncertainty=attempt.uncertainty, source_ref_id=attempt.attempt_id, ) state = "repair_attempted" assessed = _assess_attempt(attempt, rupture_type) attempt_results.append(assessed) state = assessed.outcome append_entry( event_name=f"repair.{assessed.outcome}", from_state="repair_attempted", to_state=state, evidence_refs=assessed.evidence_refs, counterevidence=assessed.counterevidence, uncertainty=assessed.uncertainty, source_ref_id=attempt.attempt_id, ) if state == "resolved": break if not attempt_results: append_entry( event_name="rupture.missed", from_state=state, to_state="missed", evidence_refs=episode.recognition_evidence_refs, counterevidence=("recognized_without_repair_attempt",), uncertainty=selected.uncertainty, source_ref_id=episode.episode_id, ) state = "missed" final_status = state final_attempt_refs = ( attempt_results[-1].evidence_refs if attempt_results else () ) assessment_evidence = _unique_refs( (*selected.evidence_refs, *episode.recognition_evidence_refs, *final_attempt_refs) ) reconciliation = _reconcile( episode, final_status=final_status, evidence_refs=assessment_evidence, ) if episode.fast_warning: append_entry( event_name="rupture.reconciled", from_state=state, to_state=state, evidence_refs=reconciliation.evidence_refs, counterevidence=(), uncertainty=max( [selected.uncertainty] + [item.uncertainty for item in attempt_results] ), source_ref_id=selected.signal_id, reconciles_event_id=episode.fast_warning.warning_id, ) return RuptureEpisodeAssessment( episode_id=episode.episode_id, detected=True, rupture_type=rupture_type, final_status=final_status, confidence=selected.confidence, uncertainty=max( [selected.uncertainty] + [item.uncertainty for item in attempt_results] ), evidence_refs=assessment_evidence, counterevidence=tuple( dict.fromkeys( (*selected.counterevidence,) + tuple( reason for attempt in attempt_results for reason in attempt.counterevidence ) ) ), repair_attempts=tuple(attempt_results), ledger=tuple(ledger), reconciliation=reconciliation, safety_signals=episode.safety_signals, ) def load_rupture_benchmark(path: Path) -> RuptureBenchmarkPack: return RuptureBenchmarkPack.model_validate_json(path.read_text(encoding="utf-8")) def _macro_f1( rows: list[tuple[RuptureType | None, RuptureType | None]], ) -> tuple[float, dict[str, float]]: per_type: dict[str, float] = {} for rupture_type in RUPTURE_TYPES: tp = sum(actual == rupture_type and predicted == rupture_type for actual, predicted in rows) fp = sum(actual != rupture_type and predicted == rupture_type for actual, predicted in rows) fn = sum(actual == rupture_type and predicted != rupture_type for actual, predicted in rows) precision = tp / (tp + fp) if tp + fp else 0.0 recall = tp / (tp + fn) if tp + fn else 0.0 per_type[rupture_type] = ( 2 * precision * recall / (precision + recall) if precision + recall else 0.0 ) return sum(per_type.values()) / len(per_type), per_type def evaluate_rupture_benchmark(pack: RuptureBenchmarkPack) -> dict[str, object]: """유형 탐지, 복구 상태, adversarial judge-gaming 실패를 각각 보고한다.""" type_rows: list[tuple[RuptureType | None, RuptureType | None]] = [] detection_tp = detection_fp = detection_fn = detection_tn = 0 status_hits = critical_misses = 0 judge_gaming_regressions = memorized_phrase_false_resolutions = 0 rows: list[dict[str, object]] = [] for case in pack.cases: actual = assess_rupture_episode(case.episode) expected = case.expected type_rows.append((expected.rupture_type, actual.rupture_type)) if expected.detected and actual.detected: detection_tp += 1 elif expected.detected: detection_fn += 1 elif actual.detected: detection_fp += 1 else: detection_tn += 1 if case.critical and expected.detected and not actual.detected: critical_misses += 1 status_match = actual.final_status == expected.final_status status_hits += int(status_match) if ( "judge_gaming" in case.tags and expected.final_status != "resolved" and actual.final_status == "resolved" ): judge_gaming_regressions += 1 if ( "memorized_phrase_trap" in case.tags and expected.final_status != "resolved" and actual.final_status == "resolved" ): memorized_phrase_false_resolutions += 1 rows.append( { "case_id": case.case_id, "expected_type": expected.rupture_type, "actual_type": actual.rupture_type, "expected_status": expected.final_status, "actual_status": actual.final_status, "status_match": status_match, "reconciliation": actual.reconciliation.disposition, "uncertainty": actual.uncertainty, "counterevidence": list(actual.counterevidence), "evidence_refs": [item.ref_id for item in actual.evidence_refs], "safety_signal_count": len(actual.safety_signals), "tags": list(case.tags), } ) macro_f1, per_type_f1 = _macro_f1(type_rows) total = len(pack.cases) return { "schema_version": "vignette.rupture-repair-benchmark-report.v1", "data_classification": "synthetic_educational", "clinical_claim_allowed": False, "benchmark_version": pack.version, "case_count": total, "rupture_type_macro_f1": macro_f1, "rupture_type_f1": per_type_f1, "repair_status_accuracy": status_hits / total if total else None, "critical_miss_count": critical_misses, "judge_gaming_regressions": judge_gaming_regressions, "memorized_phrase_false_resolutions": memorized_phrase_false_resolutions, "detection_confusion": { "true_positive": detection_tp, "false_positive": detection_fp, "false_negative": detection_fn, "true_negative": detection_tn, }, "rows": rows, } def render_rupture_benchmark_report(report: dict[str, object]) -> str: return json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) __all__ = [ "assess_rupture_episode", "evaluate_rupture_benchmark", "load_rupture_benchmark", "render_rupture_benchmark_report", ]