16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
482 lines
18 KiB
Python
482 lines
18 KiB
Python
"""G3 균열·복구 상태기계와 결정론 benchmark 코어.
|
|
|
|
문장 표면형이나 균열 개수를 점수화하지 않는다. 관찰된 복구 행동과 그 다음
|
|
내담자 반응이 함께 있어야 resolved가 되며, safety 원장은 판정과 분리해 전달한다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
from ..contracts.rupture_repair import (
|
|
RUPTURE_TYPES,
|
|
FastDeepReconciliation,
|
|
RepairAttemptAssessment,
|
|
RepairAttemptObservation,
|
|
RepairBehavior,
|
|
RuptureBenchmarkPack,
|
|
RuptureDetectionSignal,
|
|
RuptureEpisodeAssessment,
|
|
RuptureEpisodeInput,
|
|
RuptureEvidenceRef,
|
|
RuptureLedgerEntry,
|
|
RuptureLifecycleState,
|
|
RuptureType,
|
|
)
|
|
|
|
|
|
_REQUIRED_BEHAVIORS: dict[RuptureType, tuple[RepairBehavior, ...]] = {
|
|
"withdrawal": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
"confrontation": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
"goal_mismatch": ("curiosity", "goal_reagreement", "follow_up_check"),
|
|
"task_mismatch": ("curiosity", "task_reagreement", "follow_up_check"),
|
|
"empathic_miss": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
"cultural_miss": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
"boundary_tension": ("naming", "impact_acknowledgement", "follow_up_check"),
|
|
"premature_advice": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
"over_disclosure": ("curiosity", "impact_acknowledgement", "follow_up_check"),
|
|
}
|
|
|
|
_ENGAGED_RESPONSES = frozenset({"engaged", "explicit_alignment"})
|
|
_PARTIAL_RESPONSES = frozenset({"mixed", "engaged", "explicit_alignment"})
|
|
|
|
|
|
def _unique_refs(
|
|
refs: Iterable[RuptureEvidenceRef],
|
|
) -> tuple[RuptureEvidenceRef, ...]:
|
|
result: list[RuptureEvidenceRef] = []
|
|
seen: set[str] = set()
|
|
for ref in refs:
|
|
if ref.ref_id in seen:
|
|
continue
|
|
seen.add(ref.ref_id)
|
|
result.append(ref)
|
|
return tuple(result)
|
|
|
|
|
|
def _select_deep_detection(
|
|
signals: tuple[RuptureDetectionSignal, ...],
|
|
) -> RuptureDetectionSignal:
|
|
"""완료된 deep 판정이 fast 경고보다 우선하고, deep 오류는 fast를 지우지 않는다."""
|
|
|
|
deep_ready = [
|
|
signal
|
|
for signal in signals
|
|
if signal.loop == "deep" and signal.status != "error"
|
|
]
|
|
if deep_ready:
|
|
return max(deep_ready, key=lambda item: (item.observed_at_turn, item.signal_id))
|
|
fast_ready = [signal for signal in signals if signal.status != "error"]
|
|
if fast_ready:
|
|
return max(fast_ready, key=lambda item: (item.observed_at_turn, item.signal_id))
|
|
return max(signals, key=lambda item: (item.observed_at_turn, item.signal_id))
|
|
|
|
|
|
def _assess_attempt(
|
|
attempt: RepairAttemptObservation,
|
|
rupture_type: RuptureType,
|
|
) -> RepairAttemptAssessment:
|
|
required = _REQUIRED_BEHAVIORS[rupture_type]
|
|
observed = set(attempt.behaviors)
|
|
present_required = tuple(item for item in required if item in observed)
|
|
missing = tuple(item for item in required if item not in observed)
|
|
|
|
if not missing and attempt.client_response in _ENGAGED_RESPONSES:
|
|
outcome = "resolved"
|
|
derived_counterevidence: tuple[str, ...] = ()
|
|
elif (
|
|
len(present_required) >= 2
|
|
and attempt.client_response in _PARTIAL_RESPONSES
|
|
):
|
|
outcome = "partial"
|
|
derived_counterevidence = (
|
|
"required_repair_behavior_incomplete"
|
|
if missing
|
|
else "client_response_not_yet_explicitly_engaged",
|
|
)
|
|
else:
|
|
outcome = "missed"
|
|
derived_counterevidence = (
|
|
"formulaic_language_without_observed_repair_impact",
|
|
"client_response_does_not_support_resolution",
|
|
)
|
|
|
|
return RepairAttemptAssessment(
|
|
attempt_id=attempt.attempt_id,
|
|
outcome=outcome,
|
|
observed_behaviors=attempt.behaviors,
|
|
required_behaviors=required,
|
|
missing_behaviors=missing,
|
|
client_response=attempt.client_response,
|
|
evidence_refs=_unique_refs(
|
|
(*attempt.evidence_refs, *attempt.response_evidence_refs)
|
|
),
|
|
counterevidence=tuple(
|
|
dict.fromkeys((*attempt.counterevidence, *derived_counterevidence))
|
|
),
|
|
uncertainty=attempt.uncertainty,
|
|
)
|
|
|
|
|
|
def _reconcile(
|
|
episode: RuptureEpisodeInput,
|
|
*,
|
|
final_status: str,
|
|
evidence_refs: tuple[RuptureEvidenceRef, ...],
|
|
) -> FastDeepReconciliation:
|
|
warning = episode.fast_warning
|
|
if warning is None:
|
|
return FastDeepReconciliation(
|
|
disposition="not_applicable",
|
|
deep_status=final_status,
|
|
evidence_refs=evidence_refs,
|
|
reason="fast-loop warning이 없어 deep 판정을 독립 기록했다.",
|
|
)
|
|
if final_status == "not_applicable":
|
|
disposition = "dismissed"
|
|
reason = "deep-loop의 전체 장면 검토에서 균열 근거가 유지되지 않아 fast 경고를 기각했다."
|
|
elif final_status == warning.provisional_status:
|
|
disposition = "confirmed"
|
|
reason = "deep-loop의 후속 장면 검토가 fast 경고 상태를 확인했다."
|
|
elif final_status == "resolved":
|
|
disposition = "superseded_resolved"
|
|
reason = "후속 발화의 복구 행동과 내담자 반응이 확인되어 fast 경고를 resolved로 대체했다."
|
|
elif final_status == "partial":
|
|
disposition = "superseded_partial"
|
|
reason = "후속 발화에서 일부 복구가 확인되어 fast 경고를 partial로 대체했다."
|
|
else:
|
|
disposition = "confirmed"
|
|
reason = "deep-loop에서 충분한 복구 근거가 확인되지 않아 unresolved 경고를 유지했다."
|
|
return FastDeepReconciliation(
|
|
warning_id=warning.warning_id,
|
|
disposition=disposition,
|
|
provisional_status=warning.provisional_status,
|
|
deep_status=final_status,
|
|
evidence_refs=evidence_refs,
|
|
reason=reason,
|
|
)
|
|
|
|
|
|
def assess_rupture_episode(episode: RuptureEpisodeInput) -> RuptureEpisodeAssessment:
|
|
"""한 균열 episode를 append-only 상태 전이로 판정한다."""
|
|
|
|
selected = _select_deep_detection(episode.detection_signals)
|
|
if selected.status == "error":
|
|
return RuptureEpisodeAssessment(
|
|
episode_id=episode.episode_id,
|
|
assessment_status="error",
|
|
detected=False,
|
|
rupture_type=None,
|
|
final_status="insufficient_evidence",
|
|
confidence=None,
|
|
uncertainty=1.0,
|
|
evidence_refs=(),
|
|
counterevidence=(
|
|
f"all_detection_signals_failed:{selected.error_code or 'unknown'}",
|
|
),
|
|
repair_attempts=(),
|
|
ledger=(),
|
|
reconciliation=FastDeepReconciliation(
|
|
disposition="not_applicable",
|
|
deep_status="insufficient_evidence",
|
|
reason="탐지 신호가 모두 실패해 균열 부재나 복구 상태를 추정하지 않았다.",
|
|
),
|
|
safety_signals=episode.safety_signals,
|
|
)
|
|
if selected.status != "detected":
|
|
counterevidence = tuple(
|
|
dict.fromkeys(
|
|
reason
|
|
for signal in episode.detection_signals
|
|
for reason in signal.counterevidence
|
|
)
|
|
)
|
|
reconciliation = _reconcile(
|
|
episode,
|
|
final_status="not_applicable",
|
|
evidence_refs=selected.evidence_refs,
|
|
)
|
|
return RuptureEpisodeAssessment(
|
|
episode_id=episode.episode_id,
|
|
detected=False,
|
|
rupture_type=None,
|
|
final_status="not_applicable",
|
|
confidence=None,
|
|
uncertainty=selected.uncertainty,
|
|
evidence_refs=selected.evidence_refs,
|
|
counterevidence=counterevidence,
|
|
repair_attempts=(),
|
|
ledger=(),
|
|
reconciliation=reconciliation,
|
|
safety_signals=episode.safety_signals,
|
|
)
|
|
|
|
assert selected.rupture_type is not None
|
|
assert selected.confidence is not None
|
|
rupture_type = selected.rupture_type
|
|
ledger: list[RuptureLedgerEntry] = []
|
|
state: RuptureLifecycleState = "onset"
|
|
|
|
def append_entry(
|
|
*,
|
|
event_name: str,
|
|
from_state: RuptureLifecycleState | None,
|
|
to_state: RuptureLifecycleState,
|
|
evidence_refs: tuple[RuptureEvidenceRef, ...],
|
|
counterevidence: tuple[str, ...],
|
|
uncertainty: float,
|
|
source_ref_id: str,
|
|
reconciles_event_id: str | None = None,
|
|
) -> None:
|
|
ledger.append(
|
|
RuptureLedgerEntry(
|
|
sequence_no=len(ledger) + 1,
|
|
event_name=event_name,
|
|
from_state=from_state,
|
|
to_state=to_state,
|
|
evidence_refs=evidence_refs,
|
|
counterevidence=counterevidence,
|
|
uncertainty=uncertainty,
|
|
source_ref_id=source_ref_id,
|
|
reconciles_event_id=reconciles_event_id,
|
|
)
|
|
)
|
|
|
|
append_entry(
|
|
event_name="rupture.detected",
|
|
from_state=None,
|
|
to_state="onset",
|
|
evidence_refs=selected.evidence_refs,
|
|
counterevidence=selected.counterevidence,
|
|
uncertainty=selected.uncertainty,
|
|
source_ref_id=selected.signal_id,
|
|
)
|
|
|
|
attempt_results: list[RepairAttemptAssessment] = []
|
|
if episode.recognized_at_turn is None:
|
|
append_entry(
|
|
event_name="rupture.missed",
|
|
from_state=state,
|
|
to_state="missed",
|
|
evidence_refs=selected.evidence_refs,
|
|
counterevidence=("no_recognition_evidence",),
|
|
uncertainty=selected.uncertainty,
|
|
source_ref_id=episode.episode_id,
|
|
)
|
|
state = "missed"
|
|
else:
|
|
append_entry(
|
|
event_name="rupture.recognized",
|
|
from_state=state,
|
|
to_state="recognized",
|
|
evidence_refs=episode.recognition_evidence_refs,
|
|
counterevidence=(),
|
|
uncertainty=selected.uncertainty,
|
|
source_ref_id=episode.episode_id,
|
|
)
|
|
state = "recognized"
|
|
|
|
for attempt in episode.repair_attempts:
|
|
append_entry(
|
|
event_name="repair.attempted",
|
|
from_state=state,
|
|
to_state="repair_attempted",
|
|
evidence_refs=attempt.evidence_refs,
|
|
counterevidence=attempt.counterevidence,
|
|
uncertainty=attempt.uncertainty,
|
|
source_ref_id=attempt.attempt_id,
|
|
)
|
|
state = "repair_attempted"
|
|
assessed = _assess_attempt(attempt, rupture_type)
|
|
attempt_results.append(assessed)
|
|
state = assessed.outcome
|
|
append_entry(
|
|
event_name=f"repair.{assessed.outcome}",
|
|
from_state="repair_attempted",
|
|
to_state=state,
|
|
evidence_refs=assessed.evidence_refs,
|
|
counterevidence=assessed.counterevidence,
|
|
uncertainty=assessed.uncertainty,
|
|
source_ref_id=attempt.attempt_id,
|
|
)
|
|
if state == "resolved":
|
|
break
|
|
|
|
if not attempt_results:
|
|
append_entry(
|
|
event_name="rupture.missed",
|
|
from_state=state,
|
|
to_state="missed",
|
|
evidence_refs=episode.recognition_evidence_refs,
|
|
counterevidence=("recognized_without_repair_attempt",),
|
|
uncertainty=selected.uncertainty,
|
|
source_ref_id=episode.episode_id,
|
|
)
|
|
state = "missed"
|
|
|
|
final_status = state
|
|
final_attempt_refs = (
|
|
attempt_results[-1].evidence_refs if attempt_results else ()
|
|
)
|
|
assessment_evidence = _unique_refs(
|
|
(*selected.evidence_refs, *episode.recognition_evidence_refs, *final_attempt_refs)
|
|
)
|
|
reconciliation = _reconcile(
|
|
episode,
|
|
final_status=final_status,
|
|
evidence_refs=assessment_evidence,
|
|
)
|
|
if episode.fast_warning:
|
|
append_entry(
|
|
event_name="rupture.reconciled",
|
|
from_state=state,
|
|
to_state=state,
|
|
evidence_refs=reconciliation.evidence_refs,
|
|
counterevidence=(),
|
|
uncertainty=max(
|
|
[selected.uncertainty]
|
|
+ [item.uncertainty for item in attempt_results]
|
|
),
|
|
source_ref_id=selected.signal_id,
|
|
reconciles_event_id=episode.fast_warning.warning_id,
|
|
)
|
|
|
|
return RuptureEpisodeAssessment(
|
|
episode_id=episode.episode_id,
|
|
detected=True,
|
|
rupture_type=rupture_type,
|
|
final_status=final_status,
|
|
confidence=selected.confidence,
|
|
uncertainty=max(
|
|
[selected.uncertainty] + [item.uncertainty for item in attempt_results]
|
|
),
|
|
evidence_refs=assessment_evidence,
|
|
counterevidence=tuple(
|
|
dict.fromkeys(
|
|
(*selected.counterevidence,)
|
|
+ tuple(
|
|
reason
|
|
for attempt in attempt_results
|
|
for reason in attempt.counterevidence
|
|
)
|
|
)
|
|
),
|
|
repair_attempts=tuple(attempt_results),
|
|
ledger=tuple(ledger),
|
|
reconciliation=reconciliation,
|
|
safety_signals=episode.safety_signals,
|
|
)
|
|
|
|
|
|
def load_rupture_benchmark(path: Path) -> RuptureBenchmarkPack:
|
|
return RuptureBenchmarkPack.model_validate_json(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _macro_f1(
|
|
rows: list[tuple[RuptureType | None, RuptureType | None]],
|
|
) -> tuple[float, dict[str, float]]:
|
|
per_type: dict[str, float] = {}
|
|
for rupture_type in RUPTURE_TYPES:
|
|
tp = sum(actual == rupture_type and predicted == rupture_type for actual, predicted in rows)
|
|
fp = sum(actual != rupture_type and predicted == rupture_type for actual, predicted in rows)
|
|
fn = sum(actual == rupture_type and predicted != rupture_type for actual, predicted in rows)
|
|
precision = tp / (tp + fp) if tp + fp else 0.0
|
|
recall = tp / (tp + fn) if tp + fn else 0.0
|
|
per_type[rupture_type] = (
|
|
2 * precision * recall / (precision + recall)
|
|
if precision + recall
|
|
else 0.0
|
|
)
|
|
return sum(per_type.values()) / len(per_type), per_type
|
|
|
|
|
|
def evaluate_rupture_benchmark(pack: RuptureBenchmarkPack) -> dict[str, object]:
|
|
"""유형 탐지, 복구 상태, adversarial judge-gaming 실패를 각각 보고한다."""
|
|
|
|
type_rows: list[tuple[RuptureType | None, RuptureType | None]] = []
|
|
detection_tp = detection_fp = detection_fn = detection_tn = 0
|
|
status_hits = critical_misses = 0
|
|
judge_gaming_regressions = memorized_phrase_false_resolutions = 0
|
|
rows: list[dict[str, object]] = []
|
|
|
|
for case in pack.cases:
|
|
actual = assess_rupture_episode(case.episode)
|
|
expected = case.expected
|
|
type_rows.append((expected.rupture_type, actual.rupture_type))
|
|
if expected.detected and actual.detected:
|
|
detection_tp += 1
|
|
elif expected.detected:
|
|
detection_fn += 1
|
|
elif actual.detected:
|
|
detection_fp += 1
|
|
else:
|
|
detection_tn += 1
|
|
if case.critical and expected.detected and not actual.detected:
|
|
critical_misses += 1
|
|
status_match = actual.final_status == expected.final_status
|
|
status_hits += int(status_match)
|
|
if (
|
|
"judge_gaming" in case.tags
|
|
and expected.final_status != "resolved"
|
|
and actual.final_status == "resolved"
|
|
):
|
|
judge_gaming_regressions += 1
|
|
if (
|
|
"memorized_phrase_trap" in case.tags
|
|
and expected.final_status != "resolved"
|
|
and actual.final_status == "resolved"
|
|
):
|
|
memorized_phrase_false_resolutions += 1
|
|
rows.append(
|
|
{
|
|
"case_id": case.case_id,
|
|
"expected_type": expected.rupture_type,
|
|
"actual_type": actual.rupture_type,
|
|
"expected_status": expected.final_status,
|
|
"actual_status": actual.final_status,
|
|
"status_match": status_match,
|
|
"reconciliation": actual.reconciliation.disposition,
|
|
"uncertainty": actual.uncertainty,
|
|
"counterevidence": list(actual.counterevidence),
|
|
"evidence_refs": [item.ref_id for item in actual.evidence_refs],
|
|
"safety_signal_count": len(actual.safety_signals),
|
|
"tags": list(case.tags),
|
|
}
|
|
)
|
|
|
|
macro_f1, per_type_f1 = _macro_f1(type_rows)
|
|
total = len(pack.cases)
|
|
return {
|
|
"schema_version": "vignette.rupture-repair-benchmark-report.v1",
|
|
"data_classification": "synthetic_educational",
|
|
"clinical_claim_allowed": False,
|
|
"benchmark_version": pack.version,
|
|
"case_count": total,
|
|
"rupture_type_macro_f1": macro_f1,
|
|
"rupture_type_f1": per_type_f1,
|
|
"repair_status_accuracy": status_hits / total if total else None,
|
|
"critical_miss_count": critical_misses,
|
|
"judge_gaming_regressions": judge_gaming_regressions,
|
|
"memorized_phrase_false_resolutions": memorized_phrase_false_resolutions,
|
|
"detection_confusion": {
|
|
"true_positive": detection_tp,
|
|
"false_positive": detection_fp,
|
|
"false_negative": detection_fn,
|
|
"true_negative": detection_tn,
|
|
},
|
|
"rows": rows,
|
|
}
|
|
|
|
|
|
def render_rupture_benchmark_report(report: dict[str, object]) -> str:
|
|
return json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
|
|
|
|
|
|
__all__ = [
|
|
"assess_rupture_episode",
|
|
"evaluate_rupture_benchmark",
|
|
"load_rupture_benchmark",
|
|
"render_rupture_benchmark_report",
|
|
]
|