16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
343 lines
11 KiB
Python
343 lines
11 KiB
Python
"""기존 Vignette 신호를 측정 원장 의미로 안전하게 투영한다.
|
|
|
|
legacy 이름을 임상 구성개념으로 승격하지 않는다. 특히 ``rapport_credit``와
|
|
``case_profile.alliance_level``은 독립 동맹 측정이 아니라 시뮬레이션 진행 신호다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, replace
|
|
from datetime import datetime
|
|
from typing import Any, Iterable, Mapping
|
|
from uuid import UUID
|
|
|
|
from ..contracts.measurement import MeasurementEvent
|
|
from .evaluation_contract import GROWTH_APPROPRIATENESS_SCORE_01
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class LegacySignalDefinition:
|
|
signal: str
|
|
source_kind: str
|
|
perspective: str
|
|
construct: str
|
|
dimension: str
|
|
instrument_id: str
|
|
instrument_version: str
|
|
scale_min: float
|
|
scale_max: float
|
|
clinical_claim_allowed: bool
|
|
note: str
|
|
|
|
|
|
LEGACY_SIGNAL_INVENTORY: tuple[LegacySignalDefinition, ...] = (
|
|
LegacySignalDefinition(
|
|
signal="session_state.rapport_credit",
|
|
source_kind="simulated_state",
|
|
perspective="client_simulation",
|
|
construct="simulation_progress",
|
|
dimension="rapport_credit",
|
|
instrument_id="vignette-state-machine",
|
|
instrument_version="legacy-1",
|
|
scale_min=0.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="상담자 발화 규칙에 따른 결정론적 누적 상태다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="case_profile.alliance_level",
|
|
source_kind="simulated_state",
|
|
perspective="client_simulation",
|
|
construct="simulation_progress",
|
|
dimension="legacy_alliance_level",
|
|
instrument_id="vignette-alliance-ewma",
|
|
instrument_version="legacy-1",
|
|
scale_min=0.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="rapport_credit 회기말 값의 EWMA이며 Working Alliance 측정이 아니다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="TurnEvaluation.appropriateness",
|
|
source_kind="model_inferred",
|
|
perspective="independent_observer",
|
|
construct="counselor_skill",
|
|
dimension="appropriateness",
|
|
instrument_id="vignette-fast-evaluator",
|
|
instrument_version="legacy-1",
|
|
scale_min=0.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="LLM 기반 경량 훈련 피드백 신호다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="TurnEvaluation.rapport_signal",
|
|
source_kind="model_inferred",
|
|
perspective="independent_observer",
|
|
construct="counselor_skill",
|
|
dimension="rapport_signal",
|
|
instrument_id="vignette-fast-evaluator",
|
|
instrument_version="legacy-1",
|
|
scale_min=-1.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="평가 모델이 추정한 턴 단위 라포 방향 신호다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="SessionEvaluation.distribution",
|
|
source_kind="model_inferred",
|
|
perspective="independent_observer",
|
|
construct="counselor_skill",
|
|
dimension="technique_occurrence_count",
|
|
instrument_id="vignette-deep-evaluator",
|
|
instrument_version="legacy-1",
|
|
scale_min=0.0,
|
|
scale_max=1000.0,
|
|
clinical_claim_allowed=False,
|
|
note="모델 태그의 빈도이며 숙련도·치료성과가 아니다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="Phase3.prepost",
|
|
source_kind="learner_reported",
|
|
perspective="learner_self_report",
|
|
construct="self_calibration",
|
|
dimension="self_reported_training_change",
|
|
instrument_id="phase3-prepost",
|
|
instrument_version="design-1",
|
|
scale_min=0.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="문항·타당화가 확정되기 전까지 교육 파일럿 자기보고다.",
|
|
),
|
|
LegacySignalDefinition(
|
|
signal="Phase3.runtime_kpi",
|
|
source_kind="observed_runtime",
|
|
perspective="runtime_observation",
|
|
construct="transfer",
|
|
dimension="pilot_runtime_metric",
|
|
instrument_id="phase3-runtime-kpi",
|
|
instrument_version="design-1",
|
|
scale_min=0.0,
|
|
scale_max=1.0,
|
|
clinical_claim_allowed=False,
|
|
note="완주·환각검수·IAA 등 파일럿 증거이며 개인 치료성과가 아니다.",
|
|
),
|
|
)
|
|
|
|
_INVENTORY_BY_SIGNAL = {item.signal: item for item in LEGACY_SIGNAL_INVENTORY}
|
|
|
|
|
|
def _event(
|
|
definition: LegacySignalDefinition,
|
|
*,
|
|
session_id: UUID,
|
|
value: float | None,
|
|
turn_id: UUID | None = None,
|
|
model_run_id: UUID | None = None,
|
|
evidence_turn_ids: tuple[UUID, ...] = (),
|
|
status: str = "ready",
|
|
error_code: str | None = None,
|
|
created_at: datetime | None = None,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
) -> MeasurementEvent:
|
|
payload: dict[str, Any] = {
|
|
"session_id": session_id,
|
|
"turn_id": turn_id,
|
|
"construct": definition.construct,
|
|
"dimension": definition.dimension,
|
|
"perspective": definition.perspective,
|
|
"source_kind": definition.source_kind,
|
|
"instrument_id": definition.instrument_id,
|
|
"instrument_version": definition.instrument_version,
|
|
"value": value,
|
|
"scale_min": definition.scale_min,
|
|
"scale_max": definition.scale_max,
|
|
"status": status,
|
|
"error_code": error_code,
|
|
"evidence_turn_ids": evidence_turn_ids,
|
|
"model_run_id": model_run_id,
|
|
"visible_to": ("evaluator", "supervisor"),
|
|
"metadata": {
|
|
"legacy_signal": definition.signal,
|
|
"clinical_claim_allowed": definition.clinical_claim_allowed,
|
|
"provenance_note": definition.note,
|
|
**dict(metadata or {}),
|
|
},
|
|
}
|
|
if created_at is not None:
|
|
payload["created_at"] = created_at
|
|
return MeasurementEvent.model_validate(payload)
|
|
|
|
|
|
def adapt_legacy_simulation_signals(
|
|
*,
|
|
session_id: UUID,
|
|
rapport_credit: float,
|
|
alliance_level: float,
|
|
turn_id: UUID | None = None,
|
|
) -> tuple[MeasurementEvent, MeasurementEvent]:
|
|
"""기존 두 값을 ``working_alliance``가 아닌 simulation_progress로 보존한다."""
|
|
|
|
evidence = (turn_id,) if turn_id is not None else ()
|
|
return (
|
|
_event(
|
|
_INVENTORY_BY_SIGNAL["session_state.rapport_credit"],
|
|
session_id=session_id,
|
|
turn_id=turn_id,
|
|
value=float(rapport_credit),
|
|
evidence_turn_ids=evidence,
|
|
),
|
|
_event(
|
|
_INVENTORY_BY_SIGNAL["case_profile.alliance_level"],
|
|
session_id=session_id,
|
|
turn_id=turn_id,
|
|
value=float(alliance_level),
|
|
evidence_turn_ids=evidence,
|
|
),
|
|
)
|
|
|
|
|
|
def adapt_fast_evaluation(
|
|
*,
|
|
session_id: UUID,
|
|
turn_id: UUID,
|
|
evaluation: Mapping[str, Any],
|
|
model_run_id: UUID,
|
|
) -> tuple[MeasurementEvent, ...]:
|
|
"""fast-loop 평가를 모델 추정 훈련지표로 명시한다."""
|
|
|
|
error = str(evaluation.get("error") or "").strip() or None
|
|
if error:
|
|
return (
|
|
_event(
|
|
_INVENTORY_BY_SIGNAL["TurnEvaluation.appropriateness"],
|
|
session_id=session_id,
|
|
turn_id=turn_id,
|
|
value=None,
|
|
model_run_id=model_run_id,
|
|
evidence_turn_ids=(turn_id,),
|
|
status="error",
|
|
error_code=error[:120],
|
|
),
|
|
)
|
|
|
|
appropriateness = str(evaluation.get("appropriateness") or "neutral")
|
|
score = GROWTH_APPROPRIATENESS_SCORE_01.get(appropriateness, 0.5)
|
|
events = [
|
|
_event(
|
|
_INVENTORY_BY_SIGNAL["TurnEvaluation.appropriateness"],
|
|
session_id=session_id,
|
|
turn_id=turn_id,
|
|
value=score,
|
|
model_run_id=model_run_id,
|
|
evidence_turn_ids=(turn_id,),
|
|
metadata={"legacy_label": appropriateness},
|
|
)
|
|
]
|
|
rapport_signal = evaluation.get("rapport_signal")
|
|
if isinstance(rapport_signal, (int, float)):
|
|
events.append(
|
|
_event(
|
|
_INVENTORY_BY_SIGNAL["TurnEvaluation.rapport_signal"],
|
|
session_id=session_id,
|
|
turn_id=turn_id,
|
|
value=float(rapport_signal),
|
|
model_run_id=model_run_id,
|
|
evidence_turn_ids=(turn_id,),
|
|
)
|
|
)
|
|
return tuple(events)
|
|
|
|
|
|
def adapt_deep_evaluation(
|
|
*,
|
|
session_id: UUID,
|
|
evaluation: Mapping[str, Any],
|
|
model_run_id: UUID,
|
|
evidence_turn_ids: tuple[UUID, ...] = (),
|
|
) -> MeasurementEvent:
|
|
"""deep-loop 기법 분포를 숙련도가 아닌 모델 태그 빈도로 보존한다."""
|
|
|
|
definition = _INVENTORY_BY_SIGNAL["SessionEvaluation.distribution"]
|
|
error = str(evaluation.get("error") or "").strip() or None
|
|
if error:
|
|
return _event(
|
|
definition,
|
|
session_id=session_id,
|
|
value=None,
|
|
model_run_id=model_run_id,
|
|
evidence_turn_ids=evidence_turn_ids,
|
|
status="error",
|
|
error_code=error[:120],
|
|
)
|
|
distribution = evaluation.get("distribution")
|
|
total = distribution.get("total", 0) if isinstance(distribution, Mapping) else 0
|
|
return _event(
|
|
definition,
|
|
session_id=session_id,
|
|
value=float(max(0, int(total))),
|
|
model_run_id=model_run_id,
|
|
evidence_turn_ids=evidence_turn_ids,
|
|
)
|
|
|
|
|
|
_LEARNER_REPORTED_KPIS = {
|
|
"self_efficacy_prepost",
|
|
"skill_proficiency_prepost",
|
|
"training_satisfaction_prepost",
|
|
"sus",
|
|
}
|
|
|
|
|
|
def adapt_phase3_metric(
|
|
*,
|
|
session_id: UUID,
|
|
metric_name: str,
|
|
value: float | None,
|
|
status: str = "ready",
|
|
) -> MeasurementEvent:
|
|
"""Phase 3 KPI를 자기보고와 운영 관측으로 분리한다."""
|
|
|
|
base = (
|
|
_INVENTORY_BY_SIGNAL["Phase3.prepost"]
|
|
if metric_name in _LEARNER_REPORTED_KPIS
|
|
else _INVENTORY_BY_SIGNAL["Phase3.runtime_kpi"]
|
|
)
|
|
definition = replace(base, dimension=metric_name)
|
|
return _event(
|
|
definition,
|
|
session_id=session_id,
|
|
value=value,
|
|
status=status,
|
|
error_code="metric_not_ready" if status in {"error", "rejected"} else None,
|
|
)
|
|
|
|
|
|
def require_homogeneous_provenance(
|
|
events: Iterable[MeasurementEvent],
|
|
*,
|
|
operation: str,
|
|
) -> tuple[MeasurementEvent, ...]:
|
|
"""서로 다른 출처층을 하나의 평균·총점으로 합치는 것을 차단한다.
|
|
|
|
관점 비교 UI는 이 함수를 호출하지 않고 층별 series를 나란히 표시한다.
|
|
"""
|
|
|
|
materialized = tuple(events)
|
|
layers = {(event.source_kind, event.perspective) for event in materialized}
|
|
if len(layers) > 1:
|
|
raise ValueError(
|
|
f"{operation} cannot aggregate heterogeneous measurement provenance: {sorted(layers)}"
|
|
)
|
|
return materialized
|
|
|
|
|
|
__all__ = [
|
|
"LEGACY_SIGNAL_INVENTORY",
|
|
"LegacySignalDefinition",
|
|
"adapt_deep_evaluation",
|
|
"adapt_fast_evaluation",
|
|
"adapt_legacy_simulation_signals",
|
|
"adapt_phase3_metric",
|
|
"require_homogeneous_provenance",
|
|
]
|