16e791e044
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
373 lines
13 KiB
Python
373 lines
13 KiB
Python
"""Outcome & Alliance OS 측정 원장의 Python 계약.
|
|
|
|
점수의 숫자 자체보다 출처, 관점, 도구, 모델 실행, 근거를 먼저 고정한다.
|
|
이 모듈은 DB/API/프론트 계약이 맞춰야 하는 의미론적 SSOT다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Literal
|
|
from uuid import UUID, uuid4
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
|
|
SOURCE_KINDS = (
|
|
"simulated_state",
|
|
"model_inferred",
|
|
"agent_reported",
|
|
"learner_reported",
|
|
"human_rated",
|
|
"observed_runtime",
|
|
)
|
|
SourceKind = Literal[
|
|
"simulated_state",
|
|
"model_inferred",
|
|
"agent_reported",
|
|
"learner_reported",
|
|
"human_rated",
|
|
"observed_runtime",
|
|
]
|
|
|
|
MEASUREMENT_CONSTRUCTS = (
|
|
"working_alliance",
|
|
"session_outcome",
|
|
"rupture_repair",
|
|
"counselor_skill",
|
|
"self_calibration",
|
|
"transfer",
|
|
"simulation_progress",
|
|
)
|
|
MeasurementConstruct = Literal[
|
|
"working_alliance",
|
|
"session_outcome",
|
|
"rupture_repair",
|
|
"counselor_skill",
|
|
"self_calibration",
|
|
"transfer",
|
|
"simulation_progress",
|
|
]
|
|
|
|
MEASUREMENT_PERSPECTIVES = (
|
|
"client_agent_report",
|
|
"learner_self_report",
|
|
"independent_observer",
|
|
"supervisor_human",
|
|
"client_simulation",
|
|
"runtime_observation",
|
|
)
|
|
MeasurementPerspective = Literal[
|
|
"client_agent_report",
|
|
"learner_self_report",
|
|
"independent_observer",
|
|
"supervisor_human",
|
|
"client_simulation",
|
|
"runtime_observation",
|
|
]
|
|
|
|
MEASUREMENT_STATUSES = ("ready", "degraded", "error", "rejected")
|
|
MeasurementStatus = Literal["ready", "degraded", "error", "rejected"]
|
|
|
|
INSTRUMENT_KINDS = (
|
|
"validated_measure",
|
|
"training_metric",
|
|
"simulation_signal",
|
|
"runtime_metric",
|
|
)
|
|
InstrumentKind = Literal[
|
|
"validated_measure",
|
|
"training_metric",
|
|
"simulation_signal",
|
|
"runtime_metric",
|
|
]
|
|
|
|
AI_VIEWS = ("client", "counselor", "evaluator", "supervisor", "research")
|
|
AIView = Literal["client", "counselor", "evaluator", "supervisor", "research"]
|
|
|
|
MODEL_RUN_STATUSES = ("ready", "degraded", "error")
|
|
ModelRunStatus = Literal["ready", "degraded", "error"]
|
|
|
|
ALLIANCE_DIMENSIONS = ("goal", "task", "bond")
|
|
AllianceDimension = Literal["goal", "task", "bond"]
|
|
|
|
ALLIANCE_CHECKPOINTS = ("pre", "mid", "post")
|
|
AllianceCheckpoint = Literal["pre", "mid", "post"]
|
|
|
|
SOURCE_PERSPECTIVE_COMPATIBILITY: dict[str, frozenset[str]] = {
|
|
"simulated_state": frozenset({"client_simulation"}),
|
|
"model_inferred": frozenset({"independent_observer"}),
|
|
"agent_reported": frozenset({"client_agent_report"}),
|
|
"learner_reported": frozenset({"learner_self_report"}),
|
|
"human_rated": frozenset({"supervisor_human"}),
|
|
"observed_runtime": frozenset({"runtime_observation"}),
|
|
}
|
|
|
|
|
|
def _utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class MeasurementInstrument(BaseModel):
|
|
"""척도·훈련지표·시뮬레이션 신호의 버전 고정 레지스트리."""
|
|
|
|
model_config = ConfigDict(
|
|
extra="forbid",
|
|
frozen=True,
|
|
populate_by_name=True,
|
|
protected_namespaces=(),
|
|
)
|
|
|
|
instrument_id: str = Field(min_length=1, max_length=120)
|
|
instrument_version: str = Field(min_length=1, max_length=40)
|
|
name_ko: str = Field(min_length=1, max_length=200)
|
|
instrument_kind: InstrumentKind
|
|
construct_key: MeasurementConstruct = Field(alias="construct")
|
|
language: str = Field(default="ko-KR", min_length=2, max_length=35)
|
|
license_id: str | None = Field(default=None, max_length=200)
|
|
validation_basis: str = Field(min_length=1, max_length=1000)
|
|
scoring_schema: dict[str, Any] = Field(default_factory=dict)
|
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
created_at: datetime = Field(default_factory=_utc_now)
|
|
|
|
@field_validator("instrument_id", "instrument_version")
|
|
@classmethod
|
|
def strip_identifiers(cls, value: str) -> str:
|
|
stripped = value.strip()
|
|
if not stripped:
|
|
raise ValueError("instrument identifiers must not be blank")
|
|
return stripped
|
|
|
|
|
|
class ModelRun(BaseModel):
|
|
"""측정을 만든 모델 실행의 재현·드리프트 감사 계약."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
model_run_id: UUID = Field(default_factory=uuid4)
|
|
session_id: UUID | None = None
|
|
turn_id: UUID | None = None
|
|
agent_role: Literal["client", "evaluator", "coach", "scenario", "research"]
|
|
provider: str = Field(min_length=1, max_length=80)
|
|
model: str = Field(min_length=1, max_length=160)
|
|
prompt_bundle_id: str = Field(min_length=1, max_length=160)
|
|
prompt_bundle_version: str = Field(min_length=1, max_length=40)
|
|
prompt_bundle_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
|
structured_schema_version: str = Field(min_length=1, max_length=80)
|
|
input_evidence_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
|
status: ModelRunStatus = "ready"
|
|
error_code: str | None = Field(default=None, max_length=120)
|
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
created_at: datetime = Field(default_factory=_utc_now)
|
|
|
|
@model_validator(mode="after")
|
|
def require_error_code_for_failed_run(self) -> "ModelRun":
|
|
if self.status == "error" and not self.error_code:
|
|
raise ValueError("error model runs require error_code")
|
|
if self.status == "ready" and self.error_code:
|
|
raise ValueError("ready model runs cannot carry error_code")
|
|
return self
|
|
|
|
|
|
class MeasurementEvent(BaseModel):
|
|
"""append-only 측정 이벤트.
|
|
|
|
정정은 기존 행 변경이 아니라 새 이벤트의 ``supersedes_id``로 표현한다.
|
|
"""
|
|
|
|
model_config = ConfigDict(
|
|
extra="forbid",
|
|
frozen=True,
|
|
populate_by_name=True,
|
|
protected_namespaces=(),
|
|
)
|
|
|
|
measurement_id: UUID = Field(default_factory=uuid4)
|
|
session_id: UUID
|
|
pulse_id: UUID | None = None
|
|
turn_id: UUID | None = None
|
|
supersedes_id: UUID | None = None
|
|
construct_key: MeasurementConstruct = Field(alias="construct")
|
|
dimension: str = Field(min_length=1, max_length=120)
|
|
perspective: MeasurementPerspective
|
|
source_kind: SourceKind
|
|
instrument_id: str = Field(min_length=1, max_length=120)
|
|
instrument_version: str = Field(min_length=1, max_length=40)
|
|
value: float | None = None
|
|
scale_min: float
|
|
scale_max: float
|
|
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
|
status: MeasurementStatus = "ready"
|
|
error_code: str | None = Field(default=None, max_length=120)
|
|
evidence_turn_ids: tuple[UUID, ...] = ()
|
|
model_run_id: UUID | None = None
|
|
visible_to: tuple[AIView, ...] = ("evaluator",)
|
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
created_at: datetime = Field(default_factory=_utc_now)
|
|
|
|
@field_validator("dimension", "instrument_id", "instrument_version")
|
|
@classmethod
|
|
def strip_required_text(cls, value: str) -> str:
|
|
stripped = value.strip()
|
|
if not stripped:
|
|
raise ValueError("measurement identifiers must not be blank")
|
|
return stripped
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_measurement_truth(self) -> "MeasurementEvent":
|
|
allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]
|
|
if self.perspective not in allowed:
|
|
raise ValueError(
|
|
f"source_kind={self.source_kind} is incompatible with perspective={self.perspective}"
|
|
)
|
|
if self.scale_max <= self.scale_min:
|
|
raise ValueError("scale_max must be greater than scale_min")
|
|
if self.value is not None and not self.scale_min <= self.value <= self.scale_max:
|
|
raise ValueError("measurement value must stay inside its declared scale")
|
|
if self.status == "ready" and self.value is None:
|
|
raise ValueError("ready measurements require a value")
|
|
if self.status in {"error", "rejected"}:
|
|
if self.value is not None:
|
|
raise ValueError("error/rejected measurements cannot carry a score")
|
|
if not self.error_code:
|
|
raise ValueError("error/rejected measurements require error_code")
|
|
if self.status == "ready" and self.error_code:
|
|
raise ValueError("ready measurements cannot carry error_code")
|
|
if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None:
|
|
raise ValueError("model/agent measurements require model_run_id provenance")
|
|
if self.supersedes_id == self.measurement_id:
|
|
raise ValueError("a measurement cannot supersede itself")
|
|
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
|
raise ValueError("evidence_turn_ids must be unique")
|
|
if not self.visible_to:
|
|
raise ValueError("visible_to must contain at least one audience")
|
|
if len(set(self.visible_to)) != len(self.visible_to):
|
|
raise ValueError("visible_to must not contain duplicates")
|
|
return self
|
|
|
|
|
|
class AllianceScores(BaseModel):
|
|
"""goal/task/bond를 서로 가리지 않는 독립 0..1 점수."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
goal: float = Field(ge=0.0, le=1.0)
|
|
task: float = Field(ge=0.0, le=1.0)
|
|
bond: float = Field(ge=0.0, le=1.0)
|
|
|
|
|
|
class AllianceDimensionAssessment(BaseModel):
|
|
"""한 관점의 한 동맹 축 평가와 transcript 근거."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
score: float = Field(ge=0.0, le=1.0)
|
|
confidence: float = Field(ge=0.0, le=1.0)
|
|
evidence_turn_indices: tuple[int, ...] = Field(min_length=1, max_length=12)
|
|
rationale: str = Field(min_length=1, max_length=1200)
|
|
|
|
@field_validator("evidence_turn_indices")
|
|
@classmethod
|
|
def keep_evidence_indices_unique(cls, value: tuple[int, ...]) -> tuple[int, ...]:
|
|
if min(value) < 0:
|
|
raise ValueError("alliance evidence indices must be non-negative")
|
|
if len(set(value)) != len(value):
|
|
raise ValueError("alliance evidence indices must be unique")
|
|
return value
|
|
|
|
|
|
class AllianceAgentAssessment(BaseModel):
|
|
"""client-agent 또는 observer가 독립 실행으로 만든 3축 평가."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
goal: AllianceDimensionAssessment
|
|
task: AllianceDimensionAssessment
|
|
bond: AllianceDimensionAssessment
|
|
|
|
def by_dimension(self) -> dict[AllianceDimension, AllianceDimensionAssessment]:
|
|
return {"goal": self.goal, "task": self.task, "bond": self.bond}
|
|
|
|
|
|
class BenchmarkTurn(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
speaker: Literal["counselor", "client"]
|
|
text: str = Field(min_length=1)
|
|
|
|
|
|
class BenchmarkExpectation(BaseModel):
|
|
model_config = ConfigDict(
|
|
extra="forbid",
|
|
frozen=True,
|
|
populate_by_name=True,
|
|
protected_namespaces=(),
|
|
)
|
|
|
|
construct_key: MeasurementConstruct = Field(alias="construct")
|
|
dimension: str = Field(min_length=1)
|
|
perspective: MeasurementPerspective
|
|
source_kind: SourceKind
|
|
direction: Literal["low", "mid", "high", "drop", "rise", "detected", "not_detected"]
|
|
evidence_turn_indices: tuple[int, ...] = Field(min_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def enforce_expectation_truth(self) -> "BenchmarkExpectation":
|
|
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
|
raise ValueError("benchmark expectation mixes source and perspective layers")
|
|
if min(self.evidence_turn_indices) < 0:
|
|
raise ValueError("benchmark turn indices must be zero-based and non-negative")
|
|
return self
|
|
|
|
|
|
class BenchmarkCase(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
|
|
|
case_id: str = Field(pattern=r"^oas-g0-[0-9]{3}$")
|
|
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
|
scene_type: Literal[
|
|
"goal_mismatch",
|
|
"task_mismatch",
|
|
"empathic_miss",
|
|
"withdrawal",
|
|
"confrontation",
|
|
"successful_repair",
|
|
"failed_repair",
|
|
"warm_but_directionless",
|
|
]
|
|
title_ko: str = Field(min_length=1)
|
|
description_ko: str = Field(min_length=1)
|
|
turns: tuple[BenchmarkTurn, ...] = Field(min_length=2)
|
|
expected: tuple[BenchmarkExpectation, ...] = Field(min_length=1)
|
|
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
|
tags: tuple[str, ...] = ()
|
|
|
|
@model_validator(mode="after")
|
|
def keep_evidence_inside_scene(self) -> "BenchmarkCase":
|
|
for expectation in self.expected:
|
|
if max(expectation.evidence_turn_indices) >= len(self.turns):
|
|
raise ValueError("benchmark evidence points outside the scene")
|
|
return self
|
|
|
|
|
|
__all__ = [
|
|
"AI_VIEWS",
|
|
"ALLIANCE_CHECKPOINTS",
|
|
"ALLIANCE_DIMENSIONS",
|
|
"AllianceAgentAssessment",
|
|
"AllianceDimensionAssessment",
|
|
"AllianceScores",
|
|
"BenchmarkCase",
|
|
"BenchmarkExpectation",
|
|
"BenchmarkTurn",
|
|
"INSTRUMENT_KINDS",
|
|
"MEASUREMENT_CONSTRUCTS",
|
|
"MEASUREMENT_PERSPECTIVES",
|
|
"MEASUREMENT_STATUSES",
|
|
"MODEL_RUN_STATUSES",
|
|
"MeasurementEvent",
|
|
"MeasurementInstrument",
|
|
"ModelRun",
|
|
"SOURCE_KINDS",
|
|
"SOURCE_PERSPECTIVE_COMPATIBILITY",
|
|
]
|