G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
373
apps/api/app/contracts/measurement.py
Normal file
373
apps/api/app/contracts/measurement.py
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
"""Outcome & Alliance OS 측정 원장의 Python 계약.
|
||||
|
||||
점수의 숫자 자체보다 출처, 관점, 도구, 모델 실행, 근거를 먼저 고정한다.
|
||||
이 모듈은 DB/API/프론트 계약이 맞춰야 하는 의미론적 SSOT다.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
SOURCE_KINDS = (
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime",
|
||||
)
|
||||
SourceKind = Literal[
|
||||
"simulated_state",
|
||||
"model_inferred",
|
||||
"agent_reported",
|
||||
"learner_reported",
|
||||
"human_rated",
|
||||
"observed_runtime",
|
||||
]
|
||||
|
||||
MEASUREMENT_CONSTRUCTS = (
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress",
|
||||
)
|
||||
MeasurementConstruct = Literal[
|
||||
"working_alliance",
|
||||
"session_outcome",
|
||||
"rupture_repair",
|
||||
"counselor_skill",
|
||||
"self_calibration",
|
||||
"transfer",
|
||||
"simulation_progress",
|
||||
]
|
||||
|
||||
MEASUREMENT_PERSPECTIVES = (
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation",
|
||||
)
|
||||
MeasurementPerspective = Literal[
|
||||
"client_agent_report",
|
||||
"learner_self_report",
|
||||
"independent_observer",
|
||||
"supervisor_human",
|
||||
"client_simulation",
|
||||
"runtime_observation",
|
||||
]
|
||||
|
||||
MEASUREMENT_STATUSES = ("ready", "degraded", "error", "rejected")
|
||||
MeasurementStatus = Literal["ready", "degraded", "error", "rejected"]
|
||||
|
||||
INSTRUMENT_KINDS = (
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric",
|
||||
)
|
||||
InstrumentKind = Literal[
|
||||
"validated_measure",
|
||||
"training_metric",
|
||||
"simulation_signal",
|
||||
"runtime_metric",
|
||||
]
|
||||
|
||||
AI_VIEWS = ("client", "counselor", "evaluator", "supervisor", "research")
|
||||
AIView = Literal["client", "counselor", "evaluator", "supervisor", "research"]
|
||||
|
||||
MODEL_RUN_STATUSES = ("ready", "degraded", "error")
|
||||
ModelRunStatus = Literal["ready", "degraded", "error"]
|
||||
|
||||
ALLIANCE_DIMENSIONS = ("goal", "task", "bond")
|
||||
AllianceDimension = Literal["goal", "task", "bond"]
|
||||
|
||||
ALLIANCE_CHECKPOINTS = ("pre", "mid", "post")
|
||||
AllianceCheckpoint = Literal["pre", "mid", "post"]
|
||||
|
||||
SOURCE_PERSPECTIVE_COMPATIBILITY: dict[str, frozenset[str]] = {
|
||||
"simulated_state": frozenset({"client_simulation"}),
|
||||
"model_inferred": frozenset({"independent_observer"}),
|
||||
"agent_reported": frozenset({"client_agent_report"}),
|
||||
"learner_reported": frozenset({"learner_self_report"}),
|
||||
"human_rated": frozenset({"supervisor_human"}),
|
||||
"observed_runtime": frozenset({"runtime_observation"}),
|
||||
}
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class MeasurementInstrument(BaseModel):
|
||||
"""척도·훈련지표·시뮬레이션 신호의 버전 고정 레지스트리."""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
instrument_id: str = Field(min_length=1, max_length=120)
|
||||
instrument_version: str = Field(min_length=1, max_length=40)
|
||||
name_ko: str = Field(min_length=1, max_length=200)
|
||||
instrument_kind: InstrumentKind
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
language: str = Field(default="ko-KR", min_length=2, max_length=35)
|
||||
license_id: str | None = Field(default=None, max_length=200)
|
||||
validation_basis: str = Field(min_length=1, max_length=1000)
|
||||
scoring_schema: dict[str, Any] = Field(default_factory=dict)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@field_validator("instrument_id", "instrument_version")
|
||||
@classmethod
|
||||
def strip_identifiers(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("instrument identifiers must not be blank")
|
||||
return stripped
|
||||
|
||||
|
||||
class ModelRun(BaseModel):
|
||||
"""측정을 만든 모델 실행의 재현·드리프트 감사 계약."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
model_run_id: UUID = Field(default_factory=uuid4)
|
||||
session_id: UUID | None = None
|
||||
turn_id: UUID | None = None
|
||||
agent_role: Literal["client", "evaluator", "coach", "scenario", "research"]
|
||||
provider: str = Field(min_length=1, max_length=80)
|
||||
model: str = Field(min_length=1, max_length=160)
|
||||
prompt_bundle_id: str = Field(min_length=1, max_length=160)
|
||||
prompt_bundle_version: str = Field(min_length=1, max_length=40)
|
||||
prompt_bundle_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
structured_schema_version: str = Field(min_length=1, max_length=80)
|
||||
input_evidence_hash: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
status: ModelRunStatus = "ready"
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_error_code_for_failed_run(self) -> "ModelRun":
|
||||
if self.status == "error" and not self.error_code:
|
||||
raise ValueError("error model runs require error_code")
|
||||
if self.status == "ready" and self.error_code:
|
||||
raise ValueError("ready model runs cannot carry error_code")
|
||||
return self
|
||||
|
||||
|
||||
class MeasurementEvent(BaseModel):
|
||||
"""append-only 측정 이벤트.
|
||||
|
||||
정정은 기존 행 변경이 아니라 새 이벤트의 ``supersedes_id``로 표현한다.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
measurement_id: UUID = Field(default_factory=uuid4)
|
||||
session_id: UUID
|
||||
pulse_id: UUID | None = None
|
||||
turn_id: UUID | None = None
|
||||
supersedes_id: UUID | None = None
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
dimension: str = Field(min_length=1, max_length=120)
|
||||
perspective: MeasurementPerspective
|
||||
source_kind: SourceKind
|
||||
instrument_id: str = Field(min_length=1, max_length=120)
|
||||
instrument_version: str = Field(min_length=1, max_length=40)
|
||||
value: float | None = None
|
||||
scale_min: float
|
||||
scale_max: float
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
status: MeasurementStatus = "ready"
|
||||
error_code: str | None = Field(default=None, max_length=120)
|
||||
evidence_turn_ids: tuple[UUID, ...] = ()
|
||||
model_run_id: UUID | None = None
|
||||
visible_to: tuple[AIView, ...] = ("evaluator",)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=_utc_now)
|
||||
|
||||
@field_validator("dimension", "instrument_id", "instrument_version")
|
||||
@classmethod
|
||||
def strip_required_text(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("measurement identifiers must not be blank")
|
||||
return stripped
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_measurement_truth(self) -> "MeasurementEvent":
|
||||
allowed = SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]
|
||||
if self.perspective not in allowed:
|
||||
raise ValueError(
|
||||
f"source_kind={self.source_kind} is incompatible with perspective={self.perspective}"
|
||||
)
|
||||
if self.scale_max <= self.scale_min:
|
||||
raise ValueError("scale_max must be greater than scale_min")
|
||||
if self.value is not None and not self.scale_min <= self.value <= self.scale_max:
|
||||
raise ValueError("measurement value must stay inside its declared scale")
|
||||
if self.status == "ready" and self.value is None:
|
||||
raise ValueError("ready measurements require a value")
|
||||
if self.status in {"error", "rejected"}:
|
||||
if self.value is not None:
|
||||
raise ValueError("error/rejected measurements cannot carry a score")
|
||||
if not self.error_code:
|
||||
raise ValueError("error/rejected measurements require error_code")
|
||||
if self.status == "ready" and self.error_code:
|
||||
raise ValueError("ready measurements cannot carry error_code")
|
||||
if self.source_kind in {"model_inferred", "agent_reported"} and self.model_run_id is None:
|
||||
raise ValueError("model/agent measurements require model_run_id provenance")
|
||||
if self.supersedes_id == self.measurement_id:
|
||||
raise ValueError("a measurement cannot supersede itself")
|
||||
if len(set(self.evidence_turn_ids)) != len(self.evidence_turn_ids):
|
||||
raise ValueError("evidence_turn_ids must be unique")
|
||||
if not self.visible_to:
|
||||
raise ValueError("visible_to must contain at least one audience")
|
||||
if len(set(self.visible_to)) != len(self.visible_to):
|
||||
raise ValueError("visible_to must not contain duplicates")
|
||||
return self
|
||||
|
||||
|
||||
class AllianceScores(BaseModel):
|
||||
"""goal/task/bond를 서로 가리지 않는 독립 0..1 점수."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
goal: float = Field(ge=0.0, le=1.0)
|
||||
task: float = Field(ge=0.0, le=1.0)
|
||||
bond: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class AllianceDimensionAssessment(BaseModel):
|
||||
"""한 관점의 한 동맹 축 평가와 transcript 근거."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
score: float = Field(ge=0.0, le=1.0)
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
evidence_turn_indices: tuple[int, ...] = Field(min_length=1, max_length=12)
|
||||
rationale: str = Field(min_length=1, max_length=1200)
|
||||
|
||||
@field_validator("evidence_turn_indices")
|
||||
@classmethod
|
||||
def keep_evidence_indices_unique(cls, value: tuple[int, ...]) -> tuple[int, ...]:
|
||||
if min(value) < 0:
|
||||
raise ValueError("alliance evidence indices must be non-negative")
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("alliance evidence indices must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class AllianceAgentAssessment(BaseModel):
|
||||
"""client-agent 또는 observer가 독립 실행으로 만든 3축 평가."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
goal: AllianceDimensionAssessment
|
||||
task: AllianceDimensionAssessment
|
||||
bond: AllianceDimensionAssessment
|
||||
|
||||
def by_dimension(self) -> dict[AllianceDimension, AllianceDimensionAssessment]:
|
||||
return {"goal": self.goal, "task": self.task, "bond": self.bond}
|
||||
|
||||
|
||||
class BenchmarkTurn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
speaker: Literal["counselor", "client"]
|
||||
text: str = Field(min_length=1)
|
||||
|
||||
|
||||
class BenchmarkExpectation(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
populate_by_name=True,
|
||||
protected_namespaces=(),
|
||||
)
|
||||
|
||||
construct_key: MeasurementConstruct = Field(alias="construct")
|
||||
dimension: str = Field(min_length=1)
|
||||
perspective: MeasurementPerspective
|
||||
source_kind: SourceKind
|
||||
direction: Literal["low", "mid", "high", "drop", "rise", "detected", "not_detected"]
|
||||
evidence_turn_indices: tuple[int, ...] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_expectation_truth(self) -> "BenchmarkExpectation":
|
||||
if self.perspective not in SOURCE_PERSPECTIVE_COMPATIBILITY[self.source_kind]:
|
||||
raise ValueError("benchmark expectation mixes source and perspective layers")
|
||||
if min(self.evidence_turn_indices) < 0:
|
||||
raise ValueError("benchmark turn indices must be zero-based and non-negative")
|
||||
return self
|
||||
|
||||
|
||||
class BenchmarkCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
|
||||
|
||||
case_id: str = Field(pattern=r"^oas-g0-[0-9]{3}$")
|
||||
version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
scene_type: Literal[
|
||||
"goal_mismatch",
|
||||
"task_mismatch",
|
||||
"empathic_miss",
|
||||
"withdrawal",
|
||||
"confrontation",
|
||||
"successful_repair",
|
||||
"failed_repair",
|
||||
"warm_but_directionless",
|
||||
]
|
||||
title_ko: str = Field(min_length=1)
|
||||
description_ko: str = Field(min_length=1)
|
||||
turns: tuple[BenchmarkTurn, ...] = Field(min_length=2)
|
||||
expected: tuple[BenchmarkExpectation, ...] = Field(min_length=1)
|
||||
forbidden_claims: tuple[str, ...] = Field(min_length=1)
|
||||
tags: tuple[str, ...] = ()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def keep_evidence_inside_scene(self) -> "BenchmarkCase":
|
||||
for expectation in self.expected:
|
||||
if max(expectation.evidence_turn_indices) >= len(self.turns):
|
||||
raise ValueError("benchmark evidence points outside the scene")
|
||||
return self
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AI_VIEWS",
|
||||
"ALLIANCE_CHECKPOINTS",
|
||||
"ALLIANCE_DIMENSIONS",
|
||||
"AllianceAgentAssessment",
|
||||
"AllianceDimensionAssessment",
|
||||
"AllianceScores",
|
||||
"BenchmarkCase",
|
||||
"BenchmarkExpectation",
|
||||
"BenchmarkTurn",
|
||||
"INSTRUMENT_KINDS",
|
||||
"MEASUREMENT_CONSTRUCTS",
|
||||
"MEASUREMENT_PERSPECTIVES",
|
||||
"MEASUREMENT_STATUSES",
|
||||
"MODEL_RUN_STATUSES",
|
||||
"MeasurementEvent",
|
||||
"MeasurementInstrument",
|
||||
"ModelRun",
|
||||
"SOURCE_KINDS",
|
||||
"SOURCE_PERSPECTIVE_COMPATIBILITY",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue