1842 lines
72 KiB
Python
1842 lines
72 KiB
Python
"""Append-only persistence boundary for G5 calibration and unseen transfer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Mapping, Sequence
|
|
from datetime import UTC, datetime
|
|
from typing import Any, Literal
|
|
from uuid import UUID, uuid4, uuid5
|
|
|
|
import asyncpg
|
|
|
|
from .. import db
|
|
from ..contracts.calibration_transfer import (
|
|
ActualTransferExecution,
|
|
CompetencyCalibrationAssessment,
|
|
MetacognitivePrescription,
|
|
NormalizedEvaluatorLabels,
|
|
TransferVariation,
|
|
TransferSuiteInput,
|
|
)
|
|
from ..deps import Principal, Role
|
|
from .calibration_transfer import (
|
|
assess_actual_transfer_executions,
|
|
assess_synthetic_subgroup_drift,
|
|
assess_transfer,
|
|
)
|
|
|
|
|
|
class CalibrationTransferNotFoundError(LookupError):
|
|
pass
|
|
|
|
|
|
class CalibrationTransferConflictError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class CalibrationTransferStateError(ValueError):
|
|
pass
|
|
|
|
|
|
_IDEMPOTENCY_TABLES = {
|
|
"app.calibration_prediction_revision": "prediction_revision_id",
|
|
"app.calibration_prediction_lock": "lock_id",
|
|
"app.calibration_performance_observation": "observation_id",
|
|
"app.calibration_assessment_snapshot": "assessment_snapshot_id",
|
|
"app.calibration_transfer_suite": "transfer_suite_record_id",
|
|
"app.calibration_teacher_review_event": "review_id",
|
|
}
|
|
|
|
_ACTUAL_TRANSFER_NAMESPACE = UUID("4a563cc6-f8d3-51bf-a315-41d2a98b02ce")
|
|
_ACTUAL_TRANSFER_OBSERVER_VERSION = "calibration-actual-transfer-observer-v1"
|
|
_ACTUAL_TRANSFER_INSTRUMENT_ID = "unseen-transfer-g5"
|
|
_ACTUAL_TRANSFER_INSTRUMENT_VERSION = "1.0.0"
|
|
_POSITIVE_CLIENT_STATES = frozenset(
|
|
{
|
|
"affect_contact",
|
|
"thought_organizing",
|
|
"responds_to_exploration",
|
|
"expresses_plan",
|
|
"defense_loosening",
|
|
}
|
|
)
|
|
|
|
|
|
def _value(row: Mapping[str, Any], key: str, default: Any = None) -> Any:
|
|
try:
|
|
return row[key]
|
|
except (KeyError, TypeError):
|
|
return default
|
|
|
|
|
|
def _public_row(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""API 응답에서 학습자 피드백 정책 판정 전용 열을 제거한다."""
|
|
|
|
payload = dict(row)
|
|
payload.pop("source_learner_feedback_enabled", None)
|
|
return payload
|
|
|
|
|
|
def _canonical_hash(payload: Mapping[str, Any]) -> str:
|
|
serialized = json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _created_role(principal: Principal) -> str:
|
|
return "instructor" if principal.role == Role.TEACHER else principal.role.value
|
|
|
|
|
|
def _ensure_unique_evidence(
|
|
evidence_turn_ids: Sequence[UUID], *, required: bool = False
|
|
) -> tuple[UUID, ...]:
|
|
normalized = tuple(evidence_turn_ids)
|
|
if required and not normalized:
|
|
raise CalibrationTransferStateError("evidence_turn_ids must not be empty")
|
|
if len(set(normalized)) != len(normalized):
|
|
raise CalibrationTransferStateError("evidence_turn_ids must be unique")
|
|
return normalized
|
|
|
|
|
|
def _uuid_evidence(refs: Sequence[str], *, required: bool = False) -> tuple[UUID, ...]:
|
|
values: list[UUID] = []
|
|
for ref in refs:
|
|
try:
|
|
values.append(UUID(ref))
|
|
except (TypeError, ValueError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"G5 evidence refs must be transcript turn UUIDs, never transcript text"
|
|
) from exc
|
|
return _ensure_unique_evidence(values, required=required)
|
|
|
|
|
|
async def _existing_by_submission(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
table: str,
|
|
submission_id: UUID,
|
|
content_hash: str,
|
|
) -> UUID | None:
|
|
id_column = _IDEMPOTENCY_TABLES.get(table)
|
|
if id_column is None:
|
|
raise AssertionError("unsupported calibration idempotency table")
|
|
row = await conn.fetchrow(
|
|
f"SELECT {id_column}, content_hash FROM {table} WHERE submission_id = $1",
|
|
submission_id,
|
|
)
|
|
if row is None:
|
|
return None
|
|
if str(_value(row, "content_hash")) != content_hash:
|
|
raise CalibrationTransferConflictError(
|
|
"submission id was already used with different content"
|
|
)
|
|
return UUID(str(_value(row, id_column)))
|
|
|
|
|
|
async def _visible_session(
|
|
conn: asyncpg.Connection, session_id: UUID
|
|
) -> Mapping[str, Any]:
|
|
row = await conn.fetchrow(
|
|
"SELECT id, learner_id, case_id FROM app.sessions WHERE id = $1",
|
|
session_id,
|
|
)
|
|
if row is None:
|
|
raise CalibrationTransferNotFoundError("session not found or not visible")
|
|
return row
|
|
|
|
|
|
async def append_prediction_revision(
|
|
*,
|
|
principal: Principal,
|
|
submission_id: UUID,
|
|
prediction_revision_id: UUID,
|
|
history_id: UUID,
|
|
session_id: UUID,
|
|
competency_id: str,
|
|
practice_block_id: str,
|
|
scenario_variant_id: str,
|
|
phrase_family_id: str,
|
|
revision_no: int,
|
|
supersedes_prediction_revision_id: UUID | None,
|
|
predicted_success_probability: float,
|
|
confidence: float,
|
|
recorded_sequence: int,
|
|
revision_reason: str,
|
|
instrument_id: str,
|
|
instrument_version: str,
|
|
evidence_turn_ids: Sequence[UUID] = (),
|
|
) -> dict[str, Any]:
|
|
if principal.role != Role.LEARNER:
|
|
raise CalibrationTransferStateError(
|
|
"self-prediction revision requires learner role"
|
|
)
|
|
learner_id = UUID(principal.user_id)
|
|
evidence = _ensure_unique_evidence(evidence_turn_ids)
|
|
reason = revision_reason.strip()
|
|
if not reason:
|
|
raise CalibrationTransferStateError("revision_reason must not be blank")
|
|
payload = {
|
|
"prediction_revision_id": str(prediction_revision_id),
|
|
"history_id": str(history_id),
|
|
"session_id": str(session_id),
|
|
"learner_id": str(learner_id),
|
|
"competency_id": competency_id,
|
|
"practice_block_id": practice_block_id,
|
|
"scenario_variant_id": scenario_variant_id,
|
|
"phrase_family_id": phrase_family_id,
|
|
"revision_no": revision_no,
|
|
"supersedes_prediction_revision_id": (
|
|
str(supersedes_prediction_revision_id)
|
|
if supersedes_prediction_revision_id
|
|
else None
|
|
),
|
|
"predicted_success_probability": predicted_success_probability,
|
|
"confidence": confidence,
|
|
"recorded_sequence": recorded_sequence,
|
|
"revision_reason": reason,
|
|
"instrument_id": instrument_id,
|
|
"instrument_version": instrument_version,
|
|
"evidence_turn_ids": sorted(str(item) for item in evidence),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
async with db.acquire(
|
|
role=principal.role.value,
|
|
user_id=principal.user_id,
|
|
cohort_ids=principal.cohort_ids,
|
|
) as conn:
|
|
session = await _visible_session(conn, session_id)
|
|
if UUID(str(_value(session, "learner_id"))) != learner_id:
|
|
raise CalibrationTransferNotFoundError("session does not belong to learner")
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_prediction_revision",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"prediction_revision_id": existing,
|
|
"revision_no": revision_no,
|
|
"idempotent_replay": True,
|
|
}
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 5))",
|
|
str(history_id),
|
|
)
|
|
history = await conn.fetchrow(
|
|
"SELECT * FROM app.calibration_prediction_history WHERE history_id = $1",
|
|
history_id,
|
|
)
|
|
if history is None:
|
|
if revision_no != 1 or supersedes_prediction_revision_id is not None:
|
|
raise CalibrationTransferStateError(
|
|
"new prediction history must start at revision one"
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_prediction_history (
|
|
history_id, session_id, learner_id, competency_id,
|
|
practice_block_id, scenario_variant_id, phrase_family_id,
|
|
created_by_role
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,'learner')
|
|
""",
|
|
history_id,
|
|
session_id,
|
|
learner_id,
|
|
competency_id,
|
|
practice_block_id,
|
|
scenario_variant_id,
|
|
phrase_family_id,
|
|
)
|
|
else:
|
|
expected = (
|
|
str(session_id),
|
|
str(learner_id),
|
|
competency_id,
|
|
practice_block_id,
|
|
scenario_variant_id,
|
|
phrase_family_id,
|
|
)
|
|
actual = (
|
|
str(_value(history, "session_id")),
|
|
str(_value(history, "learner_id")),
|
|
str(_value(history, "competency_id")),
|
|
str(_value(history, "practice_block_id")),
|
|
str(_value(history, "scenario_variant_id")),
|
|
str(_value(history, "phrase_family_id")),
|
|
)
|
|
if actual != expected:
|
|
raise CalibrationTransferConflictError(
|
|
"prediction history target cannot change"
|
|
)
|
|
try:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.calibration_prediction_revision (
|
|
prediction_revision_id, submission_id, content_hash,
|
|
history_id, session_id, learner_id, revision_no,
|
|
supersedes_prediction_revision_id,
|
|
predicted_success_probability, confidence, recorded_sequence,
|
|
revision_reason, instrument_id, instrument_version,
|
|
evidence_turn_ids, created_by_role
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15::uuid[],'learner'
|
|
)
|
|
RETURNING prediction_revision_id, revision_no
|
|
""",
|
|
prediction_revision_id,
|
|
submission_id,
|
|
content_hash,
|
|
history_id,
|
|
session_id,
|
|
learner_id,
|
|
revision_no,
|
|
supersedes_prediction_revision_id,
|
|
predicted_success_probability,
|
|
confidence,
|
|
recorded_sequence,
|
|
reason,
|
|
instrument_id,
|
|
instrument_version,
|
|
list(evidence),
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"prediction revision submission or chain conflict"
|
|
) from exc
|
|
except (
|
|
asyncpg.CheckViolationError,
|
|
asyncpg.ForeignKeyViolationError,
|
|
asyncpg.ObjectNotInPrerequisiteStateError,
|
|
) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"prediction revision violated provenance or history invariants"
|
|
) from exc
|
|
assert row is not None
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"prediction_revision_id": UUID(
|
|
str(_value(row, "prediction_revision_id"))
|
|
),
|
|
"revision_no": int(_value(row, "revision_no")),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def append_prediction_lock(
|
|
*,
|
|
principal: Principal,
|
|
submission_id: UUID,
|
|
lock_id: UUID,
|
|
history_id: UUID,
|
|
prediction_revision_id: UUID,
|
|
locked_sequence: int,
|
|
) -> dict[str, Any]:
|
|
if principal.role != Role.LEARNER:
|
|
raise CalibrationTransferStateError("prediction lock requires learner role")
|
|
learner_id = UUID(principal.user_id)
|
|
payload = {
|
|
"lock_id": str(lock_id),
|
|
"history_id": str(history_id),
|
|
"prediction_revision_id": str(prediction_revision_id),
|
|
"locked_sequence": locked_sequence,
|
|
"learner_id": str(learner_id),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
async with db.acquire(
|
|
role=principal.role.value,
|
|
user_id=principal.user_id,
|
|
cohort_ids=principal.cohort_ids,
|
|
) as conn:
|
|
history = await conn.fetchrow(
|
|
"""
|
|
SELECT history_id, session_id, learner_id
|
|
FROM app.calibration_prediction_history
|
|
WHERE history_id = $1
|
|
""",
|
|
history_id,
|
|
)
|
|
if history is None or UUID(str(_value(history, "learner_id"))) != learner_id:
|
|
raise CalibrationTransferNotFoundError(
|
|
"prediction history not found or not visible"
|
|
)
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_prediction_lock",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"lock_id": existing,
|
|
"idempotent_replay": True,
|
|
}
|
|
try:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.calibration_prediction_lock (
|
|
lock_id, submission_id, content_hash, history_id,
|
|
prediction_revision_id, session_id, learner_id,
|
|
locked_sequence, created_by_role
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'learner')
|
|
RETURNING lock_id
|
|
""",
|
|
lock_id,
|
|
submission_id,
|
|
content_hash,
|
|
history_id,
|
|
prediction_revision_id,
|
|
_value(history, "session_id"),
|
|
learner_id,
|
|
locked_sequence,
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"prediction lock submission or history conflict"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"prediction lock must target the latest pre-reveal revision"
|
|
) from exc
|
|
assert row is not None
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"lock_id": UUID(str(_value(row, "lock_id"))),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def append_performance_observation(
|
|
*,
|
|
conn: asyncpg.Connection,
|
|
submission_id: UUID,
|
|
observation_id: UUID,
|
|
history_id: UUID,
|
|
status: Literal["passed", "failed", "insufficient_evidence"],
|
|
source_kind: Literal["model_inferred", "observed_runtime"],
|
|
perspective: Literal["independent_observer", "runtime_observation"],
|
|
model_run_id: UUID | None,
|
|
instrument_id: str,
|
|
instrument_version: str,
|
|
uncertainty: float,
|
|
evidence_turn_ids: Sequence[UUID],
|
|
counterevidence: Sequence[str],
|
|
revealed_sequence: int,
|
|
) -> dict[str, Any]:
|
|
evidence = _ensure_unique_evidence(
|
|
evidence_turn_ids, required=status != "insufficient_evidence"
|
|
)
|
|
if status == "insufficient_evidence" and (evidence or uncertainty != 1.0):
|
|
raise CalibrationTransferStateError(
|
|
"insufficient observation must be evidence-free with full uncertainty"
|
|
)
|
|
if status == "failed" and not counterevidence:
|
|
raise CalibrationTransferStateError(
|
|
"failed observation requires counterevidence"
|
|
)
|
|
if (source_kind, perspective) not in {
|
|
("model_inferred", "independent_observer"),
|
|
("observed_runtime", "runtime_observation"),
|
|
}:
|
|
raise CalibrationTransferStateError(
|
|
"performance observation source and perspective are incompatible"
|
|
)
|
|
if source_kind == "model_inferred" and model_run_id is None:
|
|
raise CalibrationTransferStateError(
|
|
"model-inferred observation requires model_run_id"
|
|
)
|
|
history = await conn.fetchrow(
|
|
"""
|
|
SELECT h.*, l.lock_id
|
|
FROM app.calibration_prediction_history h
|
|
JOIN app.calibration_prediction_lock l ON l.history_id = h.history_id
|
|
WHERE h.history_id = $1
|
|
""",
|
|
history_id,
|
|
)
|
|
if history is None:
|
|
raise CalibrationTransferNotFoundError(
|
|
"locked prediction history not found or not visible"
|
|
)
|
|
payload = {
|
|
"observation_id": str(observation_id),
|
|
"history_id": str(history_id),
|
|
"status": status,
|
|
"source_kind": source_kind,
|
|
"perspective": perspective,
|
|
"model_run_id": str(model_run_id) if model_run_id else None,
|
|
"instrument_id": instrument_id,
|
|
"instrument_version": instrument_version,
|
|
"uncertainty": uncertainty,
|
|
"evidence_turn_ids": sorted(str(item) for item in evidence),
|
|
"counterevidence": list(counterevidence),
|
|
"revealed_sequence": revealed_sequence,
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_performance_observation",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"observation_id": existing,
|
|
"idempotent_replay": True,
|
|
}
|
|
try:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.calibration_performance_observation (
|
|
observation_id, submission_id, content_hash, history_id,
|
|
prediction_lock_id, session_id, learner_id, competency_id,
|
|
practice_block_id, scenario_variant_id, phrase_family_id,
|
|
status, source_kind, perspective, model_run_id,
|
|
instrument_id, instrument_version, uncertainty,
|
|
evidence_turn_ids, counterevidence, revealed_sequence
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,
|
|
$19::uuid[],$20::text[],$21
|
|
)
|
|
RETURNING observation_id
|
|
""",
|
|
observation_id,
|
|
submission_id,
|
|
content_hash,
|
|
history_id,
|
|
_value(history, "lock_id"),
|
|
_value(history, "session_id"),
|
|
_value(history, "learner_id"),
|
|
_value(history, "competency_id"),
|
|
_value(history, "practice_block_id"),
|
|
_value(history, "scenario_variant_id"),
|
|
_value(history, "phrase_family_id"),
|
|
status,
|
|
source_kind,
|
|
perspective,
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
uncertainty,
|
|
list(evidence),
|
|
list(counterevidence),
|
|
revealed_sequence,
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"performance observation submission or reveal conflict"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"performance observation violated lock, reveal, or provenance invariants"
|
|
) from exc
|
|
assert row is not None
|
|
return {
|
|
"submission_id": submission_id,
|
|
"history_id": history_id,
|
|
"observation_id": UUID(str(_value(row, "observation_id"))),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def append_calibration_assessment(
|
|
*,
|
|
conn: asyncpg.Connection,
|
|
submission_id: UUID,
|
|
assessment_snapshot_id: UUID,
|
|
prescription_id: UUID,
|
|
session_id: UUID,
|
|
assessment: CompetencyCalibrationAssessment,
|
|
prescription: MetacognitivePrescription,
|
|
source_observation_ids: Sequence[UUID],
|
|
model_run_id: UUID,
|
|
instrument_id: str,
|
|
instrument_version: str,
|
|
evidence_turn_ids: Sequence[UUID],
|
|
) -> dict[str, Any]:
|
|
if assessment.competency_id != prescription.competency_id:
|
|
raise CalibrationTransferStateError(
|
|
"assessment and metacognitive prescription competency must match"
|
|
)
|
|
evidence = _ensure_unique_evidence(evidence_turn_ids)
|
|
observation_ids = tuple(source_observation_ids)
|
|
if not observation_ids or len(set(observation_ids)) != len(observation_ids):
|
|
raise CalibrationTransferStateError(
|
|
"source_observation_ids must be non-empty and unique"
|
|
)
|
|
session = await _visible_session(conn, session_id)
|
|
learner_id = UUID(str(_value(session, "learner_id")))
|
|
payload = {
|
|
"assessment_snapshot_id": str(assessment_snapshot_id),
|
|
"prescription_id": str(prescription_id),
|
|
"session_id": str(session_id),
|
|
"learner_id": str(learner_id),
|
|
"assessment": assessment.model_dump(mode="json"),
|
|
"prescription": prescription.model_dump(mode="json"),
|
|
"source_observation_ids": sorted(str(item) for item in observation_ids),
|
|
"model_run_id": str(model_run_id),
|
|
"instrument_id": instrument_id,
|
|
"instrument_version": instrument_version,
|
|
"evidence_turn_ids": sorted(str(item) for item in evidence),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_assessment_snapshot",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
child = await conn.fetchrow(
|
|
"""
|
|
SELECT prescription_id
|
|
FROM app.calibration_metacognitive_prescription
|
|
WHERE assessment_snapshot_id = $1
|
|
""",
|
|
existing,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"assessment_snapshot_id": existing,
|
|
"prescription_id": UUID(str(_value(child or {}, "prescription_id"))),
|
|
"idempotent_replay": True,
|
|
}
|
|
latest = await conn.fetchrow(
|
|
"""
|
|
SELECT assessment_snapshot_id, snapshot_no
|
|
FROM app.calibration_assessment_snapshot
|
|
WHERE learner_id = $1 AND competency_id = $2
|
|
ORDER BY snapshot_no DESC LIMIT 1
|
|
""",
|
|
learner_id,
|
|
assessment.competency_id,
|
|
)
|
|
snapshot_no = int(_value(latest or {}, "snapshot_no", 0)) + 1
|
|
supersedes = _value(latest or {}, "assessment_snapshot_id")
|
|
try:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_assessment_snapshot (
|
|
assessment_snapshot_id, submission_id, content_hash,
|
|
session_id, learner_id, competency_id, snapshot_no,
|
|
supersedes_assessment_snapshot_id, source_observation_ids,
|
|
assessment_payload, model_run_id, instrument_id,
|
|
instrument_version, evidence_turn_ids
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9::uuid[],$10::jsonb,$11,$12,$13,$14::uuid[]
|
|
)
|
|
""",
|
|
assessment_snapshot_id,
|
|
submission_id,
|
|
content_hash,
|
|
session_id,
|
|
learner_id,
|
|
assessment.competency_id,
|
|
snapshot_no,
|
|
supersedes,
|
|
list(observation_ids),
|
|
assessment.model_dump(mode="json"),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
list(evidence),
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_metacognitive_prescription (
|
|
prescription_id, assessment_snapshot_id, session_id, learner_id,
|
|
competency_id, prescription_payload, model_run_id,
|
|
instrument_id, instrument_version, evidence_turn_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10::uuid[])
|
|
""",
|
|
prescription_id,
|
|
assessment_snapshot_id,
|
|
session_id,
|
|
learner_id,
|
|
assessment.competency_id,
|
|
prescription.model_dump(mode="json"),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
list(evidence),
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"calibration assessment submission or snapshot conflict"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"calibration assessment violated source or provenance invariants"
|
|
) from exc
|
|
return {
|
|
"submission_id": submission_id,
|
|
"assessment_snapshot_id": assessment_snapshot_id,
|
|
"prescription_id": prescription_id,
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def append_transfer_suite(
|
|
*,
|
|
conn: asyncpg.Connection,
|
|
submission_id: UUID,
|
|
transfer_suite_record_id: UUID,
|
|
session_id: UUID,
|
|
suite: TransferSuiteInput,
|
|
model_run_id: UUID,
|
|
instrument_id: str,
|
|
instrument_version: str,
|
|
) -> dict[str, Any]:
|
|
session = await _visible_session(conn, session_id)
|
|
learner_id = UUID(str(_value(session, "learner_id")))
|
|
payload = {
|
|
"transfer_suite_record_id": str(transfer_suite_record_id),
|
|
"session_id": str(session_id),
|
|
"learner_id": str(learner_id),
|
|
"suite": suite.model_dump(mode="json"),
|
|
"model_run_id": str(model_run_id),
|
|
"instrument_id": instrument_id,
|
|
"instrument_version": instrument_version,
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_transfer_suite",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
counts = await conn.fetchrow(
|
|
"""
|
|
SELECT
|
|
(SELECT count(*) FROM app.calibration_transfer_trial WHERE transfer_suite_record_id = $1) AS trial_count,
|
|
(SELECT count(*) FROM app.calibration_transfer_assessment WHERE transfer_suite_record_id = $1) AS assessment_count,
|
|
(SELECT count(*) FROM app.calibration_subgroup_drift_report WHERE transfer_suite_record_id = $1) AS drift_report_count
|
|
""",
|
|
existing,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"transfer_suite_record_id": existing,
|
|
"trial_count": int(_value(counts or {}, "trial_count", 0)),
|
|
"assessment_count": int(_value(counts or {}, "assessment_count", 0)),
|
|
"drift_report_count": int(
|
|
_value(counts or {}, "drift_report_count", 0)
|
|
),
|
|
"idempotent_replay": True,
|
|
}
|
|
assessments = assess_transfer(suite)
|
|
drift_reports = assess_synthetic_subgroup_drift(suite)
|
|
try:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_transfer_suite (
|
|
transfer_suite_record_id, submission_id, content_hash, suite_key,
|
|
session_id, learner_id, training_phrase_family_ids,
|
|
model_run_id, instrument_id, instrument_version
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7::text[],$8,$9,$10)
|
|
""",
|
|
transfer_suite_record_id,
|
|
submission_id,
|
|
content_hash,
|
|
suite.suite_id,
|
|
session_id,
|
|
learner_id,
|
|
list(suite.training_phrase_family_ids),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
)
|
|
trial_records: dict[str, tuple[UUID, tuple[UUID, ...]]] = {}
|
|
for trial in suite.trials:
|
|
trial_record_id = uuid4()
|
|
evidence = _uuid_evidence(
|
|
trial.evidence_refs,
|
|
required=trial.status != "insufficient_evidence",
|
|
)
|
|
trial_records[trial.trial_id] = (trial_record_id, evidence)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_transfer_trial (
|
|
transfer_trial_record_id, transfer_suite_record_id,
|
|
session_id, learner_id, trial_key, competency_id,
|
|
scenario_variant_id, context_variant, relationship_style,
|
|
difficulty_level, expression_variant, synthetic_subgroup,
|
|
scenario_family_id, phrase_family_id, status, uncertainty,
|
|
evidence_turn_ids, counterevidence, model_run_id,
|
|
instrument_id, instrument_version
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,
|
|
$17::uuid[],$18::text[],$19,$20,$21
|
|
)
|
|
""",
|
|
trial_record_id,
|
|
transfer_suite_record_id,
|
|
session_id,
|
|
learner_id,
|
|
trial.trial_id,
|
|
trial.competency_id,
|
|
trial.scenario_variant_id,
|
|
trial.variation.context_variant,
|
|
trial.variation.relationship_style,
|
|
trial.variation.difficulty_level,
|
|
trial.variation.expression_variant,
|
|
trial.variation.synthetic_subgroup,
|
|
trial.variation.scenario_family_id,
|
|
trial.variation.phrase_family_id,
|
|
trial.status,
|
|
trial.uncertainty,
|
|
list(evidence),
|
|
list(trial.counterevidence),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
)
|
|
for assessment in assessments:
|
|
source = [
|
|
trial_records[trial.trial_id]
|
|
for trial in suite.trials
|
|
if trial.competency_id == assessment.competency_id
|
|
]
|
|
source_ids = [item[0] for item in source]
|
|
evidence = tuple(dict.fromkeys(ref for item in source for ref in item[1]))
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_transfer_assessment (
|
|
transfer_assessment_id, transfer_suite_record_id,
|
|
session_id, learner_id, competency_id, source_trial_ids,
|
|
assessment_payload, evidence_turn_ids, model_run_id,
|
|
instrument_id, instrument_version
|
|
) VALUES ($1,$2,$3,$4,$5,$6::uuid[],$7::jsonb,$8::uuid[],$9,$10,$11)
|
|
""",
|
|
uuid4(),
|
|
transfer_suite_record_id,
|
|
session_id,
|
|
learner_id,
|
|
assessment.competency_id,
|
|
source_ids,
|
|
assessment.model_dump(mode="json"),
|
|
list(evidence),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
)
|
|
for report in drift_reports:
|
|
source = [
|
|
trial_records[trial.trial_id]
|
|
for trial in suite.trials
|
|
if trial.competency_id == report.competency_id
|
|
]
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.calibration_subgroup_drift_report (
|
|
drift_report_id, transfer_suite_record_id, session_id,
|
|
learner_id, competency_id, source_trial_ids, report_payload,
|
|
model_run_id, instrument_id, instrument_version
|
|
) VALUES ($1,$2,$3,$4,$5,$6::uuid[],$7::jsonb,$8,$9,$10)
|
|
""",
|
|
uuid4(),
|
|
transfer_suite_record_id,
|
|
session_id,
|
|
learner_id,
|
|
report.competency_id,
|
|
[item[0] for item in source],
|
|
report.model_dump(mode="json"),
|
|
model_run_id,
|
|
instrument_id,
|
|
instrument_version,
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"transfer suite submission or child conflict"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"transfer suite violated unseen, evidence, or provenance invariants"
|
|
) from exc
|
|
return {
|
|
"submission_id": submission_id,
|
|
"transfer_suite_record_id": transfer_suite_record_id,
|
|
"trial_count": len(suite.trials),
|
|
"assessment_count": len(assessments),
|
|
"drift_report_count": len(drift_reports),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
def _actual_target_techniques(competency_id: str) -> frozenset[str]:
|
|
key = competency_id.lower()
|
|
if any(token in key for token in ("empathy", "empathic", "reflection")):
|
|
return frozenset({"empathy", "reflection", "validation", "restatement"})
|
|
if any(token in key for token in ("open_question", "open-question")):
|
|
return frozenset({"facilitative_question", "exploration", "clarification"})
|
|
if any(token in key for token in ("rupture", "repair", "impact")):
|
|
return frozenset(
|
|
{"opinion_check", "validation", "reflection", "here_and_now_focus"}
|
|
)
|
|
if any(token in key for token in ("goal", "collaborative", "reagreement")):
|
|
return frozenset(
|
|
{"consent_motivation_check", "opinion_check", "restatement"}
|
|
)
|
|
if any(token in key for token in ("presence", "response-space")):
|
|
return frozenset({"holding", "reflection", "here_and_now_focus"})
|
|
raise CalibrationTransferStateError(
|
|
f"unsupported actual transfer competency: {competency_id}"
|
|
)
|
|
|
|
|
|
def _actual_execution_from_row(row: Mapping[str, Any]) -> ActualTransferExecution:
|
|
labels_payload = _value(row, "normalized_evaluator_labels", {}) or {}
|
|
if isinstance(labels_payload, str):
|
|
labels_payload = json.loads(labels_payload)
|
|
return ActualTransferExecution(
|
|
execution_event_id=UUID(str(_value(row, "execution_event_id"))),
|
|
original_transfer_trial_record_id=UUID(
|
|
str(_value(row, "original_transfer_trial_record_id"))
|
|
),
|
|
practice_session_id=UUID(str(_value(row, "practice_session_id"))),
|
|
competency_id=str(_value(row, "competency_id")),
|
|
scenario_variant_id=str(_value(row, "scenario_variant_id")),
|
|
scenario_novelty=str(_value(row, "scenario_novelty")),
|
|
variation=TransferVariation(
|
|
context_variant=str(_value(row, "context_variant")),
|
|
relationship_style=str(_value(row, "relationship_style")),
|
|
difficulty_level=int(_value(row, "difficulty_level")),
|
|
expression_variant=str(_value(row, "expression_variant")),
|
|
synthetic_subgroup=str(_value(row, "synthetic_subgroup")),
|
|
scenario_family_id=str(_value(row, "scenario_family_id")),
|
|
phrase_family_id=str(_value(row, "phrase_family_id")),
|
|
),
|
|
status=str(_value(row, "status")),
|
|
uncertainty=float(_value(row, "uncertainty")),
|
|
evidence_turn_ids=tuple(_value(row, "evidence_turn_ids", ()) or ()),
|
|
normalized_evaluator_labels=NormalizedEvaluatorLabels.model_validate(
|
|
labels_payload
|
|
),
|
|
counterevidence=tuple(_value(row, "counterevidence", ()) or ()),
|
|
model_run_id=UUID(str(_value(row, "model_run_id"))),
|
|
source_kind=str(_value(row, "source_kind")),
|
|
perspective=str(_value(row, "perspective")),
|
|
instrument_id=str(_value(row, "instrument_id")),
|
|
instrument_version=str(_value(row, "instrument_version")),
|
|
observer_version=str(_value(row, "observer_version")),
|
|
training_phrase_collision=bool(
|
|
_value(row, "training_phrase_collision", False)
|
|
),
|
|
created_at=_value(row, "created_at") or datetime.now(UTC),
|
|
)
|
|
|
|
|
|
def _derive_actual_execution_labels(
|
|
rows: Sequence[Mapping[str, Any]], *, competency_id: str
|
|
) -> tuple[
|
|
Literal["passed", "failed", "insufficient_evidence"],
|
|
float,
|
|
tuple[UUID, ...],
|
|
NormalizedEvaluatorLabels,
|
|
tuple[str, ...],
|
|
]:
|
|
targets = _actual_target_techniques(competency_id)
|
|
technique_codes = tuple(
|
|
sorted(
|
|
{
|
|
str(code)
|
|
for row in rows
|
|
for code in (_value(row, "technique_codes", ()) or ())
|
|
}
|
|
)
|
|
)
|
|
client_state_codes = tuple(
|
|
sorted(
|
|
{
|
|
str(code)
|
|
for row in rows
|
|
for code in (_value(row, "client_state_codes", ()) or ())
|
|
}
|
|
)
|
|
)
|
|
appropriateness = tuple(
|
|
dict.fromkeys(
|
|
str(_value(row, "appropriateness", "neutral") or "neutral")
|
|
for row in rows
|
|
)
|
|
)
|
|
deviation_dimensions = tuple(
|
|
sorted(
|
|
{
|
|
str(code).lower()
|
|
for row in rows
|
|
for code in (
|
|
_value(row, "intent_deviation_dimensions", ()) or ()
|
|
)
|
|
if code
|
|
}
|
|
)
|
|
)
|
|
error_count = sum(bool(_value(row, "evaluator_error")) for row in rows)
|
|
labels = NormalizedEvaluatorLabels(
|
|
technique_codes=technique_codes,
|
|
client_state_codes=client_state_codes,
|
|
appropriateness=appropriateness,
|
|
intent_deviation_dimensions=deviation_dimensions,
|
|
evaluator_error_count=error_count,
|
|
)
|
|
complete = [item for item in rows if _value(item, "client_turn_id")]
|
|
if not complete:
|
|
return "insufficient_evidence", 1.0, (), labels, ()
|
|
|
|
competency_tokens = {
|
|
token
|
|
for token in competency_id.lower().replace("competency.", "").replace("-", "_").split("_")
|
|
if len(token) >= 4
|
|
}
|
|
counterevidence: list[str] = []
|
|
passed = False
|
|
evidence: list[UUID] = []
|
|
for row in complete:
|
|
counselor_id = UUID(str(_value(row, "counselor_turn_id")))
|
|
client_id = UUID(str(_value(row, "client_turn_id")))
|
|
evidence.extend((counselor_id, client_id))
|
|
techniques = set(_value(row, "technique_codes", ()) or ())
|
|
states = set(_value(row, "client_state_codes", ()) or ())
|
|
dimensions = {
|
|
str(item).lower()
|
|
for item in (_value(row, "intent_deviation_dimensions", ()) or ())
|
|
if item
|
|
}
|
|
target_deviation = any(
|
|
token in dimension or dimension in token
|
|
for token in competency_tokens
|
|
for dimension in dimensions
|
|
)
|
|
technique_match = bool(techniques & targets)
|
|
client_support = bool(states & _POSITIVE_CLIENT_STATES)
|
|
row_passed = (
|
|
technique_match
|
|
and _value(row, "appropriateness", "neutral") == "pos"
|
|
and client_support
|
|
and not target_deviation
|
|
and not bool(_value(row, "evaluator_error"))
|
|
)
|
|
passed = passed or row_passed
|
|
if not technique_match:
|
|
counterevidence.append("target_technique_not_observed")
|
|
if _value(row, "appropriateness", "neutral") != "pos":
|
|
counterevidence.append("appropriateness_not_positive")
|
|
if not client_support:
|
|
counterevidence.append("client_response_does_not_support_effect")
|
|
if target_deviation:
|
|
counterevidence.append("target_intent_deviation_observed")
|
|
if _value(row, "evaluator_error"):
|
|
counterevidence.append("turn_evaluation_error")
|
|
unique_evidence = tuple(dict.fromkeys(evidence))
|
|
if passed:
|
|
return "passed", 0.25, unique_evidence, labels, ()
|
|
return (
|
|
"failed",
|
|
0.4,
|
|
unique_evidence,
|
|
labels,
|
|
tuple(dict.fromkeys(counterevidence)) or ("target_behavior_not_observed",),
|
|
)
|
|
|
|
|
|
async def _actual_events_for_competency(
|
|
conn: asyncpg.Connection, *, learner_id: UUID, competency_id: str
|
|
) -> tuple[ActualTransferExecution, ...]:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT *
|
|
FROM app.calibration_transfer_execution_event
|
|
WHERE learner_id = $1 AND competency_id = $2
|
|
ORDER BY created_at, execution_event_id
|
|
""",
|
|
learner_id,
|
|
competency_id,
|
|
)
|
|
return tuple(_actual_execution_from_row(row) for row in rows)
|
|
|
|
|
|
async def append_actual_transfer_execution(
|
|
*,
|
|
principal: Principal,
|
|
original_transfer_trial_record_id: UUID,
|
|
practice_session_id: UUID,
|
|
) -> dict[str, Any]:
|
|
"""완료된 후속 회기의 정규화 라벨만으로 실제 transfer 실행을 기록한다."""
|
|
|
|
if principal.role != Role.LEARNER:
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer execution requires learner role"
|
|
)
|
|
learner_id = UUID(principal.user_id)
|
|
event_id = uuid5(
|
|
_ACTUAL_TRANSFER_NAMESPACE,
|
|
f"event:{original_transfer_trial_record_id}:{practice_session_id}",
|
|
)
|
|
model_run_id = uuid5(
|
|
_ACTUAL_TRANSFER_NAMESPACE,
|
|
f"model:{original_transfer_trial_record_id}:{practice_session_id}",
|
|
)
|
|
async with db.acquire(
|
|
role=principal.role.value,
|
|
user_id=principal.user_id,
|
|
cohort_ids=principal.cohort_ids,
|
|
ai_context=True,
|
|
ai_view="evaluator",
|
|
) as conn:
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 5))",
|
|
f"{original_transfer_trial_record_id}:{practice_session_id}",
|
|
)
|
|
original = await conn.fetchrow(
|
|
"""
|
|
SELECT trial.transfer_trial_record_id,
|
|
trial.transfer_suite_record_id, trial.session_id,
|
|
trial.learner_id, trial.competency_id,
|
|
trial.scenario_variant_id, trial.scenario_novelty,
|
|
trial.context_variant, trial.relationship_style,
|
|
trial.difficulty_level, trial.expression_variant,
|
|
trial.synthetic_subgroup, trial.scenario_family_id,
|
|
trial.phrase_family_id, trial.created_at,
|
|
suite.training_phrase_family_ids,
|
|
EXISTS (
|
|
SELECT 1
|
|
FROM app.calibration_prediction_history history
|
|
JOIN app.calibration_prediction_lock prediction_lock
|
|
ON prediction_lock.history_id = history.history_id
|
|
WHERE history.learner_id = trial.learner_id
|
|
AND history.competency_id = trial.competency_id
|
|
AND prediction_lock.created_at <= trial.created_at
|
|
) AS prediction_locked
|
|
FROM app.calibration_transfer_trial trial
|
|
JOIN app.calibration_transfer_suite suite
|
|
ON suite.transfer_suite_record_id = trial.transfer_suite_record_id
|
|
WHERE trial.transfer_trial_record_id = $1
|
|
AND trial.learner_id = $2
|
|
""",
|
|
original_transfer_trial_record_id,
|
|
learner_id,
|
|
)
|
|
if original is None:
|
|
raise CalibrationTransferNotFoundError(
|
|
"original transfer trial not found or not visible"
|
|
)
|
|
if not bool(_value(original, "prediction_locked")):
|
|
raise CalibrationTransferStateError(
|
|
"self-prediction must be locked before transfer reveal"
|
|
)
|
|
|
|
existing = await conn.fetchrow(
|
|
"""
|
|
SELECT * FROM app.calibration_transfer_execution_event
|
|
WHERE original_transfer_trial_record_id = $1
|
|
AND practice_session_id = $2 AND learner_id = $3
|
|
""",
|
|
original_transfer_trial_record_id,
|
|
practice_session_id,
|
|
learner_id,
|
|
)
|
|
if existing is not None:
|
|
execution = _actual_execution_from_row(existing)
|
|
all_events = await _actual_events_for_competency(
|
|
conn,
|
|
learner_id=learner_id,
|
|
competency_id=execution.competency_id,
|
|
)
|
|
assessment = assess_actual_transfer_executions(all_events)[0]
|
|
return {
|
|
"execution": execution.model_dump(mode="python"),
|
|
"assessment": assessment.model_dump(mode="python"),
|
|
"idempotent_replay": True,
|
|
}
|
|
|
|
session = await conn.fetchrow(
|
|
"""
|
|
SELECT session.id, session.learner_id, session.started_at,
|
|
session.ended_at, evaluation.status AS evaluation_status,
|
|
evaluation.scope AS evaluation_scope
|
|
FROM app.sessions session
|
|
LEFT JOIN app.session_evaluation evaluation
|
|
ON evaluation.session_id = session.id
|
|
WHERE session.id = $1 AND session.learner_id = $2
|
|
""",
|
|
practice_session_id,
|
|
learner_id,
|
|
)
|
|
if session is None:
|
|
raise CalibrationTransferNotFoundError(
|
|
"actual transfer practice session not found or not visible"
|
|
)
|
|
if practice_session_id == UUID(str(_value(original, "session_id"))):
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer evidence requires a later practice session"
|
|
)
|
|
if _value(session, "ended_at") is None:
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer practice session must be ended"
|
|
)
|
|
if (
|
|
_value(session, "evaluation_status") != "ready"
|
|
or _value(session, "evaluation_scope") != "session_end"
|
|
):
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer practice evaluation must be ready at session_end"
|
|
)
|
|
if _value(session, "started_at") <= _value(original, "created_at"):
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer practice must start after the original trial"
|
|
)
|
|
|
|
rows = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT
|
|
counselor.id AS counselor_turn_id,
|
|
counselor.seq AS counselor_turn_seq,
|
|
response.id AS client_turn_id,
|
|
response.seq AS client_turn_seq,
|
|
ARRAY(
|
|
SELECT definition.code
|
|
FROM app.turn_technique tagged
|
|
JOIN app.technique_label_def definition
|
|
ON definition.label_id = tagged.label_id
|
|
WHERE tagged.turn_id = counselor.id
|
|
ORDER BY definition.code
|
|
) AS technique_codes,
|
|
ARRAY(
|
|
SELECT definition.code
|
|
FROM app.turn_client_state tagged
|
|
JOIN app.client_state_def definition
|
|
ON definition.label_id = tagged.label_id
|
|
WHERE tagged.turn_id = response.id
|
|
ORDER BY definition.code
|
|
) AS client_state_codes,
|
|
CASE
|
|
WHEN appropriateness.score >= 4 THEN 'pos'
|
|
WHEN appropriateness.score <= 2 THEN 'warn'
|
|
ELSE 'neutral'
|
|
END AS appropriateness,
|
|
ARRAY(
|
|
SELECT lower(comment.intent_deviation->>'dimension')
|
|
FROM app.supervisor_comment comment
|
|
WHERE comment.turn_id = counselor.id
|
|
AND comment.intent_deviation IS NOT NULL
|
|
ORDER BY comment.created_at, comment.id
|
|
) AS intent_deviation_dimensions,
|
|
(evaluator_error.rationale IS NOT NULL) AS evaluator_error
|
|
FROM app.turns counselor
|
|
LEFT JOIN LATERAL (
|
|
SELECT candidate.id, candidate.seq
|
|
FROM app.turns candidate
|
|
WHERE candidate.session_id = counselor.session_id
|
|
AND candidate.speaker = 'client'
|
|
AND candidate.seq > counselor.seq
|
|
ORDER BY candidate.seq LIMIT 1
|
|
) response ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT score
|
|
FROM app.feedback_scores score
|
|
WHERE score.turn_id = counselor.id
|
|
AND score.dimension = 'appropriateness'
|
|
ORDER BY score.created_at DESC LIMIT 1
|
|
) appropriateness ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT rationale
|
|
FROM app.feedback_scores score
|
|
WHERE score.turn_id = counselor.id AND score.dimension = 'error'
|
|
ORDER BY score.created_at DESC LIMIT 1
|
|
) evaluator_error ON TRUE
|
|
WHERE counselor.session_id = $1 AND counselor.speaker = 'counselor'
|
|
ORDER BY counselor.seq
|
|
""",
|
|
practice_session_id,
|
|
)
|
|
)
|
|
competency_id = str(_value(original, "competency_id"))
|
|
status_value, uncertainty, evidence, labels, counterevidence = (
|
|
_derive_actual_execution_labels(rows, competency_id=competency_id)
|
|
)
|
|
label_payload = labels.model_dump(mode="json")
|
|
evidence_payload = {
|
|
"observer_version": _ACTUAL_TRANSFER_OBSERVER_VERSION,
|
|
"original_transfer_trial_record_id": str(
|
|
original_transfer_trial_record_id
|
|
),
|
|
"practice_session_id": str(practice_session_id),
|
|
"competency_id": competency_id,
|
|
"evidence_turn_ids": [str(item) for item in evidence],
|
|
"normalized_evaluator_labels": label_payload,
|
|
}
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO audit.model_run (
|
|
model_run_id, session_id, agent_role, provider, model,
|
|
prompt_bundle_id, prompt_bundle_version, prompt_bundle_hash,
|
|
structured_schema_version, input_evidence_hash, status, metadata
|
|
) VALUES (
|
|
$1,$2,'evaluator','vignette-runtime','calibration-actual-transfer-observer',
|
|
'calibration-actual-transfer-observer',$3,$4,
|
|
'vignette.calibration-actual-transfer-execution.v1',$5,'ready',$6::jsonb
|
|
) ON CONFLICT (model_run_id) DO NOTHING
|
|
""",
|
|
model_run_id,
|
|
practice_session_id,
|
|
_ACTUAL_TRANSFER_OBSERVER_VERSION,
|
|
_canonical_hash(
|
|
{"observer_version": _ACTUAL_TRANSFER_OBSERVER_VERSION}
|
|
),
|
|
_canonical_hash(evidence_payload),
|
|
evidence_payload,
|
|
)
|
|
training_phrases = set(
|
|
_value(original, "training_phrase_family_ids", ()) or ()
|
|
)
|
|
try:
|
|
inserted = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.calibration_transfer_execution_event (
|
|
execution_event_id, original_transfer_trial_record_id,
|
|
transfer_suite_record_id, practice_session_id, learner_id,
|
|
competency_id, scenario_variant_id, context_variant,
|
|
relationship_style, difficulty_level, expression_variant,
|
|
synthetic_subgroup, scenario_family_id, phrase_family_id,
|
|
training_phrase_collision, status, uncertainty,
|
|
evidence_turn_ids, normalized_evaluator_labels,
|
|
counterevidence, model_run_id, source_kind, perspective,
|
|
instrument_id, instrument_version, observer_version
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,
|
|
$18::uuid[],$19::jsonb,$20::text[],$21,$22,$23,$24,$25,$26
|
|
) RETURNING *
|
|
""",
|
|
event_id,
|
|
original_transfer_trial_record_id,
|
|
_value(original, "transfer_suite_record_id"),
|
|
practice_session_id,
|
|
learner_id,
|
|
competency_id,
|
|
_value(original, "scenario_variant_id"),
|
|
_value(original, "context_variant"),
|
|
_value(original, "relationship_style"),
|
|
_value(original, "difficulty_level"),
|
|
_value(original, "expression_variant"),
|
|
_value(original, "synthetic_subgroup"),
|
|
_value(original, "scenario_family_id"),
|
|
_value(original, "phrase_family_id"),
|
|
_value(original, "phrase_family_id") in training_phrases,
|
|
status_value,
|
|
uncertainty,
|
|
list(evidence),
|
|
label_payload,
|
|
list(counterevidence),
|
|
model_run_id,
|
|
"model_inferred",
|
|
"independent_observer",
|
|
_ACTUAL_TRANSFER_INSTRUMENT_ID,
|
|
_ACTUAL_TRANSFER_INSTRUMENT_VERSION,
|
|
_ACTUAL_TRANSFER_OBSERVER_VERSION,
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"actual transfer practice session was already recorded"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"actual transfer execution violated source or evidence invariants"
|
|
) from exc
|
|
assert inserted is not None
|
|
execution = _actual_execution_from_row(inserted)
|
|
all_events = await _actual_events_for_competency(
|
|
conn, learner_id=learner_id, competency_id=competency_id
|
|
)
|
|
assessment = assess_actual_transfer_executions(all_events)[0]
|
|
return {
|
|
"execution": execution.model_dump(mode="python"),
|
|
"assessment": assessment.model_dump(mode="python"),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def append_teacher_review(
|
|
*,
|
|
principal: Principal,
|
|
submission_id: UUID,
|
|
review_id: UUID,
|
|
target_kind: Literal[
|
|
"calibration_assessment", "transfer_assessment", "drift_report"
|
|
],
|
|
target_id: UUID,
|
|
disposition: Literal["confirmed", "corrected", "needs_more_evidence"],
|
|
correction_payload: Mapping[str, Any],
|
|
review_reason: str,
|
|
evidence_turn_ids: Sequence[UUID],
|
|
counterevidence: Sequence[str],
|
|
) -> dict[str, Any]:
|
|
if principal.role not in {Role.TEACHER, Role.ADMIN}:
|
|
raise CalibrationTransferStateError(
|
|
"calibration review requires teacher or admin role"
|
|
)
|
|
reason = review_reason.strip()
|
|
if not reason:
|
|
raise CalibrationTransferStateError("review_reason must not be blank")
|
|
if disposition != "corrected" and correction_payload:
|
|
raise CalibrationTransferStateError(
|
|
"only corrected reviews may carry correction_payload"
|
|
)
|
|
evidence = _ensure_unique_evidence(evidence_turn_ids)
|
|
target_tables = {
|
|
"calibration_assessment": (
|
|
"app.calibration_assessment_snapshot",
|
|
"assessment_snapshot_id",
|
|
),
|
|
"transfer_assessment": (
|
|
"app.calibration_transfer_assessment",
|
|
"transfer_assessment_id",
|
|
),
|
|
"drift_report": (
|
|
"app.calibration_subgroup_drift_report",
|
|
"drift_report_id",
|
|
),
|
|
}
|
|
table, id_column = target_tables[target_kind]
|
|
payload = {
|
|
"review_id": str(review_id),
|
|
"target_kind": target_kind,
|
|
"target_id": str(target_id),
|
|
"disposition": disposition,
|
|
"correction_payload": dict(correction_payload),
|
|
"review_reason": reason,
|
|
"evidence_turn_ids": sorted(str(item) for item in evidence),
|
|
"counterevidence": list(counterevidence),
|
|
"created_by_uid": principal.user_id,
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
async with db.acquire(
|
|
role=principal.role.value,
|
|
user_id=principal.user_id,
|
|
cohort_ids=principal.cohort_ids,
|
|
) as conn:
|
|
target = await conn.fetchrow(
|
|
f"SELECT session_id, learner_id FROM {table} WHERE {id_column} = $1",
|
|
target_id,
|
|
)
|
|
if target is None:
|
|
raise CalibrationTransferNotFoundError(
|
|
"review target not found or outside cohort scope"
|
|
)
|
|
existing = await _existing_by_submission(
|
|
conn,
|
|
table="app.calibration_teacher_review_event",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
row = await conn.fetchrow(
|
|
"SELECT review_no FROM app.calibration_teacher_review_event WHERE review_id = $1",
|
|
existing,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"review_id": existing,
|
|
"review_no": int(_value(row or {}, "review_no", 1)),
|
|
"idempotent_replay": True,
|
|
}
|
|
latest = await conn.fetchrow(
|
|
"""
|
|
SELECT review_id, review_no
|
|
FROM app.calibration_teacher_review_event
|
|
WHERE target_kind = $1 AND target_id = $2
|
|
ORDER BY review_no DESC LIMIT 1
|
|
""",
|
|
target_kind,
|
|
target_id,
|
|
)
|
|
review_no = int(_value(latest or {}, "review_no", 0)) + 1
|
|
try:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO app.calibration_teacher_review_event (
|
|
review_id, submission_id, content_hash, target_kind,
|
|
target_id, session_id, learner_id, review_no,
|
|
supersedes_review_id, disposition, correction_payload,
|
|
review_reason, evidence_turn_ids, counterevidence,
|
|
created_by_uid, created_by_role
|
|
) VALUES (
|
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,
|
|
$13::uuid[],$14::text[],$15,$16
|
|
) RETURNING review_id, review_no
|
|
""",
|
|
review_id,
|
|
submission_id,
|
|
content_hash,
|
|
target_kind,
|
|
target_id,
|
|
_value(target, "session_id"),
|
|
_value(target, "learner_id"),
|
|
review_no,
|
|
_value(latest or {}, "review_id"),
|
|
disposition,
|
|
dict(correction_payload),
|
|
reason,
|
|
list(evidence),
|
|
list(counterevidence),
|
|
UUID(principal.user_id),
|
|
_created_role(principal),
|
|
)
|
|
except asyncpg.UniqueViolationError as exc:
|
|
raise CalibrationTransferConflictError(
|
|
"teacher review submission or supersession conflict"
|
|
) from exc
|
|
except (asyncpg.CheckViolationError, asyncpg.ForeignKeyViolationError) as exc:
|
|
raise CalibrationTransferStateError(
|
|
"teacher review violated target, evidence, or payload invariants"
|
|
) from exc
|
|
assert row is not None
|
|
return {
|
|
"submission_id": submission_id,
|
|
"review_id": UUID(str(_value(row, "review_id"))),
|
|
"review_no": int(_value(row, "review_no")),
|
|
"idempotent_replay": False,
|
|
}
|
|
|
|
|
|
async def read_calibration_transfer(
|
|
*, principal: Principal, learner_id: UUID | None = None
|
|
) -> dict[str, Any]:
|
|
if principal.role == Role.LEARNER:
|
|
target_learner_id = UUID(principal.user_id)
|
|
if learner_id is not None and learner_id != target_learner_id:
|
|
raise CalibrationTransferNotFoundError(
|
|
"learner calibration is not visible"
|
|
)
|
|
requested_view = "learner"
|
|
elif principal.role in {Role.TEACHER, Role.ADMIN}:
|
|
if learner_id is None:
|
|
raise CalibrationTransferStateError(
|
|
"teacher/admin calibration read requires learner_id"
|
|
)
|
|
target_learner_id = learner_id
|
|
requested_view = "supervisor"
|
|
else:
|
|
raise CalibrationTransferStateError("unsupported calibration reader role")
|
|
async with db.acquire(
|
|
role=principal.role.value,
|
|
user_id=principal.user_id,
|
|
cohort_ids=principal.cohort_ids,
|
|
) as conn:
|
|
if principal.role in {Role.TEACHER, Role.ADMIN}:
|
|
visible = await conn.fetchval(
|
|
"SELECT EXISTS(SELECT 1 FROM app.sessions WHERE learner_id = $1)",
|
|
target_learner_id,
|
|
)
|
|
if not visible:
|
|
raise CalibrationTransferNotFoundError(
|
|
"learner calibration not found or outside cohort scope"
|
|
)
|
|
histories = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT history.history_id, history.session_id,
|
|
history.competency_id, history.practice_block_id,
|
|
history.scenario_variant_id, history.phrase_family_id,
|
|
history.created_at,
|
|
source_session.learner_feedback_enabled
|
|
AS source_learner_feedback_enabled
|
|
FROM app.calibration_prediction_history history
|
|
JOIN app.sessions source_session
|
|
ON source_session.id = history.session_id
|
|
WHERE history.learner_id = $1
|
|
ORDER BY history.created_at, history.history_id
|
|
""",
|
|
target_learner_id,
|
|
)
|
|
)
|
|
history_ids = [UUID(str(_value(item, "history_id"))) for item in histories]
|
|
revisions = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT prediction_revision_id, submission_id, history_id,
|
|
revision_no, supersedes_prediction_revision_id,
|
|
predicted_success_probability, confidence,
|
|
recorded_sequence, revision_reason, source_kind,
|
|
perspective, instrument_id, instrument_version,
|
|
evidence_turn_ids, created_at
|
|
FROM app.calibration_prediction_revision
|
|
WHERE history_id = ANY($1::uuid[])
|
|
ORDER BY history_id, revision_no
|
|
""",
|
|
history_ids,
|
|
)
|
|
)
|
|
if history_ids
|
|
else []
|
|
)
|
|
locks = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT lock_id, submission_id, history_id,
|
|
prediction_revision_id, locked_sequence, created_at
|
|
FROM app.calibration_prediction_lock
|
|
WHERE history_id = ANY($1::uuid[])
|
|
""",
|
|
history_ids,
|
|
)
|
|
)
|
|
if history_ids
|
|
else []
|
|
)
|
|
observations = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT observation_id, submission_id, history_id, status,
|
|
source_kind, perspective, model_run_id,
|
|
instrument_id, instrument_version, uncertainty,
|
|
evidence_turn_ids, counterevidence,
|
|
revealed_sequence, created_at
|
|
FROM app.calibration_performance_observation
|
|
WHERE history_id = ANY($1::uuid[])
|
|
""",
|
|
history_ids,
|
|
)
|
|
)
|
|
if history_ids
|
|
else []
|
|
)
|
|
assessments = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT a.assessment_snapshot_id, a.submission_id, a.session_id,
|
|
a.competency_id, a.snapshot_no,
|
|
a.supersedes_assessment_snapshot_id,
|
|
a.source_observation_ids, a.assessment_payload,
|
|
a.model_run_id, a.instrument_id, a.instrument_version,
|
|
a.evidence_turn_ids, a.created_at,
|
|
p.prescription_id, p.prescription_payload,
|
|
source_session.learner_feedback_enabled
|
|
AS source_learner_feedback_enabled
|
|
FROM app.calibration_assessment_snapshot a
|
|
JOIN app.calibration_metacognitive_prescription p
|
|
ON p.assessment_snapshot_id = a.assessment_snapshot_id
|
|
JOIN app.sessions source_session ON source_session.id = a.session_id
|
|
WHERE a.learner_id = $1
|
|
ORDER BY a.competency_id, a.snapshot_no
|
|
""",
|
|
target_learner_id,
|
|
)
|
|
)
|
|
suites = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT suite.transfer_suite_record_id, suite.submission_id,
|
|
suite.suite_key, suite.session_id,
|
|
suite.training_phrase_family_ids, suite.model_run_id,
|
|
suite.instrument_id, suite.instrument_version,
|
|
suite.data_classification, suite.clinical_claim_allowed,
|
|
suite.created_at,
|
|
source_session.learner_feedback_enabled
|
|
AS source_learner_feedback_enabled
|
|
FROM app.calibration_transfer_suite suite
|
|
JOIN app.sessions source_session ON source_session.id = suite.session_id
|
|
WHERE suite.learner_id = $1
|
|
ORDER BY suite.created_at, suite.transfer_suite_record_id
|
|
""",
|
|
target_learner_id,
|
|
)
|
|
)
|
|
suite_ids = [
|
|
UUID(str(_value(item, "transfer_suite_record_id"))) for item in suites
|
|
]
|
|
trials = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT transfer_trial_record_id, transfer_suite_record_id,
|
|
trial_key, competency_id, scenario_variant_id,
|
|
scenario_novelty, context_variant, relationship_style,
|
|
difficulty_level, expression_variant, synthetic_subgroup,
|
|
scenario_family_id, phrase_family_id, status, uncertainty,
|
|
evidence_turn_ids, counterevidence, model_run_id,
|
|
instrument_id, instrument_version, created_at
|
|
FROM app.calibration_transfer_trial
|
|
WHERE transfer_suite_record_id = ANY($1::uuid[])
|
|
ORDER BY transfer_suite_record_id, created_at, transfer_trial_record_id
|
|
""",
|
|
suite_ids,
|
|
)
|
|
)
|
|
if suite_ids
|
|
else []
|
|
)
|
|
transfer_assessments = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT transfer_assessment_id, transfer_suite_record_id,
|
|
competency_id, source_trial_ids, assessment_payload,
|
|
evidence_turn_ids, model_run_id, instrument_id,
|
|
instrument_version, created_at
|
|
FROM app.calibration_transfer_assessment
|
|
WHERE transfer_suite_record_id = ANY($1::uuid[])
|
|
ORDER BY transfer_suite_record_id, competency_id
|
|
""",
|
|
suite_ids,
|
|
)
|
|
)
|
|
if suite_ids
|
|
else []
|
|
)
|
|
drift_reports = (
|
|
list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT drift_report_id, transfer_suite_record_id,
|
|
competency_id, source_trial_ids, report_payload,
|
|
model_run_id, instrument_id, instrument_version,
|
|
data_classification, clinical_claim_allowed, created_at
|
|
FROM app.calibration_subgroup_drift_report
|
|
WHERE transfer_suite_record_id = ANY($1::uuid[])
|
|
ORDER BY transfer_suite_record_id, competency_id
|
|
""",
|
|
suite_ids,
|
|
)
|
|
)
|
|
if suite_ids
|
|
else []
|
|
)
|
|
reviews = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT review_id, submission_id, target_kind, target_id,
|
|
review_no, supersedes_review_id, disposition,
|
|
correction_payload, review_reason, evidence_turn_ids,
|
|
counterevidence, created_by_uid, created_by_role, created_at
|
|
FROM app.calibration_teacher_review_event
|
|
WHERE learner_id = $1
|
|
ORDER BY target_kind, target_id, review_no
|
|
""",
|
|
target_learner_id,
|
|
)
|
|
)
|
|
actual_event_rows = list(
|
|
await conn.fetch(
|
|
"""
|
|
SELECT execution.*,
|
|
practice_session.learner_feedback_enabled
|
|
AS source_learner_feedback_enabled
|
|
FROM app.calibration_transfer_execution_event execution
|
|
JOIN app.sessions practice_session
|
|
ON practice_session.id = execution.practice_session_id
|
|
WHERE execution.learner_id = $1
|
|
ORDER BY execution.competency_id, execution.created_at,
|
|
execution.execution_event_id
|
|
""",
|
|
target_learner_id,
|
|
)
|
|
)
|
|
learner_feedback_snapshot_enabled = all(
|
|
bool(
|
|
_value(
|
|
item,
|
|
"source_learner_feedback_enabled",
|
|
True,
|
|
)
|
|
)
|
|
for item in (*histories, *assessments, *suites, *actual_event_rows)
|
|
)
|
|
|
|
revisions_by_history: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in revisions:
|
|
revisions_by_history.setdefault(
|
|
UUID(str(_value(row, "history_id"))), []
|
|
).append(dict(row))
|
|
locks_by_history = {
|
|
UUID(str(_value(row, "history_id"))): dict(row) for row in locks
|
|
}
|
|
observations_by_history = {
|
|
UUID(str(_value(row, "history_id"))): dict(row) for row in observations
|
|
}
|
|
prediction_histories: list[dict[str, Any]] = []
|
|
for row in histories:
|
|
item = _public_row(row)
|
|
history_id = UUID(str(_value(row, "history_id")))
|
|
item["revisions"] = revisions_by_history.get(history_id, [])
|
|
item["lock"] = locks_by_history.get(history_id)
|
|
item["external_observation"] = observations_by_history.get(history_id)
|
|
prediction_histories.append(item)
|
|
trials_by_suite: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in trials:
|
|
trials_by_suite.setdefault(
|
|
UUID(str(_value(row, "transfer_suite_record_id"))), []
|
|
).append(dict(row))
|
|
assessments_by_suite: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in transfer_assessments:
|
|
assessments_by_suite.setdefault(
|
|
UUID(str(_value(row, "transfer_suite_record_id"))), []
|
|
).append(dict(row))
|
|
drift_by_suite: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in drift_reports:
|
|
drift_by_suite.setdefault(
|
|
UUID(str(_value(row, "transfer_suite_record_id"))), []
|
|
).append(dict(row))
|
|
suite_payloads: list[dict[str, Any]] = []
|
|
for row in suites:
|
|
item = _public_row(row)
|
|
suite_id = UUID(str(_value(row, "transfer_suite_record_id")))
|
|
item["trials"] = trials_by_suite.get(suite_id, [])
|
|
item["assessments"] = assessments_by_suite.get(suite_id, [])
|
|
item["drift_reports"] = drift_by_suite.get(suite_id, [])
|
|
suite_payloads.append(item)
|
|
actual_executions = tuple(
|
|
_actual_execution_from_row(row) for row in actual_event_rows
|
|
)
|
|
actual_assessments = assess_actual_transfer_executions(actual_executions)
|
|
return {
|
|
"learner_id": target_learner_id,
|
|
"requested_view": requested_view,
|
|
"clinical_claim_allowed": False,
|
|
"_learner_feedback_snapshot_enabled": learner_feedback_snapshot_enabled,
|
|
"prediction_histories": prediction_histories,
|
|
"calibration_assessments": [_public_row(item) for item in assessments],
|
|
"transfer_suites": suite_payloads,
|
|
"teacher_reviews": [dict(item) for item in reviews],
|
|
"actual_executions": [
|
|
item.model_dump(mode="python") for item in actual_executions
|
|
],
|
|
"actual_transfer_assessments": [
|
|
item.model_dump(mode="python") for item in actual_assessments
|
|
],
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"CalibrationTransferConflictError",
|
|
"CalibrationTransferNotFoundError",
|
|
"CalibrationTransferStateError",
|
|
"append_calibration_assessment",
|
|
"append_performance_observation",
|
|
"append_prediction_lock",
|
|
"append_prediction_revision",
|
|
"append_teacher_review",
|
|
"append_transfer_suite",
|
|
"append_actual_transfer_execution",
|
|
"read_calibration_transfer",
|
|
]
|