G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
967
apps/api/app/services/supervision_research_store.py
Normal file
967
apps/api/app/services/supervision_research_store.py
Normal file
|
|
@ -0,0 +1,967 @@
|
|||
"""Append-only persistence for the G6 Supervision & Research OS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
import asyncpg
|
||||
|
||||
from ..contracts.supervision_research import (
|
||||
EvaluationVersionBatch,
|
||||
LedgerEvidencePointer,
|
||||
LearnerAttentionSignal,
|
||||
Phase3EvidenceArtifact,
|
||||
TeacherAiDisagreement,
|
||||
)
|
||||
from .supervision_research import (
|
||||
build_attention_queue,
|
||||
build_calibration_dataset,
|
||||
build_phase3_outcome_manifest,
|
||||
compare_evaluation_versions,
|
||||
)
|
||||
|
||||
|
||||
_POINTER_NAMESPACE = UUID("f99f95ba-365e-46ea-a613-2239275f8a2d")
|
||||
|
||||
|
||||
class SupervisionResearchError(ValueError):
|
||||
"""Base error for the G6 persistence boundary."""
|
||||
|
||||
|
||||
class SupervisionResearchConflictError(SupervisionResearchError):
|
||||
"""A stable submission id was reused with changed content."""
|
||||
|
||||
|
||||
class SupervisionResearchNotFoundError(SupervisionResearchError):
|
||||
"""Required source evidence or a visible aggregate does not exist."""
|
||||
|
||||
|
||||
def _value(row: Mapping[str, Any], key: str, default: Any = None) -> Any:
|
||||
try:
|
||||
return row[key]
|
||||
except (KeyError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def _canonical_hash(payload: Any) -> str:
|
||||
serialized = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
default=str,
|
||||
)
|
||||
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def _existing_submission(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
table: str,
|
||||
id_column: str,
|
||||
submission_id: UUID,
|
||||
content_hash: str,
|
||||
) -> UUID | None:
|
||||
row = await conn.fetchrow(
|
||||
f"SELECT {id_column}, content_hash FROM {table} WHERE submission_id = $1",
|
||||
submission_id,
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
if str(_value(row, "content_hash")) != content_hash:
|
||||
raise SupervisionResearchConflictError(
|
||||
"submission id was already used with different content"
|
||||
)
|
||||
return UUID(str(_value(row, id_column)))
|
||||
|
||||
|
||||
def _pointer_id(
|
||||
learner_id: UUID, consumer_view: str, pointer: LedgerEvidencePointer
|
||||
) -> UUID:
|
||||
return uuid5(
|
||||
_POINTER_NAMESPACE,
|
||||
"|".join(
|
||||
(
|
||||
str(learner_id),
|
||||
consumer_view,
|
||||
pointer.ledger,
|
||||
pointer.event_id,
|
||||
pointer.route_hint,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _ensure_pointer(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
learner_id: UUID,
|
||||
cohort_id: str,
|
||||
consumer_view: str,
|
||||
pointer: LedgerEvidencePointer,
|
||||
) -> UUID:
|
||||
pointer_id = _pointer_id(learner_id, consumer_view, pointer)
|
||||
payload = {
|
||||
"pointer_id": str(pointer_id),
|
||||
"learner_id": str(learner_id),
|
||||
"cohort_id": cohort_id,
|
||||
"consumer_view": consumer_view,
|
||||
**pointer.model_dump(mode="json"),
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
try:
|
||||
session_id = UUID(pointer.session_id) if pointer.session_id else None
|
||||
except ValueError as exc:
|
||||
raise SupervisionResearchError(
|
||||
"ledger evidence session_id must be a UUID when provided"
|
||||
) from exc
|
||||
try:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_evidence_pointer (
|
||||
pointer_id, learner_id, cohort_id, consumer_view, ledger,
|
||||
event_id, session_id, route_hint, content_hash
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (pointer_id) DO NOTHING
|
||||
""",
|
||||
pointer_id,
|
||||
learner_id,
|
||||
cohort_id,
|
||||
consumer_view,
|
||||
pointer.ledger,
|
||||
pointer.event_id,
|
||||
session_id,
|
||||
pointer.route_hint,
|
||||
content_hash,
|
||||
)
|
||||
except (asyncpg.CheckViolationError, asyncpg.InvalidTextRepresentationError) as exc:
|
||||
raise SupervisionResearchNotFoundError(
|
||||
"ledger evidence pointer is invalid or outside learner scope"
|
||||
) from exc
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT content_hash FROM app.supervision_evidence_pointer
|
||||
WHERE pointer_id = $1
|
||||
""",
|
||||
pointer_id,
|
||||
)
|
||||
if row is None:
|
||||
raise SupervisionResearchNotFoundError("ledger evidence pointer is not visible")
|
||||
if str(_value(row, "content_hash")) != content_hash:
|
||||
raise SupervisionResearchConflictError(
|
||||
"stable evidence pointer resolved to changed metadata"
|
||||
)
|
||||
return pointer_id
|
||||
|
||||
|
||||
async def append_attention_snapshot(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
snapshot_id: UUID,
|
||||
cohort_id: str,
|
||||
signals: Sequence[LearnerAttentionSignal],
|
||||
learner_ids_by_ref: Mapping[str, UUID],
|
||||
) -> dict[str, Any]:
|
||||
queue = build_attention_queue(signals)
|
||||
if not queue:
|
||||
raise SupervisionResearchError("attention snapshot requires an active item")
|
||||
missing = {item.learner_ref for item in queue} - set(learner_ids_by_ref)
|
||||
if missing:
|
||||
raise SupervisionResearchError(
|
||||
f"learner UUID mapping is missing: {','.join(sorted(missing))}"
|
||||
)
|
||||
payload = {
|
||||
"snapshot_id": str(snapshot_id),
|
||||
"cohort_id": cohort_id,
|
||||
"signals": [item.model_dump(mode="json") for item in signals],
|
||||
"learner_ids_by_ref": {
|
||||
key: str(value) for key, value in sorted(learner_ids_by_ref.items())
|
||||
},
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 61))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_attention_snapshot",
|
||||
id_column="snapshot_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"snapshot_id": existing,
|
||||
"item_count": len(queue),
|
||||
"idempotent_replay": True,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
staged: list[tuple[Any, UUID, list[tuple[Any, UUID]]]] = []
|
||||
all_pointer_ids: list[UUID] = []
|
||||
for item in queue:
|
||||
learner_id = learner_ids_by_ref[item.learner_ref]
|
||||
selected: list[tuple[Any, UUID]] = []
|
||||
seen_pointer_ids: set[UUID] = set()
|
||||
for reason in item.reasons:
|
||||
if len(selected) >= 3:
|
||||
break
|
||||
pointer = reason.evidence[0]
|
||||
pointer_id = await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_id,
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="supervisor",
|
||||
pointer=pointer,
|
||||
)
|
||||
if pointer_id in seen_pointer_ids:
|
||||
continue
|
||||
seen_pointer_ids.add(pointer_id)
|
||||
selected.append((reason, pointer_id))
|
||||
all_pointer_ids.append(pointer_id)
|
||||
if not selected:
|
||||
raise SupervisionResearchError("attention item requires direct evidence")
|
||||
staged.append((item, learner_id, selected))
|
||||
|
||||
unique_source_ids = list(dict.fromkeys(all_pointer_ids))
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_attention_snapshot (
|
||||
snapshot_id, submission_id, content_hash, cohort_id, item_count,
|
||||
source_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6)
|
||||
""",
|
||||
snapshot_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
cohort_id,
|
||||
len(queue),
|
||||
unique_source_ids,
|
||||
)
|
||||
for item, learner_id, selected in staged:
|
||||
item_id = uuid5(snapshot_id, item.learner_ref)
|
||||
pointer_ids = [entry[1] for entry in selected]
|
||||
routes = list(dict.fromkeys(entry[0].evidence[0].route_hint for entry in selected))
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_attention_item (
|
||||
item_id, snapshot_id, learner_id, learner_ref, cohort_id,
|
||||
queue_position, primary_signal, oldest_active_sequence,
|
||||
drilldown_routes, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
""",
|
||||
item_id,
|
||||
snapshot_id,
|
||||
learner_id,
|
||||
item.learner_ref,
|
||||
cohort_id,
|
||||
item.queue_position,
|
||||
item.primary_signal,
|
||||
item.oldest_active_sequence,
|
||||
routes,
|
||||
pointer_ids,
|
||||
)
|
||||
for reason, pointer_id in selected:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_attention_reason (
|
||||
reason_id, item_id, signal_id, signal_type, severity,
|
||||
uncertainty, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||
""",
|
||||
uuid5(item_id, reason.signal_id),
|
||||
item_id,
|
||||
reason.signal_id,
|
||||
reason.signal_type,
|
||||
reason.severity,
|
||||
reason.uncertainty,
|
||||
[pointer_id],
|
||||
)
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"item_count": len(queue),
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def append_teacher_disagreement(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
disagreement_record_id: UUID,
|
||||
dataset_row_id: UUID,
|
||||
audit_event_id: UUID,
|
||||
learner_id: UUID,
|
||||
cohort_id: str,
|
||||
actor_uid: UUID,
|
||||
disagreement: TeacherAiDisagreement,
|
||||
) -> dict[str, Any]:
|
||||
dataset_row = build_calibration_dataset((disagreement,))[0]
|
||||
payload = {
|
||||
"disagreement_record_id": str(disagreement_record_id),
|
||||
"dataset_row_id": str(dataset_row_id),
|
||||
"audit_event_id": str(audit_event_id),
|
||||
"learner_id": str(learner_id),
|
||||
"cohort_id": cohort_id,
|
||||
"actor_uid": str(actor_uid),
|
||||
"disagreement": disagreement.model_dump(mode="json"),
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 62))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_teacher_ai_disagreement",
|
||||
id_column="disagreement_record_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"disagreement_record_id": existing,
|
||||
"dataset_row_hash": dataset_row.row_id,
|
||||
"idempotent_replay": True,
|
||||
"raw_transcript_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
ai_ids = [
|
||||
await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_id,
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="research",
|
||||
pointer=pointer,
|
||||
)
|
||||
for pointer in disagreement.ai_evidence
|
||||
]
|
||||
teacher_ids = [
|
||||
await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_id,
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="research",
|
||||
pointer=pointer,
|
||||
)
|
||||
for pointer in disagreement.teacher_correction_evidence
|
||||
]
|
||||
evidence_ids = list(dict.fromkeys((*ai_ids, *teacher_ids)))
|
||||
if len(evidence_ids) < 2:
|
||||
raise SupervisionResearchError(
|
||||
"calibration dataset requires two distinct ledger UUID pointers"
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_teacher_ai_disagreement (
|
||||
disagreement_record_id, submission_id, content_hash, disagreement_id,
|
||||
learner_id, cohort_id, case_ref, competency_id, ai_label, teacher_label,
|
||||
ai_model, prompt_version, instrument_id, instrument_version,
|
||||
correction_reason_code, ai_evidence_pointer_ids,
|
||||
teacher_evidence_pointer_ids, created_by_uid
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
|
||||
""",
|
||||
disagreement_record_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
disagreement.disagreement_id,
|
||||
learner_id,
|
||||
cohort_id,
|
||||
disagreement.case_ref,
|
||||
disagreement.competency_id,
|
||||
disagreement.ai_label,
|
||||
disagreement.teacher_label,
|
||||
disagreement.ai_model,
|
||||
disagreement.prompt_version,
|
||||
disagreement.instrument_id,
|
||||
disagreement.instrument_version,
|
||||
disagreement.correction_reason_code,
|
||||
ai_ids,
|
||||
teacher_ids,
|
||||
actor_uid,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_calibration_dataset_row (
|
||||
dataset_row_id, disagreement_record_id, learner_id, cohort_id,
|
||||
row_hash, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6)
|
||||
""",
|
||||
dataset_row_id,
|
||||
disagreement_record_id,
|
||||
learner_id,
|
||||
cohort_id,
|
||||
dataset_row.row_id,
|
||||
evidence_ids,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO audit.supervision_teacher_event (
|
||||
audit_event_id, disagreement_record_id, actor_uid, learner_id,
|
||||
cohort_id, action, content_hash, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,'teacher_ai_disagreement.corrected',$6,$7)
|
||||
""",
|
||||
audit_event_id,
|
||||
disagreement_record_id,
|
||||
actor_uid,
|
||||
learner_id,
|
||||
cohort_id,
|
||||
content_hash,
|
||||
evidence_ids,
|
||||
)
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"disagreement_record_id": disagreement_record_id,
|
||||
"dataset_row_hash": dataset_row.row_id,
|
||||
"idempotent_replay": False,
|
||||
"raw_transcript_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def append_curriculum_gap(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
gap_snapshot_id: UUID,
|
||||
cohort_id: str,
|
||||
competency_id: str,
|
||||
gap_kind: str,
|
||||
status: str,
|
||||
uncertainty: float,
|
||||
affected_learner_count: int,
|
||||
evidence: Sequence[tuple[UUID, LedgerEvidencePointer]],
|
||||
) -> dict[str, Any]:
|
||||
payload = {
|
||||
"gap_snapshot_id": str(gap_snapshot_id),
|
||||
"cohort_id": cohort_id,
|
||||
"competency_id": competency_id,
|
||||
"gap_kind": gap_kind,
|
||||
"status": status,
|
||||
"uncertainty": uncertainty,
|
||||
"affected_learner_count": affected_learner_count,
|
||||
"evidence": [
|
||||
{"learner_id": str(learner_id), **pointer.model_dump(mode="json")}
|
||||
for learner_id, pointer in evidence
|
||||
],
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 63))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_curriculum_gap_snapshot",
|
||||
id_column="gap_snapshot_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"gap_snapshot_id": existing,
|
||||
"idempotent_replay": True,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
if status == "insufficient_evidence":
|
||||
if evidence or uncertainty != 1.0:
|
||||
raise SupervisionResearchError(
|
||||
"insufficient curriculum gap must remain evidence-free"
|
||||
)
|
||||
pointer_ids: list[UUID] = []
|
||||
else:
|
||||
if not evidence:
|
||||
raise SupervisionResearchError("observed curriculum gap requires evidence")
|
||||
pointer_ids = [
|
||||
await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_id,
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="supervisor",
|
||||
pointer=pointer,
|
||||
)
|
||||
for learner_id, pointer in evidence
|
||||
]
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_curriculum_gap_snapshot (
|
||||
gap_snapshot_id, submission_id, content_hash, cohort_id, competency_id,
|
||||
gap_kind, status, uncertainty, affected_learner_count, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
""",
|
||||
gap_snapshot_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
cohort_id,
|
||||
competency_id,
|
||||
gap_kind,
|
||||
status,
|
||||
uncertainty,
|
||||
affected_learner_count,
|
||||
list(dict.fromkeys(pointer_ids)),
|
||||
)
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"gap_snapshot_id": gap_snapshot_id,
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def _append_batch(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
batch_record_id: UUID,
|
||||
cohort_id: str,
|
||||
batch: EvaluationVersionBatch,
|
||||
pointers_by_event_id: Mapping[str, LedgerEvidencePointer],
|
||||
learner_ids_by_event_id: Mapping[str, UUID],
|
||||
) -> tuple[UUID, bool, list[UUID]]:
|
||||
payload = {
|
||||
"batch_record_id": str(batch_record_id),
|
||||
"cohort_id": cohort_id,
|
||||
"batch": batch.model_dump(mode="json"),
|
||||
"pointers": {
|
||||
key: value.model_dump(mode="json")
|
||||
for key, value in sorted(pointers_by_event_id.items())
|
||||
},
|
||||
"learner_ids": {
|
||||
key: str(value) for key, value in sorted(learner_ids_by_event_id.items())
|
||||
},
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 66))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_evaluation_batch",
|
||||
id_column="batch_record_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
rows = await conn.fetch(
|
||||
"SELECT evidence_pointer_id FROM app.supervision_evaluation_observation WHERE batch_record_id = $1",
|
||||
existing,
|
||||
)
|
||||
return existing, True, [UUID(str(row["evidence_pointer_id"])) for row in rows]
|
||||
event_ids = [item.evidence_event_id for item in batch.observations]
|
||||
missing = set(event_ids) - set(pointers_by_event_id) | set(event_ids) - set(
|
||||
learner_ids_by_event_id
|
||||
)
|
||||
if missing:
|
||||
raise SupervisionResearchError(
|
||||
f"evaluation evidence mapping is missing: {','.join(sorted(missing))}"
|
||||
)
|
||||
pointer_ids: list[UUID] = []
|
||||
for item in batch.observations:
|
||||
pointer = pointers_by_event_id[item.evidence_event_id]
|
||||
if pointer.event_id != item.evidence_event_id:
|
||||
raise SupervisionResearchError("evaluation evidence event id mismatch")
|
||||
pointer_ids.append(
|
||||
await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_ids_by_event_id[item.evidence_event_id],
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="research",
|
||||
pointer=pointer,
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_evaluation_batch (
|
||||
batch_record_id, submission_id, content_hash, batch_id, cohort_id,
|
||||
model_name, prompt_version, instrument_id, instrument_version,
|
||||
observation_count
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
""",
|
||||
batch_record_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
batch.batch_id,
|
||||
cohort_id,
|
||||
batch.model,
|
||||
batch.prompt_version,
|
||||
batch.instrument_id,
|
||||
batch.instrument_version,
|
||||
len(batch.observations),
|
||||
)
|
||||
for index, item in enumerate(batch.observations):
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_evaluation_observation (
|
||||
observation_record_id, batch_record_id, cohort_id, case_ref,
|
||||
competency_id, synthetic_subgroup, gold_label, predicted_label,
|
||||
evidence_pointer_id
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
""",
|
||||
uuid5(batch_record_id, f"{item.case_ref}|{item.competency_id}"),
|
||||
batch_record_id,
|
||||
cohort_id,
|
||||
item.case_ref,
|
||||
item.competency_id,
|
||||
item.synthetic_subgroup,
|
||||
item.gold_label,
|
||||
item.predicted_label,
|
||||
pointer_ids[index],
|
||||
)
|
||||
return batch_record_id, False, pointer_ids
|
||||
|
||||
|
||||
async def append_evaluation_comparison(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
drift_report_id: UUID,
|
||||
baseline_submission_id: UUID,
|
||||
baseline_batch_record_id: UUID,
|
||||
candidate_submission_id: UUID,
|
||||
candidate_batch_record_id: UUID,
|
||||
cohort_id: str,
|
||||
baseline: EvaluationVersionBatch,
|
||||
candidate: EvaluationVersionBatch,
|
||||
pointers_by_event_id: Mapping[str, LedgerEvidencePointer],
|
||||
learner_ids_by_event_id: Mapping[str, UUID],
|
||||
) -> dict[str, Any]:
|
||||
report = compare_evaluation_versions(baseline, candidate)
|
||||
payload = {
|
||||
"drift_report_id": str(drift_report_id),
|
||||
"cohort_id": cohort_id,
|
||||
"baseline": baseline.model_dump(mode="json"),
|
||||
"candidate": candidate.model_dump(mode="json"),
|
||||
"report": report.model_dump(mode="json"),
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 64))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_drift_report",
|
||||
id_column="drift_report_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"drift_report_id": existing,
|
||||
"status": report.status,
|
||||
"matched_count": report.matched_count,
|
||||
"idempotent_replay": True,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
baseline_record_id, _, baseline_pointers = await _append_batch(
|
||||
conn,
|
||||
submission_id=baseline_submission_id,
|
||||
batch_record_id=baseline_batch_record_id,
|
||||
cohort_id=cohort_id,
|
||||
batch=baseline,
|
||||
pointers_by_event_id=pointers_by_event_id,
|
||||
learner_ids_by_event_id=learner_ids_by_event_id,
|
||||
)
|
||||
candidate_record_id, _, candidate_pointers = await _append_batch(
|
||||
conn,
|
||||
submission_id=candidate_submission_id,
|
||||
batch_record_id=candidate_batch_record_id,
|
||||
cohort_id=cohort_id,
|
||||
batch=candidate,
|
||||
pointers_by_event_id=pointers_by_event_id,
|
||||
learner_ids_by_event_id=learner_ids_by_event_id,
|
||||
)
|
||||
evidence_ids = list(dict.fromkeys((*baseline_pointers, *candidate_pointers)))
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_drift_report (
|
||||
drift_report_id, submission_id, content_hash, cohort_id,
|
||||
baseline_batch_record_id, candidate_batch_record_id, matched_count,
|
||||
status, baseline_accuracy, candidate_accuracy, accuracy_delta,
|
||||
disagreement_case_refs, alerts, evidence_pointer_ids
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)
|
||||
""",
|
||||
drift_report_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
cohort_id,
|
||||
baseline_record_id,
|
||||
candidate_record_id,
|
||||
report.matched_count,
|
||||
report.status,
|
||||
report.baseline_accuracy,
|
||||
report.candidate_accuracy,
|
||||
report.accuracy_delta,
|
||||
list(report.disagreement_case_refs),
|
||||
list(report.alerts),
|
||||
evidence_ids,
|
||||
)
|
||||
for metric in report.subgroup_metrics:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_drift_subgroup_metric (
|
||||
subgroup_metric_id, drift_report_id, cohort_id, subgroup,
|
||||
matched_count, baseline_accuracy, candidate_accuracy, accuracy_delta
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
""",
|
||||
uuid5(drift_report_id, metric.subgroup),
|
||||
drift_report_id,
|
||||
cohort_id,
|
||||
metric.subgroup,
|
||||
metric.matched_count,
|
||||
metric.baseline_accuracy,
|
||||
metric.candidate_accuracy,
|
||||
metric.accuracy_delta,
|
||||
)
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"drift_report_id": drift_report_id,
|
||||
"status": report.status,
|
||||
"matched_count": report.matched_count,
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def append_phase3_manifest(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
submission_id: UUID,
|
||||
manifest_id: UUID,
|
||||
cohort_id: str,
|
||||
artifacts: Sequence[Phase3EvidenceArtifact],
|
||||
source_by_domain: Mapping[str, tuple[UUID, LedgerEvidencePointer]],
|
||||
) -> dict[str, Any]:
|
||||
manifest = build_phase3_outcome_manifest(artifacts)
|
||||
missing = {item.domain for item in manifest.artifacts} - set(source_by_domain)
|
||||
if missing:
|
||||
raise SupervisionResearchError(
|
||||
f"manifest provenance mapping is missing: {','.join(sorted(missing))}"
|
||||
)
|
||||
payload = {
|
||||
"manifest_id": str(manifest_id),
|
||||
"cohort_id": cohort_id,
|
||||
"manifest": manifest.model_dump(mode="json"),
|
||||
"source_by_domain": {
|
||||
key: {
|
||||
"learner_id": str(value[0]),
|
||||
"pointer": value[1].model_dump(mode="json"),
|
||||
}
|
||||
for key, value in sorted(source_by_domain.items())
|
||||
},
|
||||
}
|
||||
content_hash = _canonical_hash(payload)
|
||||
await conn.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 65))",
|
||||
str(submission_id),
|
||||
)
|
||||
existing = await _existing_submission(
|
||||
conn,
|
||||
table="app.supervision_phase3_manifest",
|
||||
id_column="manifest_id",
|
||||
submission_id=submission_id,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"manifest_id": existing,
|
||||
"artifact_count": 4,
|
||||
"idempotent_replay": True,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
staged: list[tuple[Phase3EvidenceArtifact, UUID]] = []
|
||||
for artifact in manifest.artifacts:
|
||||
learner_id, pointer = source_by_domain[artifact.domain]
|
||||
staged.append(
|
||||
(
|
||||
artifact,
|
||||
await _ensure_pointer(
|
||||
conn,
|
||||
learner_id=learner_id,
|
||||
cohort_id=cohort_id,
|
||||
consumer_view="research",
|
||||
pointer=pointer,
|
||||
),
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_phase3_manifest (
|
||||
manifest_id, submission_id, content_hash, cohort_id,
|
||||
schema_version, artifact_count
|
||||
) VALUES ($1,$2,$3,$4,$5,4)
|
||||
""",
|
||||
manifest_id,
|
||||
submission_id,
|
||||
content_hash,
|
||||
cohort_id,
|
||||
manifest.schema_version,
|
||||
)
|
||||
for artifact, pointer_id in staged:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO app.supervision_phase3_artifact (
|
||||
artifact_record_id, manifest_id, cohort_id, domain, artifact_id,
|
||||
schema_version, content_sha256, record_count, provenance_uri,
|
||||
source_pointer_id
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
""",
|
||||
uuid5(manifest_id, artifact.domain),
|
||||
manifest_id,
|
||||
cohort_id,
|
||||
artifact.domain,
|
||||
artifact.artifact_id,
|
||||
artifact.schema_version,
|
||||
artifact.content_sha256,
|
||||
artifact.record_count,
|
||||
artifact.provenance_uri,
|
||||
pointer_id,
|
||||
)
|
||||
return {
|
||||
"submission_id": submission_id,
|
||||
"manifest_id": manifest_id,
|
||||
"artifact_count": 4,
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def read_supervision_view(conn: asyncpg.Connection) -> dict[str, Any]:
|
||||
items = await conn.fetch(
|
||||
"""
|
||||
SELECT item_id, snapshot_id, learner_id, learner_ref, cohort_id,
|
||||
queue_position, primary_signal, oldest_active_sequence,
|
||||
drilldown_routes, evidence_pointer_ids, created_at
|
||||
FROM app.supervision_attention_item ORDER BY created_at DESC, queue_position
|
||||
"""
|
||||
)
|
||||
gaps = await conn.fetch(
|
||||
"""
|
||||
SELECT gap_snapshot_id, cohort_id, competency_id, gap_kind, status,
|
||||
uncertainty, affected_learner_count, evidence_pointer_ids, created_at
|
||||
FROM app.supervision_curriculum_gap_snapshot ORDER BY created_at DESC
|
||||
"""
|
||||
)
|
||||
return {
|
||||
"attention_items": [dict(row) for row in items],
|
||||
"curriculum_gaps": [dict(row) for row in gaps],
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
async def read_research_view(conn: asyncpg.Connection) -> dict[str, Any]:
|
||||
datasets = await conn.fetch(
|
||||
"""
|
||||
SELECT d.dataset_row_id, d.row_hash, d.disagreement_record_id,
|
||||
d.evidence_pointer_ids, d.raw_transcript_included, d.created_at,
|
||||
x.case_ref, x.competency_id, x.ai_label, x.teacher_label,
|
||||
x.ai_model, x.prompt_version, x.instrument_id,
|
||||
x.instrument_version, x.correction_reason_code
|
||||
FROM app.supervision_calibration_dataset_row d
|
||||
JOIN app.supervision_teacher_ai_disagreement x
|
||||
ON x.disagreement_record_id = d.disagreement_record_id
|
||||
ORDER BY d.created_at DESC
|
||||
"""
|
||||
)
|
||||
drift = await conn.fetch(
|
||||
"""
|
||||
SELECT r.drift_report_id, r.cohort_id, r.matched_count, r.status,
|
||||
r.baseline_accuracy, r.candidate_accuracy, r.accuracy_delta,
|
||||
r.disagreement_case_refs, r.alerts, r.evidence_pointer_ids,
|
||||
r.created_at,
|
||||
b.model_name AS baseline_model,
|
||||
c.model_name AS candidate_model,
|
||||
b.prompt_version AS baseline_prompt_version,
|
||||
c.prompt_version AS candidate_prompt_version,
|
||||
b.instrument_id,
|
||||
b.instrument_version AS baseline_instrument_version,
|
||||
c.instrument_version AS candidate_instrument_version
|
||||
FROM app.supervision_drift_report r
|
||||
JOIN app.supervision_evaluation_batch b
|
||||
ON b.batch_record_id = r.baseline_batch_record_id
|
||||
JOIN app.supervision_evaluation_batch c
|
||||
ON c.batch_record_id = r.candidate_batch_record_id
|
||||
ORDER BY r.created_at DESC
|
||||
"""
|
||||
)
|
||||
subgroup_metrics = await conn.fetch(
|
||||
"""
|
||||
SELECT drift_report_id, subgroup, matched_count, baseline_accuracy,
|
||||
candidate_accuracy, accuracy_delta
|
||||
FROM app.supervision_drift_subgroup_metric
|
||||
ORDER BY drift_report_id, subgroup
|
||||
"""
|
||||
)
|
||||
manifests = await conn.fetch(
|
||||
"""
|
||||
SELECT manifest_id, cohort_id, schema_version, artifact_count, created_at
|
||||
FROM app.supervision_phase3_manifest ORDER BY created_at DESC
|
||||
"""
|
||||
)
|
||||
manifest_artifacts = await conn.fetch(
|
||||
"""
|
||||
SELECT manifest_id, domain, artifact_id, schema_version, content_sha256,
|
||||
record_count, provenance_uri, clinical_claim_allowed
|
||||
FROM app.supervision_phase3_artifact
|
||||
ORDER BY manifest_id, domain
|
||||
"""
|
||||
)
|
||||
subgroup_by_report: dict[UUID, list[dict[str, Any]]] = {}
|
||||
for row in subgroup_metrics:
|
||||
payload = dict(row)
|
||||
report_id = payload.pop("drift_report_id")
|
||||
subgroup_by_report.setdefault(report_id, []).append(payload)
|
||||
drift_payloads: list[dict[str, Any]] = []
|
||||
for row in drift:
|
||||
payload = dict(row)
|
||||
payload["subgroup_metrics"] = subgroup_by_report.get(
|
||||
payload["drift_report_id"], []
|
||||
)
|
||||
drift_payloads.append(payload)
|
||||
|
||||
artifacts_by_manifest: dict[UUID, list[dict[str, Any]]] = {}
|
||||
for row in manifest_artifacts:
|
||||
payload = dict(row)
|
||||
manifest_id = payload.pop("manifest_id")
|
||||
artifacts_by_manifest.setdefault(manifest_id, []).append(payload)
|
||||
manifest_payloads: list[dict[str, Any]] = []
|
||||
for row in manifests:
|
||||
payload = dict(row)
|
||||
payload["artifacts"] = artifacts_by_manifest.get(payload["manifest_id"], [])
|
||||
manifest_payloads.append(payload)
|
||||
return {
|
||||
"calibration_dataset": [dict(row) for row in datasets],
|
||||
"drift_reports": drift_payloads,
|
||||
"phase3_manifests": manifest_payloads,
|
||||
"raw_transcript_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SupervisionResearchConflictError",
|
||||
"SupervisionResearchError",
|
||||
"SupervisionResearchNotFoundError",
|
||||
"append_attention_snapshot",
|
||||
"append_curriculum_gap",
|
||||
"append_evaluation_comparison",
|
||||
"append_phase3_manifest",
|
||||
"append_teacher_disagreement",
|
||||
"read_research_view",
|
||||
"read_supervision_view",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue