951 lines
32 KiB
Python
951 lines
32 KiB
Python
"""Append-only persistence for the G6 Supervision & Research OS."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
from uuid import UUID, uuid5
|
|
|
|
import asyncpg
|
|
|
|
from ..contracts.supervision_research import (
|
|
EvaluationVersionBatch,
|
|
LedgerEvidencePointer,
|
|
LearnerAttentionSignal,
|
|
Phase3EvidenceArtifact,
|
|
TeacherAiDisagreement,
|
|
)
|
|
from .supervision_research import (
|
|
build_attention_queue,
|
|
build_calibration_dataset,
|
|
build_phase3_outcome_manifest,
|
|
compare_evaluation_versions,
|
|
)
|
|
from .outcome_repository_values import (
|
|
canonical_hash as _canonical_hash,
|
|
value as _value,
|
|
)
|
|
|
|
|
|
_POINTER_NAMESPACE = UUID("f99f95ba-365e-46ea-a613-2239275f8a2d")
|
|
|
|
|
|
class SupervisionResearchError(ValueError):
|
|
"""Base error for the G6 persistence boundary."""
|
|
|
|
|
|
class SupervisionResearchConflictError(SupervisionResearchError):
|
|
"""A stable submission id was reused with changed content."""
|
|
|
|
|
|
class SupervisionResearchNotFoundError(SupervisionResearchError):
|
|
"""Required source evidence or a visible aggregate does not exist."""
|
|
|
|
|
|
async def _existing_submission(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
table: str,
|
|
id_column: str,
|
|
submission_id: UUID,
|
|
content_hash: str,
|
|
) -> UUID | None:
|
|
row = await conn.fetchrow(
|
|
f"SELECT {id_column}, content_hash FROM {table} WHERE submission_id = $1",
|
|
submission_id,
|
|
)
|
|
if row is None:
|
|
return None
|
|
if str(_value(row, "content_hash")) != content_hash:
|
|
raise SupervisionResearchConflictError(
|
|
"submission id was already used with different content"
|
|
)
|
|
return UUID(str(_value(row, id_column)))
|
|
|
|
|
|
def _pointer_id(
|
|
learner_id: UUID, consumer_view: str, pointer: LedgerEvidencePointer
|
|
) -> UUID:
|
|
return uuid5(
|
|
_POINTER_NAMESPACE,
|
|
"|".join(
|
|
(
|
|
str(learner_id),
|
|
consumer_view,
|
|
pointer.ledger,
|
|
pointer.event_id,
|
|
pointer.route_hint,
|
|
)
|
|
),
|
|
)
|
|
|
|
|
|
async def _ensure_pointer(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
learner_id: UUID,
|
|
cohort_id: str,
|
|
consumer_view: str,
|
|
pointer: LedgerEvidencePointer,
|
|
) -> UUID:
|
|
pointer_id = _pointer_id(learner_id, consumer_view, pointer)
|
|
payload = {
|
|
"pointer_id": str(pointer_id),
|
|
"learner_id": str(learner_id),
|
|
"cohort_id": cohort_id,
|
|
"consumer_view": consumer_view,
|
|
**pointer.model_dump(mode="json"),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
try:
|
|
session_id = UUID(pointer.session_id) if pointer.session_id else None
|
|
except ValueError as exc:
|
|
raise SupervisionResearchError(
|
|
"ledger evidence session_id must be a UUID when provided"
|
|
) from exc
|
|
try:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_evidence_pointer (
|
|
pointer_id, learner_id, cohort_id, consumer_view, ledger,
|
|
event_id, session_id, route_hint, content_hash
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
|
ON CONFLICT (pointer_id) DO NOTHING
|
|
""",
|
|
pointer_id,
|
|
learner_id,
|
|
cohort_id,
|
|
consumer_view,
|
|
pointer.ledger,
|
|
pointer.event_id,
|
|
session_id,
|
|
pointer.route_hint,
|
|
content_hash,
|
|
)
|
|
except (asyncpg.CheckViolationError, asyncpg.InvalidTextRepresentationError) as exc:
|
|
raise SupervisionResearchNotFoundError(
|
|
"ledger evidence pointer is invalid or outside learner scope"
|
|
) from exc
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT content_hash FROM app.supervision_evidence_pointer
|
|
WHERE pointer_id = $1
|
|
""",
|
|
pointer_id,
|
|
)
|
|
if row is None:
|
|
raise SupervisionResearchNotFoundError("ledger evidence pointer is not visible")
|
|
if str(_value(row, "content_hash")) != content_hash:
|
|
raise SupervisionResearchConflictError(
|
|
"stable evidence pointer resolved to changed metadata"
|
|
)
|
|
return pointer_id
|
|
|
|
|
|
async def append_attention_snapshot(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
snapshot_id: UUID,
|
|
cohort_id: str,
|
|
signals: Sequence[LearnerAttentionSignal],
|
|
learner_ids_by_ref: Mapping[str, UUID],
|
|
) -> dict[str, Any]:
|
|
queue = build_attention_queue(signals)
|
|
if not queue:
|
|
raise SupervisionResearchError("attention snapshot requires an active item")
|
|
missing = {item.learner_ref for item in queue} - set(learner_ids_by_ref)
|
|
if missing:
|
|
raise SupervisionResearchError(
|
|
f"learner UUID mapping is missing: {','.join(sorted(missing))}"
|
|
)
|
|
payload = {
|
|
"snapshot_id": str(snapshot_id),
|
|
"cohort_id": cohort_id,
|
|
"signals": [item.model_dump(mode="json") for item in signals],
|
|
"learner_ids_by_ref": {
|
|
key: str(value) for key, value in sorted(learner_ids_by_ref.items())
|
|
},
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 61))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_attention_snapshot",
|
|
id_column="snapshot_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"snapshot_id": existing,
|
|
"item_count": len(queue),
|
|
"idempotent_replay": True,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
staged: list[tuple[Any, UUID, list[tuple[Any, UUID]]]] = []
|
|
all_pointer_ids: list[UUID] = []
|
|
for item in queue:
|
|
learner_id = learner_ids_by_ref[item.learner_ref]
|
|
selected: list[tuple[Any, UUID]] = []
|
|
seen_pointer_ids: set[UUID] = set()
|
|
for reason in item.reasons:
|
|
if len(selected) >= 3:
|
|
break
|
|
pointer = reason.evidence[0]
|
|
pointer_id = await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_id,
|
|
cohort_id=cohort_id,
|
|
consumer_view="supervisor",
|
|
pointer=pointer,
|
|
)
|
|
if pointer_id in seen_pointer_ids:
|
|
continue
|
|
seen_pointer_ids.add(pointer_id)
|
|
selected.append((reason, pointer_id))
|
|
all_pointer_ids.append(pointer_id)
|
|
if not selected:
|
|
raise SupervisionResearchError("attention item requires direct evidence")
|
|
staged.append((item, learner_id, selected))
|
|
|
|
unique_source_ids = list(dict.fromkeys(all_pointer_ids))
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_attention_snapshot (
|
|
snapshot_id, submission_id, content_hash, cohort_id, item_count,
|
|
source_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6)
|
|
""",
|
|
snapshot_id,
|
|
submission_id,
|
|
content_hash,
|
|
cohort_id,
|
|
len(queue),
|
|
unique_source_ids,
|
|
)
|
|
for item, learner_id, selected in staged:
|
|
item_id = uuid5(snapshot_id, item.learner_ref)
|
|
pointer_ids = [entry[1] for entry in selected]
|
|
routes = list(dict.fromkeys(entry[0].evidence[0].route_hint for entry in selected))
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_attention_item (
|
|
item_id, snapshot_id, learner_id, learner_ref, cohort_id,
|
|
queue_position, primary_signal, oldest_active_sequence,
|
|
drilldown_routes, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
|
""",
|
|
item_id,
|
|
snapshot_id,
|
|
learner_id,
|
|
item.learner_ref,
|
|
cohort_id,
|
|
item.queue_position,
|
|
item.primary_signal,
|
|
item.oldest_active_sequence,
|
|
routes,
|
|
pointer_ids,
|
|
)
|
|
for reason, pointer_id in selected:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_attention_reason (
|
|
reason_id, item_id, signal_id, signal_type, severity,
|
|
uncertainty, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
""",
|
|
uuid5(item_id, reason.signal_id),
|
|
item_id,
|
|
reason.signal_id,
|
|
reason.signal_type,
|
|
reason.severity,
|
|
reason.uncertainty,
|
|
[pointer_id],
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"snapshot_id": snapshot_id,
|
|
"item_count": len(queue),
|
|
"idempotent_replay": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def append_teacher_disagreement(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
disagreement_record_id: UUID,
|
|
dataset_row_id: UUID,
|
|
audit_event_id: UUID,
|
|
learner_id: UUID,
|
|
cohort_id: str,
|
|
actor_uid: UUID,
|
|
disagreement: TeacherAiDisagreement,
|
|
) -> dict[str, Any]:
|
|
dataset_row = build_calibration_dataset((disagreement,))[0]
|
|
payload = {
|
|
"disagreement_record_id": str(disagreement_record_id),
|
|
"dataset_row_id": str(dataset_row_id),
|
|
"audit_event_id": str(audit_event_id),
|
|
"learner_id": str(learner_id),
|
|
"cohort_id": cohort_id,
|
|
"actor_uid": str(actor_uid),
|
|
"disagreement": disagreement.model_dump(mode="json"),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 62))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_teacher_ai_disagreement",
|
|
id_column="disagreement_record_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"disagreement_record_id": existing,
|
|
"dataset_row_hash": dataset_row.row_id,
|
|
"idempotent_replay": True,
|
|
"raw_transcript_included": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
ai_ids = [
|
|
await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_id,
|
|
cohort_id=cohort_id,
|
|
consumer_view="research",
|
|
pointer=pointer,
|
|
)
|
|
for pointer in disagreement.ai_evidence
|
|
]
|
|
teacher_ids = [
|
|
await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_id,
|
|
cohort_id=cohort_id,
|
|
consumer_view="research",
|
|
pointer=pointer,
|
|
)
|
|
for pointer in disagreement.teacher_correction_evidence
|
|
]
|
|
evidence_ids = list(dict.fromkeys((*ai_ids, *teacher_ids)))
|
|
if len(evidence_ids) < 2:
|
|
raise SupervisionResearchError(
|
|
"calibration dataset requires two distinct ledger UUID pointers"
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_teacher_ai_disagreement (
|
|
disagreement_record_id, submission_id, content_hash, disagreement_id,
|
|
learner_id, cohort_id, case_ref, competency_id, ai_label, teacher_label,
|
|
ai_model, prompt_version, instrument_id, instrument_version,
|
|
correction_reason_code, ai_evidence_pointer_ids,
|
|
teacher_evidence_pointer_ids, created_by_uid
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
|
|
""",
|
|
disagreement_record_id,
|
|
submission_id,
|
|
content_hash,
|
|
disagreement.disagreement_id,
|
|
learner_id,
|
|
cohort_id,
|
|
disagreement.case_ref,
|
|
disagreement.competency_id,
|
|
disagreement.ai_label,
|
|
disagreement.teacher_label,
|
|
disagreement.ai_model,
|
|
disagreement.prompt_version,
|
|
disagreement.instrument_id,
|
|
disagreement.instrument_version,
|
|
disagreement.correction_reason_code,
|
|
ai_ids,
|
|
teacher_ids,
|
|
actor_uid,
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_calibration_dataset_row (
|
|
dataset_row_id, disagreement_record_id, learner_id, cohort_id,
|
|
row_hash, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6)
|
|
""",
|
|
dataset_row_id,
|
|
disagreement_record_id,
|
|
learner_id,
|
|
cohort_id,
|
|
dataset_row.row_id,
|
|
evidence_ids,
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO audit.supervision_teacher_event (
|
|
audit_event_id, disagreement_record_id, actor_uid, learner_id,
|
|
cohort_id, action, content_hash, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,'teacher_ai_disagreement.corrected',$6,$7)
|
|
""",
|
|
audit_event_id,
|
|
disagreement_record_id,
|
|
actor_uid,
|
|
learner_id,
|
|
cohort_id,
|
|
content_hash,
|
|
evidence_ids,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"disagreement_record_id": disagreement_record_id,
|
|
"dataset_row_hash": dataset_row.row_id,
|
|
"idempotent_replay": False,
|
|
"raw_transcript_included": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def append_curriculum_gap(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
gap_snapshot_id: UUID,
|
|
cohort_id: str,
|
|
competency_id: str,
|
|
gap_kind: str,
|
|
status: str,
|
|
uncertainty: float,
|
|
affected_learner_count: int,
|
|
evidence: Sequence[tuple[UUID, LedgerEvidencePointer]],
|
|
) -> dict[str, Any]:
|
|
payload = {
|
|
"gap_snapshot_id": str(gap_snapshot_id),
|
|
"cohort_id": cohort_id,
|
|
"competency_id": competency_id,
|
|
"gap_kind": gap_kind,
|
|
"status": status,
|
|
"uncertainty": uncertainty,
|
|
"affected_learner_count": affected_learner_count,
|
|
"evidence": [
|
|
{"learner_id": str(learner_id), **pointer.model_dump(mode="json")}
|
|
for learner_id, pointer in evidence
|
|
],
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 63))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_curriculum_gap_snapshot",
|
|
id_column="gap_snapshot_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"gap_snapshot_id": existing,
|
|
"idempotent_replay": True,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
if status == "insufficient_evidence":
|
|
if evidence or uncertainty != 1.0:
|
|
raise SupervisionResearchError(
|
|
"insufficient curriculum gap must remain evidence-free"
|
|
)
|
|
pointer_ids: list[UUID] = []
|
|
else:
|
|
if not evidence:
|
|
raise SupervisionResearchError("observed curriculum gap requires evidence")
|
|
pointer_ids = [
|
|
await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_id,
|
|
cohort_id=cohort_id,
|
|
consumer_view="supervisor",
|
|
pointer=pointer,
|
|
)
|
|
for learner_id, pointer in evidence
|
|
]
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_curriculum_gap_snapshot (
|
|
gap_snapshot_id, submission_id, content_hash, cohort_id, competency_id,
|
|
gap_kind, status, uncertainty, affected_learner_count, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
|
""",
|
|
gap_snapshot_id,
|
|
submission_id,
|
|
content_hash,
|
|
cohort_id,
|
|
competency_id,
|
|
gap_kind,
|
|
status,
|
|
uncertainty,
|
|
affected_learner_count,
|
|
list(dict.fromkeys(pointer_ids)),
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"gap_snapshot_id": gap_snapshot_id,
|
|
"idempotent_replay": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def _append_batch(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
batch_record_id: UUID,
|
|
cohort_id: str,
|
|
batch: EvaluationVersionBatch,
|
|
pointers_by_event_id: Mapping[str, LedgerEvidencePointer],
|
|
learner_ids_by_event_id: Mapping[str, UUID],
|
|
) -> tuple[UUID, bool, list[UUID]]:
|
|
payload = {
|
|
"batch_record_id": str(batch_record_id),
|
|
"cohort_id": cohort_id,
|
|
"batch": batch.model_dump(mode="json"),
|
|
"pointers": {
|
|
key: value.model_dump(mode="json")
|
|
for key, value in sorted(pointers_by_event_id.items())
|
|
},
|
|
"learner_ids": {
|
|
key: str(value) for key, value in sorted(learner_ids_by_event_id.items())
|
|
},
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 66))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_evaluation_batch",
|
|
id_column="batch_record_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
rows = await conn.fetch(
|
|
"SELECT evidence_pointer_id FROM app.supervision_evaluation_observation WHERE batch_record_id = $1",
|
|
existing,
|
|
)
|
|
return existing, True, [UUID(str(row["evidence_pointer_id"])) for row in rows]
|
|
event_ids = [item.evidence_event_id for item in batch.observations]
|
|
missing = set(event_ids) - set(pointers_by_event_id) | set(event_ids) - set(
|
|
learner_ids_by_event_id
|
|
)
|
|
if missing:
|
|
raise SupervisionResearchError(
|
|
f"evaluation evidence mapping is missing: {','.join(sorted(missing))}"
|
|
)
|
|
pointer_ids: list[UUID] = []
|
|
for item in batch.observations:
|
|
pointer = pointers_by_event_id[item.evidence_event_id]
|
|
if pointer.event_id != item.evidence_event_id:
|
|
raise SupervisionResearchError("evaluation evidence event id mismatch")
|
|
pointer_ids.append(
|
|
await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_ids_by_event_id[item.evidence_event_id],
|
|
cohort_id=cohort_id,
|
|
consumer_view="research",
|
|
pointer=pointer,
|
|
)
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_evaluation_batch (
|
|
batch_record_id, submission_id, content_hash, batch_id, cohort_id,
|
|
model_name, prompt_version, instrument_id, instrument_version,
|
|
observation_count
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
|
""",
|
|
batch_record_id,
|
|
submission_id,
|
|
content_hash,
|
|
batch.batch_id,
|
|
cohort_id,
|
|
batch.model,
|
|
batch.prompt_version,
|
|
batch.instrument_id,
|
|
batch.instrument_version,
|
|
len(batch.observations),
|
|
)
|
|
for index, item in enumerate(batch.observations):
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_evaluation_observation (
|
|
observation_record_id, batch_record_id, cohort_id, case_ref,
|
|
competency_id, synthetic_subgroup, gold_label, predicted_label,
|
|
evidence_pointer_id
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
|
""",
|
|
uuid5(batch_record_id, f"{item.case_ref}|{item.competency_id}"),
|
|
batch_record_id,
|
|
cohort_id,
|
|
item.case_ref,
|
|
item.competency_id,
|
|
item.synthetic_subgroup,
|
|
item.gold_label,
|
|
item.predicted_label,
|
|
pointer_ids[index],
|
|
)
|
|
return batch_record_id, False, pointer_ids
|
|
|
|
|
|
async def append_evaluation_comparison(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
drift_report_id: UUID,
|
|
baseline_submission_id: UUID,
|
|
baseline_batch_record_id: UUID,
|
|
candidate_submission_id: UUID,
|
|
candidate_batch_record_id: UUID,
|
|
cohort_id: str,
|
|
baseline: EvaluationVersionBatch,
|
|
candidate: EvaluationVersionBatch,
|
|
pointers_by_event_id: Mapping[str, LedgerEvidencePointer],
|
|
learner_ids_by_event_id: Mapping[str, UUID],
|
|
) -> dict[str, Any]:
|
|
report = compare_evaluation_versions(baseline, candidate)
|
|
payload = {
|
|
"drift_report_id": str(drift_report_id),
|
|
"cohort_id": cohort_id,
|
|
"baseline": baseline.model_dump(mode="json"),
|
|
"candidate": candidate.model_dump(mode="json"),
|
|
"report": report.model_dump(mode="json"),
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 64))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_drift_report",
|
|
id_column="drift_report_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"drift_report_id": existing,
|
|
"status": report.status,
|
|
"matched_count": report.matched_count,
|
|
"idempotent_replay": True,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
baseline_record_id, _, baseline_pointers = await _append_batch(
|
|
conn,
|
|
submission_id=baseline_submission_id,
|
|
batch_record_id=baseline_batch_record_id,
|
|
cohort_id=cohort_id,
|
|
batch=baseline,
|
|
pointers_by_event_id=pointers_by_event_id,
|
|
learner_ids_by_event_id=learner_ids_by_event_id,
|
|
)
|
|
candidate_record_id, _, candidate_pointers = await _append_batch(
|
|
conn,
|
|
submission_id=candidate_submission_id,
|
|
batch_record_id=candidate_batch_record_id,
|
|
cohort_id=cohort_id,
|
|
batch=candidate,
|
|
pointers_by_event_id=pointers_by_event_id,
|
|
learner_ids_by_event_id=learner_ids_by_event_id,
|
|
)
|
|
evidence_ids = list(dict.fromkeys((*baseline_pointers, *candidate_pointers)))
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_drift_report (
|
|
drift_report_id, submission_id, content_hash, cohort_id,
|
|
baseline_batch_record_id, candidate_batch_record_id, matched_count,
|
|
status, baseline_accuracy, candidate_accuracy, accuracy_delta,
|
|
disagreement_case_refs, alerts, evidence_pointer_ids
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)
|
|
""",
|
|
drift_report_id,
|
|
submission_id,
|
|
content_hash,
|
|
cohort_id,
|
|
baseline_record_id,
|
|
candidate_record_id,
|
|
report.matched_count,
|
|
report.status,
|
|
report.baseline_accuracy,
|
|
report.candidate_accuracy,
|
|
report.accuracy_delta,
|
|
list(report.disagreement_case_refs),
|
|
list(report.alerts),
|
|
evidence_ids,
|
|
)
|
|
for metric in report.subgroup_metrics:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_drift_subgroup_metric (
|
|
subgroup_metric_id, drift_report_id, cohort_id, subgroup,
|
|
matched_count, baseline_accuracy, candidate_accuracy, accuracy_delta
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
|
""",
|
|
uuid5(drift_report_id, metric.subgroup),
|
|
drift_report_id,
|
|
cohort_id,
|
|
metric.subgroup,
|
|
metric.matched_count,
|
|
metric.baseline_accuracy,
|
|
metric.candidate_accuracy,
|
|
metric.accuracy_delta,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"drift_report_id": drift_report_id,
|
|
"status": report.status,
|
|
"matched_count": report.matched_count,
|
|
"idempotent_replay": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def append_phase3_manifest(
|
|
conn: asyncpg.Connection,
|
|
*,
|
|
submission_id: UUID,
|
|
manifest_id: UUID,
|
|
cohort_id: str,
|
|
artifacts: Sequence[Phase3EvidenceArtifact],
|
|
source_by_domain: Mapping[str, tuple[UUID, LedgerEvidencePointer]],
|
|
) -> dict[str, Any]:
|
|
manifest = build_phase3_outcome_manifest(artifacts)
|
|
missing = {item.domain for item in manifest.artifacts} - set(source_by_domain)
|
|
if missing:
|
|
raise SupervisionResearchError(
|
|
f"manifest provenance mapping is missing: {','.join(sorted(missing))}"
|
|
)
|
|
payload = {
|
|
"manifest_id": str(manifest_id),
|
|
"cohort_id": cohort_id,
|
|
"manifest": manifest.model_dump(mode="json"),
|
|
"source_by_domain": {
|
|
key: {
|
|
"learner_id": str(value[0]),
|
|
"pointer": value[1].model_dump(mode="json"),
|
|
}
|
|
for key, value in sorted(source_by_domain.items())
|
|
},
|
|
}
|
|
content_hash = _canonical_hash(payload)
|
|
await conn.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended($1::text, 65))",
|
|
str(submission_id),
|
|
)
|
|
existing = await _existing_submission(
|
|
conn,
|
|
table="app.supervision_phase3_manifest",
|
|
id_column="manifest_id",
|
|
submission_id=submission_id,
|
|
content_hash=content_hash,
|
|
)
|
|
if existing is not None:
|
|
return {
|
|
"submission_id": submission_id,
|
|
"manifest_id": existing,
|
|
"artifact_count": 4,
|
|
"idempotent_replay": True,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
staged: list[tuple[Phase3EvidenceArtifact, UUID]] = []
|
|
for artifact in manifest.artifacts:
|
|
learner_id, pointer = source_by_domain[artifact.domain]
|
|
staged.append(
|
|
(
|
|
artifact,
|
|
await _ensure_pointer(
|
|
conn,
|
|
learner_id=learner_id,
|
|
cohort_id=cohort_id,
|
|
consumer_view="research",
|
|
pointer=pointer,
|
|
),
|
|
)
|
|
)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_phase3_manifest (
|
|
manifest_id, submission_id, content_hash, cohort_id,
|
|
schema_version, artifact_count
|
|
) VALUES ($1,$2,$3,$4,$5,4)
|
|
""",
|
|
manifest_id,
|
|
submission_id,
|
|
content_hash,
|
|
cohort_id,
|
|
manifest.schema_version,
|
|
)
|
|
for artifact, pointer_id in staged:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO app.supervision_phase3_artifact (
|
|
artifact_record_id, manifest_id, cohort_id, domain, artifact_id,
|
|
schema_version, content_sha256, record_count, provenance_uri,
|
|
source_pointer_id
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
|
""",
|
|
uuid5(manifest_id, artifact.domain),
|
|
manifest_id,
|
|
cohort_id,
|
|
artifact.domain,
|
|
artifact.artifact_id,
|
|
artifact.schema_version,
|
|
artifact.content_sha256,
|
|
artifact.record_count,
|
|
artifact.provenance_uri,
|
|
pointer_id,
|
|
)
|
|
return {
|
|
"submission_id": submission_id,
|
|
"manifest_id": manifest_id,
|
|
"artifact_count": 4,
|
|
"idempotent_replay": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def read_supervision_view(conn: asyncpg.Connection) -> dict[str, Any]:
|
|
items = await conn.fetch(
|
|
"""
|
|
SELECT item_id, snapshot_id, learner_id, learner_ref, cohort_id,
|
|
queue_position, primary_signal, oldest_active_sequence,
|
|
drilldown_routes, evidence_pointer_ids, created_at
|
|
FROM app.supervision_attention_item ORDER BY created_at DESC, queue_position
|
|
"""
|
|
)
|
|
gaps = await conn.fetch(
|
|
"""
|
|
SELECT gap_snapshot_id, cohort_id, competency_id, gap_kind, status,
|
|
uncertainty, affected_learner_count, evidence_pointer_ids, created_at
|
|
FROM app.supervision_curriculum_gap_snapshot ORDER BY created_at DESC
|
|
"""
|
|
)
|
|
return {
|
|
"attention_items": [dict(row) for row in items],
|
|
"curriculum_gaps": [dict(row) for row in gaps],
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
async def read_research_view(conn: asyncpg.Connection) -> dict[str, Any]:
|
|
datasets = await conn.fetch(
|
|
"""
|
|
SELECT d.dataset_row_id, d.row_hash, d.disagreement_record_id,
|
|
d.evidence_pointer_ids, d.raw_transcript_included, d.created_at,
|
|
x.case_ref, x.competency_id, x.ai_label, x.teacher_label,
|
|
x.ai_model, x.prompt_version, x.instrument_id,
|
|
x.instrument_version, x.correction_reason_code
|
|
FROM app.supervision_calibration_dataset_row d
|
|
JOIN app.supervision_teacher_ai_disagreement x
|
|
ON x.disagreement_record_id = d.disagreement_record_id
|
|
ORDER BY d.created_at DESC
|
|
"""
|
|
)
|
|
drift = await conn.fetch(
|
|
"""
|
|
SELECT r.drift_report_id, r.cohort_id, r.matched_count, r.status,
|
|
r.baseline_accuracy, r.candidate_accuracy, r.accuracy_delta,
|
|
r.disagreement_case_refs, r.alerts, r.evidence_pointer_ids,
|
|
r.created_at,
|
|
b.model_name AS baseline_model,
|
|
c.model_name AS candidate_model,
|
|
b.prompt_version AS baseline_prompt_version,
|
|
c.prompt_version AS candidate_prompt_version,
|
|
b.instrument_id,
|
|
b.instrument_version AS baseline_instrument_version,
|
|
c.instrument_version AS candidate_instrument_version
|
|
FROM app.supervision_drift_report r
|
|
JOIN app.supervision_evaluation_batch b
|
|
ON b.batch_record_id = r.baseline_batch_record_id
|
|
JOIN app.supervision_evaluation_batch c
|
|
ON c.batch_record_id = r.candidate_batch_record_id
|
|
ORDER BY r.created_at DESC
|
|
"""
|
|
)
|
|
subgroup_metrics = await conn.fetch(
|
|
"""
|
|
SELECT drift_report_id, subgroup, matched_count, baseline_accuracy,
|
|
candidate_accuracy, accuracy_delta
|
|
FROM app.supervision_drift_subgroup_metric
|
|
ORDER BY drift_report_id, subgroup
|
|
"""
|
|
)
|
|
manifests = await conn.fetch(
|
|
"""
|
|
SELECT manifest_id, cohort_id, schema_version, artifact_count, created_at
|
|
FROM app.supervision_phase3_manifest ORDER BY created_at DESC
|
|
"""
|
|
)
|
|
manifest_artifacts = await conn.fetch(
|
|
"""
|
|
SELECT manifest_id, domain, artifact_id, schema_version, content_sha256,
|
|
record_count, provenance_uri, clinical_claim_allowed
|
|
FROM app.supervision_phase3_artifact
|
|
ORDER BY manifest_id, domain
|
|
"""
|
|
)
|
|
subgroup_by_report: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in subgroup_metrics:
|
|
payload = dict(row)
|
|
report_id = payload.pop("drift_report_id")
|
|
subgroup_by_report.setdefault(report_id, []).append(payload)
|
|
drift_payloads: list[dict[str, Any]] = []
|
|
for row in drift:
|
|
payload = dict(row)
|
|
payload["subgroup_metrics"] = subgroup_by_report.get(
|
|
payload["drift_report_id"], []
|
|
)
|
|
drift_payloads.append(payload)
|
|
|
|
artifacts_by_manifest: dict[UUID, list[dict[str, Any]]] = {}
|
|
for row in manifest_artifacts:
|
|
payload = dict(row)
|
|
manifest_id = payload.pop("manifest_id")
|
|
artifacts_by_manifest.setdefault(manifest_id, []).append(payload)
|
|
manifest_payloads: list[dict[str, Any]] = []
|
|
for row in manifests:
|
|
payload = dict(row)
|
|
payload["artifacts"] = artifacts_by_manifest.get(payload["manifest_id"], [])
|
|
manifest_payloads.append(payload)
|
|
return {
|
|
"calibration_dataset": [dict(row) for row in datasets],
|
|
"drift_reports": drift_payloads,
|
|
"phase3_manifests": manifest_payloads,
|
|
"raw_transcript_included": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"SupervisionResearchConflictError",
|
|
"SupervisionResearchError",
|
|
"SupervisionResearchNotFoundError",
|
|
"append_attention_snapshot",
|
|
"append_curriculum_gap",
|
|
"append_evaluation_comparison",
|
|
"append_phase3_manifest",
|
|
"append_teacher_disagreement",
|
|
"read_research_view",
|
|
"read_supervision_view",
|
|
]
|