import { api, apiUrl, supplementalApi } from "../../lib/api"; import type { components } from "../../lib/api.gen"; type ApiSchema = components["schemas"][Name]; type WireMetadata = ApiSchema<"MultimodalSessionMetadataResponse">; type WireRawAudio = ApiSchema<"RawAudioAccessResponse">; export type MultimodalConsentRequest = ApiSchema<"MultimodalConsentRequest">; export type MultimodalConsentResponse = ApiSchema<"MultimodalConsentResponse">; export type MultimodalWithdrawalRequest = ApiSchema<"MultimodalWithdrawalRequest">; export type MultimodalDeletionRequest = ApiSchema<"MultimodalDeletionRequest">; export type MultimodalDeletionResponse = ApiSchema<"MultimodalDeletionRequestResponse">; export type AllianceAxis = "goal" | "task" | "bond"; export type AllianceModality = "text" | "voice"; export type MeasurementStatus = "ready" | "missing" | "error"; export type ConsentStatus = "granted" | "withdrawn" | "not_granted"; export type VoiceEventType = | "silence" | "overlap" | "interruption" | "prosody" | "pace" | "audio_quality"; export interface MultimodalConsentSnapshot { consent_snapshot_id: string; sequence_no: number; consent_status: ConsentStatus; retain_audio: boolean; retain_derived_features: boolean; transcript_retained: boolean; retention_days: number | null; policy_version: string; reason_code: string | null; created_at: string; } export interface MultimodalTimeline { timeline_id: string; audio_duration_ms: number; clock_version: string; word_count: number; event_count: number; created_at: string; derived_features_available: boolean; } export interface MultimodalWordTimestamp { timeline_id: string; word_index: number; start_ms: number; end_ms: number; speaker: "learner" | "client"; token_hash: string; } export interface MultimodalVoiceEvent { timeline_id: string; event_id: string; event_type: VoiceEventType; start_ms: number; end_ms: number; actor: "learner" | "client" | "both" | "channel"; observed_feature: string; uncertainty: number; source: "observed_audio_runtime" | "stt_word_timestamps"; claim_scope: "interaction_signal"; clinical_claim_allowed: false; } export interface MultimodalMeasurement { measurement_id: string; axis: AllianceAxis; modality: AllianceModality; status: MeasurementStatus; value: number | null; confidence: number | null; uncertainty: number; evidence_refs: string[]; model_run_id: string | null; instrument_id: string; instrument_version: string; model_name: string; prompt_version: string; source_kind: string; error_code: string | null; created_at: string; } export interface MultimodalFusionDecision { fusion_record_id: string; axis: AllianceAxis; status: MeasurementStatus; value: number | null; uncertainty: number; modalities_used: AllianceModality[]; measurement_ids: string[]; fusion_applied: boolean; calibration_id: string | null; benchmark_version: string | null; incremental_gain: number | null; counterevidence: string[]; created_at: string; } export interface MultimodalDeletionRecord { deletion_request_id: string; scopes: Array<"audio" | "derived_features">; request_reason: string; requested_at: string; completed_scopes: Array<"audio" | "derived_features">; } export interface RawAudioAsset { audio_asset_id: string; session_id: string; media_type: string; byte_size: number; duration_ms: number; retained_until: string; created_at: string; } export interface MultimodalAllianceReadModel { session_id: string; learner_id: string; clinical_claim_allowed: false; consent_snapshots: MultimodalConsentSnapshot[]; timelines: MultimodalTimeline[]; word_timestamps: MultimodalWordTimestamp[]; voice_events: MultimodalVoiceEvent[]; measurements: MultimodalMeasurement[]; fusion_decisions: MultimodalFusionDecision[]; deletion_requests: MultimodalDeletionRecord[]; } type Row = Record; function objectRows(value: unknown): Row[] { if (!Array.isArray(value)) return []; return value.filter( (item): item is Row => Boolean(item) && typeof item === "object", ); } function text(row: Row, key: string, fallback = ""): string { const value = row[key]; return typeof value === "string" ? value : fallback; } function optionalText(row: Row, key: string): string | null { const value = row[key]; return typeof value === "string" && value.length > 0 ? value : null; } function numberValue(row: Row, key: string, fallback = 0): number { const value = row[key]; if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim() !== "") { const parsed = Number(value); if (Number.isFinite(parsed)) return parsed; } return fallback; } function optionalNumber(row: Row, key: string): number | null { const value = row[key]; if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim() !== "") { const parsed = Number(value); if (Number.isFinite(parsed)) return parsed; } return null; } function bool(row: Row, key: string): boolean { return row[key] === true; } function stringArray(row: Row, key: string): string[] { const value = row[key]; return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } function isAllianceAxis(value: string): value is AllianceAxis { return value === "goal" || value === "task" || value === "bond"; } function isModality(value: string): value is AllianceModality { return value === "text" || value === "voice"; } function measurementStatus(value: string): MeasurementStatus { return value === "ready" || value === "error" ? value : "missing"; } function consentStatus(value: string): ConsentStatus { if (value === "granted" || value === "withdrawn") return value; return "not_granted"; } function voiceEventType(value: string): VoiceEventType { const allowed: VoiceEventType[] = [ "silence", "overlap", "interruption", "prosody", "pace", "audio_quality", ]; return allowed.includes(value as VoiceEventType) ? (value as VoiceEventType) : "audio_quality"; } function normalizeMetadata(payload: WireMetadata): MultimodalAllianceReadModel { const consentSnapshots = objectRows(payload.consent_snapshots).map((row) => ({ consent_snapshot_id: text(row, "consent_snapshot_id"), sequence_no: numberValue(row, "sequence_no"), consent_status: consentStatus(text(row, "consent_status")), retain_audio: bool(row, "retain_audio"), retain_derived_features: bool(row, "retain_derived_features"), transcript_retained: bool(row, "transcript_retained"), retention_days: optionalNumber(row, "retention_days"), policy_version: text(row, "policy_version"), reason_code: optionalText(row, "reason_code"), created_at: text(row, "created_at"), })); const timelines = objectRows(payload.timelines).map((row) => ({ timeline_id: text(row, "timeline_id"), audio_duration_ms: numberValue(row, "audio_duration_ms"), clock_version: text(row, "clock_version"), word_count: numberValue(row, "word_count"), event_count: numberValue(row, "event_count"), created_at: text(row, "created_at"), derived_features_available: bool(row, "derived_features_available"), })); const words = objectRows(payload.word_timestamps) .map((row) => ({ timeline_id: text(row, "timeline_id"), word_index: numberValue(row, "word_index"), start_ms: numberValue(row, "start_ms"), end_ms: numberValue(row, "end_ms"), speaker: text(row, "speaker") === "learner" ? "learner" as const : "client" as const, token_hash: text(row, "token_hash"), })) .sort((a, b) => a.word_index - b.word_index); const events = objectRows(payload.voice_events).map((row) => ({ timeline_id: text(row, "timeline_id"), event_id: text(row, "event_id"), event_type: voiceEventType(text(row, "event_type")), start_ms: numberValue(row, "start_ms"), end_ms: numberValue(row, "end_ms"), actor: (["learner", "client", "both", "channel"].includes(text(row, "actor")) ? text(row, "actor") : "channel") as MultimodalVoiceEvent["actor"], observed_feature: text(row, "observed_feature"), uncertainty: numberValue(row, "uncertainty", 1), source: text(row, "source") === "stt_word_timestamps" ? "stt_word_timestamps" as const : "observed_audio_runtime" as const, claim_scope: "interaction_signal" as const, clinical_claim_allowed: false as const, })); const measurements = objectRows(payload.measurements).flatMap((row) => { const axis = text(row, "axis"); const modality = text(row, "modality"); if (!isAllianceAxis(axis) || !isModality(modality)) return []; return [{ measurement_id: text(row, "measurement_id"), axis, modality, status: measurementStatus(text(row, "status")), value: optionalNumber(row, "value"), confidence: optionalNumber(row, "confidence"), uncertainty: numberValue(row, "uncertainty", 1), evidence_refs: stringArray(row, "evidence_refs"), model_run_id: optionalText(row, "model_run_id"), instrument_id: text(row, "instrument_id"), instrument_version: text(row, "instrument_version"), model_name: text(row, "model_name"), prompt_version: text(row, "prompt_version"), source_kind: text(row, "source_kind"), error_code: optionalText(row, "error_code"), created_at: text(row, "created_at"), }]; }); const fusions = objectRows(payload.fusion_decisions).flatMap((row) => { const axis = text(row, "axis"); if (!isAllianceAxis(axis)) return []; return [{ fusion_record_id: text(row, "fusion_record_id"), axis, status: measurementStatus(text(row, "status")), value: optionalNumber(row, "value"), uncertainty: numberValue(row, "uncertainty", 1), modalities_used: stringArray(row, "modalities_used").filter(isModality), measurement_ids: stringArray(row, "measurement_ids"), fusion_applied: bool(row, "fusion_applied"), calibration_id: optionalText(row, "calibration_id"), benchmark_version: optionalText(row, "benchmark_version"), incremental_gain: optionalNumber(row, "incremental_gain"), counterevidence: stringArray(row, "counterevidence"), created_at: text(row, "created_at"), }]; }); const deletionRequests = objectRows(payload.deletion_requests).map((row) => ({ deletion_request_id: text(row, "deletion_request_id"), scopes: stringArray(row, "scopes").filter( (scope): scope is "audio" | "derived_features" => scope === "audio" || scope === "derived_features", ), request_reason: text(row, "request_reason"), requested_at: text(row, "requested_at"), completed_scopes: stringArray(row, "completed_scopes").filter( (scope): scope is "audio" | "derived_features" => scope === "audio" || scope === "derived_features", ), })); return { session_id: payload.session_id, learner_id: payload.learner_id, clinical_claim_allowed: false, consent_snapshots: consentSnapshots, timelines, word_timestamps: words, voice_events: events, measurements, fusion_decisions: fusions, deletion_requests: deletionRequests, }; } function normalizeRawAudio(payload: WireRawAudio): RawAudioAsset[] { return objectRows(payload.items).map((row) => ({ audio_asset_id: text(row, "audio_asset_id"), session_id: text(row, "session_id"), media_type: text(row, "media_type"), byte_size: numberValue(row, "byte_size"), duration_ms: numberValue(row, "duration_ms"), retained_until: text(row, "retained_until"), created_at: text(row, "created_at"), })); } function basePath(sessionId: string): string { return `/sessions/${encodeURIComponent(sessionId)}/multimodal-alliance`; } export function rawAudioPlaybackUrl( sessionId: string, audioAssetId: string, ): string { return apiUrl( `${basePath(sessionId)}/raw-audio/${encodeURIComponent(audioAssetId)}`, ); } export const multimodalAllianceApi = { getMetadata: (sessionId: string, signal?: AbortSignal) => supplementalApi .get(basePath(sessionId), { signal }) .then(normalizeMetadata), getRawAudio: (sessionId: string, signal?: AbortSignal) => supplementalApi .get(`${basePath(sessionId)}/raw-audio`, { signal }) .then(normalizeRawAudio), saveConsent: (sessionId: string, body: MultimodalConsentRequest) => api.post(`${basePath(sessionId)}/consent`, body), withdraw: (sessionId: string, body: MultimodalWithdrawalRequest) => api.post(`${basePath(sessionId)}/withdraw`, body), requestDeletion: (sessionId: string, body: MultimodalDeletionRequest) => api.post( `${basePath(sessionId)}/deletion-requests`, body, ), };