fix(mobile-queue): arm retries at the earliest deadline and stop losing or replaying queued audio

apps/mobile-rn/src/features/recording/durable-processing-queue.ts

@@ -14,6 +14,8 @@ import {
   queueMeetingRecording,
 } from '../meetings/meetings-service';
 import { createUuidV4 } from '../../lib/random-id';
+import { createQueueRepository, type QueueRepository } from './durable-queue-repository';
+import { createRetryScheduler, type RetryScheduler } from './retry-scheduler';
 
 const STORAGE_KEY = '@d3ro/audio-processing-queue/v1';
 const QUEUE_DIRECTORY = `${Dirs.DocumentDir}/d3ro-processing-queue`;
@@ -64,9 +66,6 @@ export class QueuedAudioPendingError extends Error {
 }
 
 const listeners = new Set<(event: DurableQueueEvent) => void>();
-let storageLock: Promise<void> = Promise.resolve();
-const runningUsers = new Set<string>();
-const scheduledRetries = new Map<string, ReturnType<typeof setTimeout>>();
 const runningControllers = new Map<string, AbortController>();
 const runningItemPromises = new Map<string, Promise<AudioPipelineResult>>();
 const discardedItems = new Set<string>();
@@ -131,24 +130,28 @@ function parseQueue(raw: string | null): DurableQueueItem[] {
   });
 }
 
-async function readQueue(): Promise<DurableQueueItem[]> {
-  await storageLock;
-  return parseQueue(await AsyncStorage.getItem(STORAGE_KEY));
+// Composition root: the queue logic below depends only on these two ports.
+const repository: QueueRepository<DurableQueueItem> = createQueueRepository({
+  storage: AsyncStorage,
+  key: STORAGE_KEY,
+  parse: parseQueue,
+});
+
+const scheduler: RetryScheduler = createRetryScheduler({
+  now: () => Date.now(),
+  setTimer: (callback, delayMs) => setTimeout(callback, delayMs),
+  clearTimer: handle => clearTimeout(handle),
+  run: runResumePass,
+});
+
+function readQueue(): Promise<DurableQueueItem[]> {
+  return repository.read();
 }
 
-async function mutateQueue(
+function mutateQueue(
   mutation: (items: DurableQueueItem[]) => DurableQueueItem[],
 ): Promise<DurableQueueItem[]> {
-  let result: DurableQueueItem[] = [];
-  const operation = storageLock.then(async () => {
-    const current = parseQueue(await AsyncStorage.getItem(STORAGE_KEY));
-    result = mutation(current);
-    if (result.length === 0) await AsyncStorage.removeItem(STORAGE_KEY);
-    else await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(result));
-  });
-  storageLock = operation.catch(() => undefined);
-  await operation;
-  return result;
+  return repository.mutate(mutation);
 }
 
 async function ensureQueueDirectory(): Promise<void> {
@@ -375,7 +378,7 @@ async function processItemCore(item: DurableQueueItem): Promise<AudioPipelineRes
     };
     await replaceItem(current);
     await reportMeetingFailure(current, error, !canRetry);
-    if (canRetry) scheduleRetry(current.userId, current.nextAttemptAtMs);
+    if (canRetry) scheduler.arm(current.userId, current.nextAttemptAtMs);
     throw canRetry ? new QueuedAudioPendingError(current) : error;
   } finally {
     runningControllers.delete(item.id);
@@ -394,16 +397,6 @@ function processItem(item: DurableQueueItem): Promise<AudioPipelineResult> {
   return operation;
 }
 
-function scheduleRetry(userId: string, atMs: number): void {
-  const existing = scheduledRetries.get(userId);
-  if (existing !== undefined) clearTimeout(existing);
-  const timer = setTimeout(() => {
-    scheduledRetries.delete(userId);
-    void resumeQueuedAudioForUser(userId).catch(() => undefined);
-  }, Math.max(0, atMs - Date.now()));
-  scheduledRetries.set(userId, timer);
-}
-
 export async function processQueuedAudioNow(
   userId: string,
   itemId: string,
@@ -413,39 +406,52 @@ export async function processQueuedAudioNow(
   return processItem(item);
 }
 
-export async function resumeQueuedAudioForUser(userId: string): Promise<void> {
-  if (runningUsers.has(userId)) return;
-  runningUsers.add(userId);
-  try {
-    const items = (await readQueue())
-      .filter(item => item.userId === userId)
-      .sort((left, right) => left.createdAtMs - right.createdAtMs);
-    for (const item of items) {
-      if (item.status === 'terminal') continue;
-      if (item.status === 'cleanup') {
-        try {
-          await removeOwnedFile(item.path);
-          await removeItem(item);
-        } catch {
-          scheduleRetry(userId, Date.now() + 60_000);
-        }
-        continue;
-      }
-      if (item.nextAttemptAtMs > Date.now()) {
-        scheduleRetry(userId, item.nextAttemptAtMs);
-        continue;
-      }
+/**
+ * One pass over the user's queue. The listing only fixes the visiting order;
+ * every item is re-read right before acting on it, so work done meanwhile by
+ * processQueuedAudioNow, retryQueuedAudio or discardQueuedAudio (new status,
+ * attempts, nextAttemptAtMs, or removal) is respected instead of being
+ * overwritten from a stale snapshot.
+ */
+async function runResumePass(userId: string): Promise<void> {
+  const order = (await readQueue())
+    .filter(item => item.userId === userId)
+    .sort((left, right) => left.createdAtMs - right.createdAtMs)
+    .map(item => item.id);
+  for (const itemId of order) {
+    const item = await repository.get(itemId);
+    if (item === undefined || item.userId !== userId) continue;
+    if (item.status === 'terminal') continue;
+    if (item.status === 'cleanup') {
       try {
-        await processItem(item);
+        await removeOwnedFile(item.path);
+        await removeItem(item);
       } catch {
-        // The item itself holds the truthful retry/terminal state.
+        scheduler.arm(userId, Date.now() + 60_000);
       }
+      continue;
+    }
+    if (item.nextAttemptAtMs > Date.now()) {
+      scheduler.arm(userId, item.nextAttemptAtMs);
+      continue;
+    }
+    try {
+      await processItem(item);
+    } catch {
+      // The item itself holds the truthful retry/terminal state.
     }
-  } finally {
-    runningUsers.delete(userId);
   }
 }
 
+/**
+ * Processes every due item of the user and arms a timer for the earliest
+ * pending retry. A call while a pass is already running requests one more
+ * pass after it, and the returned promise settles when that pass is done.
+ */
+export function resumeQueuedAudioForUser(userId: string): Promise<void> {
+  return scheduler.requestRun(userId);
+}
+
 export async function listQueuedAudio(userId: string): Promise<DurableQueueItem[]> {
   return (await readQueue()).filter(item => item.userId === userId);
 }
@@ -476,9 +482,7 @@ export async function discardQueuedAudio(userId: string, itemId: string): Promis
 }
 
 export async function clearQueuedAudioForUser(userId: string): Promise<void> {
-  const timer = scheduledRetries.get(userId);
-  if (timer !== undefined) clearTimeout(timer);
-  scheduledRetries.delete(userId);
+  scheduler.cancel(userId);
   const owned = (await readQueue()).filter(item => item.userId === userId);
   const removedIds = new Set<string>();
   let firstRemovalError: unknown = null;

apps/mobile-rn/src/features/recording/durable-queue-repository.ts

@@ -0,0 +1,59 @@
+/**
+ * Persistence port for the durable audio processing queue.
+ *
+ * The queue itself only needs "read everything", "atomically rewrite" and
+ * "look one item up"; how the rows are stored (AsyncStorage JSON under one key)
+ * and how writes are serialized (a promise-chain lock) live here.
+ */
+
+export interface KeyValueStorage {
+  getItem(key: string): Promise<string | null>;
+  setItem(key: string, value: string): Promise<void>;
+  removeItem(key: string): Promise<void>;
+}
+
+export interface QueueRepository<T extends { id: string }> {
+  /** Reads all rows after any in-flight write has settled. */
+  read(): Promise<T[]>;
+  /** Serialized read-modify-write. An empty result removes the storage key. */
+  mutate(mutation: (items: T[]) => T[]): Promise<T[]>;
+  /** Reads the current version of a single row, or undefined when it is gone. */
+  get(itemId: string): Promise<T | undefined>;
+}
+
+export interface QueueRepositoryOptions<T> {
+  storage: KeyValueStorage;
+  key: string;
+  parse: (raw: string | null) => T[];
+}
+
+export function createQueueRepository<T extends { id: string }>(
+  options: QueueRepositoryOptions<T>,
+): QueueRepository<T> {
+  const { storage, key, parse } = options;
+  let storageLock: Promise<void> = Promise.resolve();
+
+  async function read(): Promise<T[]> {
+    await storageLock;
+    return parse(await storage.getItem(key));
+  }
+
+  async function mutate(mutation: (items: T[]) => T[]): Promise<T[]> {
+    let result: T[] = [];
+    const operation = storageLock.then(async () => {
+      const current = parse(await storage.getItem(key));
+      result = mutation(current);
+      if (result.length === 0) await storage.removeItem(key);
+      else await storage.setItem(key, JSON.stringify(result));
+    });
+    storageLock = operation.catch(() => undefined);
+    await operation;
+    return result;
+  }
+
+  async function get(itemId: string): Promise<T | undefined> {
+    return (await read()).find(item => item.id === itemId);
+  }
+
+  return { read, mutate, get };
+}

apps/mobile-rn/src/features/recording/retry-scheduler.ts

@@ -0,0 +1,118 @@
+/**
+ * Per-key retry scheduling policy for the durable audio processing queue.
+ *
+ * - `arm(key, atMs)` keeps ONE timer per key, always at the earliest pending
+ *   deadline. A later deadline never replaces an earlier one, so an item that
+ *   is due soon is not starved by an item that is due much later.
+ * - `requestRun(key)` coalesces re-entrant triggers: while a pass is running,
+ *   further requests (a timer firing, a manual retry, a foreground transition)
+ *   mark the key dirty and one more pass runs after the current one. The
+ *   returned promise settles when the whole run, including that follow-up
+ *   pass, has finished.
+ *
+ * Clock and timer functions are injected so the policy is testable without
+ * real time.
+ */
+
+export interface RetrySchedulerDeps<TimerHandle> {
+  now: () => number;
+  setTimer: (callback: () => void, delayMs: number) => TimerHandle;
+  clearTimer: (handle: TimerHandle) => void;
+  /** One processing pass for the key. */
+  run: (key: string) => Promise<void>;
+}
+
+export interface RetryScheduler {
+  arm(key: string, atMs: number): void;
+  cancel(key: string): void;
+  requestRun(key: string): Promise<void>;
+  /** Earliest armed deadline for the key, if any. */
+  nextDeadline(key: string): number | undefined;
+  isRunning(key: string): boolean;
+}
+
+interface ArmedTimer<TimerHandle> {
+  atMs: number;
+  handle: TimerHandle;
+}
+
+interface ActiveRun {
+  rerunRequested: boolean;
+  done: Promise<void>;
+}
+
+export function createRetryScheduler<TimerHandle>(
+  deps: RetrySchedulerDeps<TimerHandle>,
+): RetryScheduler {
+  const armed = new Map<string, ArmedTimer<TimerHandle>>();
+  const active = new Map<string, ActiveRun>();
+
+  function arm(key: string, atMs: number): void {
+    const existing = armed.get(key);
+    if (existing !== undefined) {
+      if (existing.atMs <= atMs) return;
+      deps.clearTimer(existing.handle);
+    }
+    const entry: ArmedTimer<TimerHandle> = {
+      atMs,
+      handle: deps.setTimer(() => {
+        if (armed.get(key) === entry) armed.delete(key);
+        void requestRun(key).catch(() => undefined);
+      }, Math.max(0, atMs - deps.now())),
+    };
+    armed.set(key, entry);
+  }
+
+  function cancel(key: string): void {
+    const existing = armed.get(key);
+    if (existing !== undefined) deps.clearTimer(existing.handle);
+    armed.delete(key);
+    const current = active.get(key);
+    if (current !== undefined) current.rerunRequested = false;
+  }
+
+  async function loop(key: string, state: ActiveRun): Promise<void> {
+    try {
+      let failed = false;
+      let failure: unknown = null;
+      do {
+        state.rerunRequested = false;
+        try {
+          await deps.run(key);
+          failed = false;
+        } catch (error) {
+          failed = true;
+          failure = error;
+        }
+      } while (state.rerunRequested);
+      if (failed) throw failure;
+    } finally {
+      if (active.get(key) === state) active.delete(key);
+    }
+  }
+
+  function requestRun(key: string): Promise<void> {
+    const current = active.get(key);
+    if (current !== undefined) {
+      current.rerunRequested = true;
+      return current.done;
+    }
+    const state: ActiveRun = { rerunRequested: false, done: Promise.resolve() };
+    // Register before the first pass starts, and start it on a microtask, so a
+    // re-entrant request from inside a pass joins this run (and its promise)
+    // instead of starting a parallel one.
+    active.set(key, state);
+    state.done = Promise.resolve().then(() => loop(key, state));
+    return state.done;
+  }
+
+  function nextDeadline(key: string): number | undefined {
+    return armed.get(key)?.atMs;
+  }
+
+  function isRunning(key: string): boolean {
+    return active.has(key);
+  }
+
+  return { arm, cancel, requestRun, nextDeadline, isRunning };
+}