Try to fix background glitching over long periods of time

PROGRESS.md

@@ -44,7 +44,9 @@ cap the applied Opus hint at 30%, and feed it back over TLS.
 - Exercise iOS background/lock, interruption, Bluetooth, route-change, ReplayKit, and iOS 27
   ScreenCaptureKit paths on devices. The background/lock gate keeps a call active for 15+ minutes
   backgrounded and screen-locked with no periodic glitches and flat `VC_AUDIO` `feedDrops`/`starved`
-  counters (the render callback now paces the mix and the 20 ms capture handoff). Complete a
+  counters (the render callback now paces the mix and the 20 ms capture handoff, and a watchdog
+  rebuilds a graph that stops calling back). Take a Siri or phone-call interruption while
+  backgrounded and confirm audio resumes without foregrounding. Complete a
   30-minute iOS call and Wi-Fi/cellular switching with voice restoration, plus extended
   mono/stereo/voice-chat switching while joined. Verify Windows desktop/per-app stereo sharing.
 - Complete Developer ID signing/notarization. The iOS host and ReplayKit extension have been

clients/apple/VoiceCat.iOS/AppModel.cs

@@ -70,7 +70,7 @@ internal sealed class AppModel
     internal void WillEnterForeground()
     {
         backgrounded = false;
-        IosAudioRouter.Shared.Recover("foreground");
+        IosAudioRouter.Shared.ResumeForeground();
     }
 
     internal void DidBecomeActive()

clients/apple/VoiceCat.iOS/IosAudioEngine.cs

@@ -40,7 +40,12 @@ internal sealed class IosAudioEngine
     private bool inputProvided;
     private bool tapInstalled;
     private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames;
+    private long renderCallbacks, lastRenderTimestamp;
     internal bool IsConnected { get; private set; }
+    internal bool IsRunning => engine?.Running == true;
+    // The watchdog compares this across ticks: a graph that stops calling back while it still
+    // reports Running leaves the whole device-clocked pipeline frozen until it is rebuilt.
+    internal long RenderCallbacks => Interlocked.Read(ref renderCallbacks);
     internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; }
 
     internal void StartListening(VoiceCatClient owner)
@@ -201,15 +206,24 @@ internal sealed class IosAudioEngine
         // This callback is the cadence iOS keeps exact while the app is backgrounded or the device
         // is locked, so it owns both managed 20 ms hands-offs: capture into the sender and one mix
         // cycle per 20 ms of render demand. Both run allocation-free and without locks or I/O.
+        Interlocked.Increment(ref renderCallbacks);
+        long now = System.Diagnostics.Stopwatch.GetTimestamp(), previous = lastRenderTimestamp;
+        lastRenderTimestamp = now;
         VoiceCatClient? owner = Volatile.Read(ref client);
         if (owner is null || !IsConnected) microphoneCredit = 0;
         else
         {
+            if (previous != 0 && (now - previous) * 1000.0 / System.Diagnostics.Stopwatch.Frequency > 100)
+            {
+                Volatile.Read(ref microphone)?.Ring.Resynchronize();
+                playbackRing.Resynchronize(); owner.Audio.ResynchronizeInputs(); microphoneCredit = 0;
+            }
             microphoneCredit += frames;
             while (microphoneCredit >= 960) { microphoneCredit -= 960; PumpMicrophoneChunk(owner); }
             // Top the mix ring up to cover this callback plus its configured target so the read
-            // below never starves and the buffer keeps its chosen buffering latency.
-            int deficit = frames + playbackRing.TargetFrames - playbackRing.CountFrames;
+            // below never starves and the buffer keeps its chosen buffering latency. Catch-up is
+            // capped at one extra cycle so a refill cannot overrun this callback's deadline.
+            int deficit = Math.Min(frames + playbackRing.TargetFrames - playbackRing.CountFrames, frames + 960);
             for (int produced = 0; produced < deficit; produced += 960) owner.Audio.RunCycle();
         }
         Span<short> input = renderScratch.AsSpan(0, requested);
@@ -245,7 +259,7 @@ internal sealed class IosAudioEngine
             if (tapInstalled) old.InputNode.RemoveTapOnBus(0);
             old.Stop(); if (source is not null) old.DetachNode(source); old.Dispose();
         }
-        tapInstalled = false; pendingInput = null; inputProvider = null;
+        tapInstalled = false; pendingInput = null; inputProvider = null; lastRenderTimestamp = 0; microphoneCredit = 0;
         convertedMicrophone?.Dispose(); convertedMicrophone = null;
         microphoneConverter?.Dispose(); microphoneConverter = null;
         microphoneFormat?.Dispose(); microphoneFormat = null;

clients/apple/VoiceCat.iOS/IosAudioRouter.cs

@@ -18,6 +18,11 @@ internal sealed class IosAudioRouter
     internal static IosAudioRouter Shared { get; } = new();
     private readonly NSUserDefaults defaults = NSUserDefaults.StandardUserDefaults;
     private bool applying;
+    private System.Threading.Timer? watchdog;
+    private long lastRenderCallbacks = -1;
+    private int watchdogMisses;
+    private bool watchdogTicking;
+    private bool interrupted;
     internal event Action? Changed;
     internal IosAudioPreset Preset { get; private set; } = IosAudioPreset.VoiceChat;
     internal IosBluetoothMode BluetoothMode { get; private set; } = IosBluetoothMode.HfpVoice;
@@ -117,6 +122,7 @@ internal sealed class IosAudioRouter
                 $"preferred={session.PreferredInput?.PortName ?? "default"} dataSource={session.InputDataSource?.DataSourceName ?? "default"} " +
                 $"pattern={session.InputDataSource?.SelectedPolarPattern.ToString() ?? "default"}");
             session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes();
+            ResetWatchdog(); EnsureWatchdog();
         }
         finally { applying = false; }
     }
@@ -210,10 +216,14 @@ internal sealed class IosAudioRouter
     }
 
     private void Set(string key, string? value) { if (value is null) defaults.RemoveObject(key); else defaults.SetString(value, key); }
-    internal void Deactivate() => AVAudioSession.SharedInstance().SetActive(false, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out _);
+    internal void Deactivate()
+    {
+        watchdog?.Dispose(); watchdog = null; ResetWatchdog();
+        AVAudioSession.SharedInstance().SetActive(false, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out _);
+    }
     internal void EnsureAudio(string reason)
     {
-        if (!IosAudioEngine.Shared.IsConnected) return;
+        if (!IosAudioEngine.Shared.IsConnected || interrupted) return;
         try { IosAudioEngine.Shared.EnsureRunning(); }
         catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); }
     }
@@ -221,13 +231,47 @@ internal sealed class IosAudioRouter
     internal void Recover(string reason)
     {
         RefreshRoutes();
-        if (!IosAudioEngine.Shared.IsConnected) return;
+        if (!IosAudioEngine.Shared.IsConnected || interrupted) return;
         UIApplication.SharedApplication.BeginInvokeOnMainThread(() =>
         {
             try { IosAudioEngine.Shared.Reconfigure(); }
             catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); }
         });
     }
+
+    // The render callback is the only clock for the device-clocked pipeline, so a graph that
+    // stops while the app is backgrounded freezes capture, mix and send with no notification to
+    // recover from. Poll for that and rebuild; a failed rebuild is retried on the next tick.
+    private void EnsureWatchdog()
+    {
+        watchdog ??= new System.Threading.Timer(_ => UIApplication.SharedApplication.BeginInvokeOnMainThread(TickWatchdog),
+            null, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1));
+    }
+
+    private void ResetWatchdog() { lastRenderCallbacks = -1; watchdogMisses = 0; }
+
+    // An interruption that ends while the app is suspended never delivers its Ended notification.
+    internal void ResumeForeground() { interrupted = false; ResetWatchdog(); Recover("foreground"); }
+
+    private void TickWatchdog()
+    {
+        if (watchdogTicking) return;
+        watchdogTicking = true;
+        try
+        {
+            if (!IosAudioEngine.Shared.IsConnected || interrupted || applying) { ResetWatchdog(); return; }
+            long callbacks = IosAudioEngine.Shared.RenderCallbacks;
+            bool stalled = !IosAudioEngine.Shared.IsRunning || callbacks == lastRenderCallbacks;
+            lastRenderCallbacks = callbacks;
+            if (!stalled) { watchdogMisses = 0; return; }
+            // One missed tick can be a route change already rebuilding the graph.
+            if (++watchdogMisses < 2) return;
+            ResetWatchdog();
+            try { IosAudioEngine.Shared.Reconfigure(); }
+            catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio watchdog rebuild failed: {exception}"); }
+        }
+        finally { watchdogTicking = false; }
+    }
     private void HandleRouteChange(NSNotification note)
     {
         NSNumber? value = note.UserInfo?[new NSString("AVAudioSessionRouteChangeReasonKey")] as NSNumber;
@@ -243,6 +287,10 @@ internal sealed class IosAudioRouter
     {
         NSNumber? type = note.UserInfo?[new NSString("AVAudioSessionInterruptionTypeKey")] as NSNumber;
         AVAudioSessionInterruptionType interruption = (AVAudioSessionInterruptionType)(type?.UInt32Value ?? 0);
-        if (interruption == AVAudioSessionInterruptionType.Ended) Recover("interruption ended");
+        // The system stops the graph on Began and SetActive fails until the interruption clears,
+        // so suppress recovery until Ended and let the watchdog retry if that rebuild fails.
+        if (interruption == AVAudioSessionInterruptionType.Began) { interrupted = true; ResetWatchdog(); return; }
+        if (interruption != AVAudioSessionInterruptionType.Ended) return;
+        interrupted = false; ResetWatchdog(); Recover("interruption ended");
     }
 }

src/VoiceCat.Audio/AudioEngine.cs

@@ -177,6 +177,13 @@ public sealed class AudioEngine : IDisposable
         catch (Exception exception) { Failure = exception; stop.Cancel(); }
     }
 
+    // Drop each capture backlog to its buffer target. The consumer calls this after a stall it
+    // cannot catch up in place: one bounded gap instead of a queue that ratchets toward its edge.
+    public void ResynchronizeInputs()
+    {
+        foreach (LocalStream stream in Volatile.Read(ref routes).Local) stream.Input.Resynchronize();
+    }
+
     private void Work()
     {
         long deadline = Stopwatch.GetTimestamp();
@@ -189,12 +196,8 @@ public sealed class AudioEngine : IDisposable
                 WaitUntil(deadline);
                 if ((deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency < -100)
                 {
-                    // A stall this long cannot be caught up in place. Discarding only the schedule
-                    // deficit would leave the producer backlog queued at a fixed offset forever and
-                    // eventually overflow its ring, so drop back to the buffer targets instead: one
-                    // bounded gap per stall rather than growing latency and feed drops.
                     deadline = Stopwatch.GetTimestamp();
-                    foreach (LocalStream stream in Volatile.Read(ref routes).Local) stream.Input.Resynchronize();
+                    ResynchronizeInputs();
                 }
             }
         }

tests/VoiceCat.Tests/AudioEngineTests.cs

@@ -446,6 +446,22 @@ public class AudioEngineTests
         Assert.Contains("public void Resynchronize()", buffer);
     }
 
+    [Fact]
+    public void ResynchronizeInputsDropsStalledBacklogToTheBufferTarget()
+    {
+        using var send = new AudioEngine((_, _, _, _) => true, false);
+        send.DeviceBufferMilliseconds = 40; send.AddLocalStream(Stream());
+        short[] mono = Tone();
+        for (int i = 0; i < 40; i++) send.FeedPcm(1, mono, 1);
+        Assert.True(send.GetLocalDiagnostics(1).BufferedFrames > 1920 * 2);
+
+        send.ResynchronizeInputs();
+
+        Assert.Equal(1920, send.GetLocalDiagnostics(1).BufferedFrames);
+        send.ProcessCycle();
+        Assert.Equal(0, send.GetLocalDiagnostics(1).StarvedCycles);
+    }
+
     [Fact]
     public void RemotePlaybackSettingsAreIndependentForEachUserStream()
     {

tests/VoiceCat.Tests/PublishServerScriptTests.cs

@@ -164,6 +164,38 @@ public class PublishServerScriptTests
         Assert.Contains("deviceClockedAudio: true", model);
     }
 
+    [Fact]
+    public async Task IosRecoversBackgroundAudioWithoutForegrounding()
+    {
+        string root = FindRoot();
+        string microphone = await File.ReadAllTextAsync(Path.Combine(
+            root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs"));
+        string router = await File.ReadAllTextAsync(Path.Combine(
+            root, "clients", "apple", "VoiceCat.iOS", "IosAudioRouter.cs"));
+
+        // The render callback is the only clock for the device-clocked pipeline, so a graph that
+        // stops while backgrounded froze capture, mix and send until the app was foregrounded.
+        Assert.Contains("internal long RenderCallbacks", microphone);
+        Assert.Contains("internal bool IsRunning", microphone);
+        Assert.Contains("private void TickWatchdog()", router);
+        Assert.Contains("IosAudioEngine.Shared.RenderCallbacks", router);
+        Assert.Contains("IosAudioEngine.Shared.IsRunning", router);
+        Assert.Contains("IosAudioEngine.Shared.Reconfigure()", router);
+
+        // SetActive fails for as long as an interruption is in force, so Began suppresses
+        // recovery and the watchdog retries the rebuild that Ended asks for.
+        Assert.Contains("AVAudioSessionInterruptionType.Began", router);
+        Assert.Contains("interrupted = true", router);
+        Assert.Contains("internal void ResumeForeground()", router);
+
+        // A stalled render callback resynchronizes its own rings, and a refill is bounded so it
+        // cannot overrun the callback deadline that it is recovering from.
+        Assert.Contains("owner.Audio.ResynchronizeInputs()", microphone);
+        Assert.Contains("playbackRing.Resynchronize()", microphone);
+        Assert.Contains("Ring.Resynchronize()", microphone);
+        Assert.Contains("Math.Min(frames + playbackRing.TargetFrames - playbackRing.CountFrames, frames + 960)", microphone);
+    }
+
     [Fact]
     public async Task IosAudioCommitsInputRouteBeforeChannelsAndVoiceProcessingBeforeConnections()
     {