From fb740bcfb26d6d380c6da42dce53ed4436e87af0 Mon Sep 17 00:00:00 2001 From: Talon Date: Thu, 24 Sep 2026 13:31:16 +0200 Subject: [PATCH] Try to fix background glitching over long periods of time --- PROGRESS.md | 4 +- clients/apple/VoiceCat.iOS/AppModel.cs | 2 +- clients/apple/VoiceCat.iOS/IosAudioEngine.cs | 20 ++++++- clients/apple/VoiceCat.iOS/IosAudioRouter.cs | 56 +++++++++++++++++-- src/VoiceCat.Audio/AudioEngine.cs | 13 +++-- tests/VoiceCat.Tests/AudioEngineTests.cs | 16 ++++++ .../PublishServerScriptTests.cs | 32 +++++++++++ 7 files changed, 129 insertions(+), 14 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 226fe95..7f37c7c 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -44,7 +44,9 @@ cap the applied Opus hint at 30%, and feed it back over TLS. - Exercise iOS background/lock, interruption, Bluetooth, route-change, ReplayKit, and iOS 27 ScreenCaptureKit paths on devices. The background/lock gate keeps a call active for 15+ minutes backgrounded and screen-locked with no periodic glitches and flat `VC_AUDIO` `feedDrops`/`starved` - counters (the render callback now paces the mix and the 20 ms capture handoff). Complete a + counters (the render callback now paces the mix and the 20 ms capture handoff, and a watchdog + rebuilds a graph that stops calling back). Take a Siri or phone-call interruption while + backgrounded and confirm audio resumes without foregrounding. Complete a 30-minute iOS call and Wi-Fi/cellular switching with voice restoration, plus extended mono/stereo/voice-chat switching while joined. Verify Windows desktop/per-app stereo sharing. - Complete Developer ID signing/notarization. The iOS host and ReplayKit extension have been diff --git a/clients/apple/VoiceCat.iOS/AppModel.cs b/clients/apple/VoiceCat.iOS/AppModel.cs index d04cea6..44d7f38 100644 --- a/clients/apple/VoiceCat.iOS/AppModel.cs +++ b/clients/apple/VoiceCat.iOS/AppModel.cs @@ -70,7 +70,7 @@ internal sealed class AppModel internal void WillEnterForeground() { backgrounded = false; - IosAudioRouter.Shared.Recover("foreground"); + IosAudioRouter.Shared.ResumeForeground(); } internal void DidBecomeActive() diff --git a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs index 5c64cff..bcc37e0 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs @@ -40,7 +40,12 @@ internal sealed class IosAudioEngine private bool inputProvided; private bool tapInstalled; private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames; + private long renderCallbacks, lastRenderTimestamp; internal bool IsConnected { get; private set; } + internal bool IsRunning => engine?.Running == true; + // The watchdog compares this across ticks: a graph that stops calling back while it still + // reports Running leaves the whole device-clocked pipeline frozen until it is rebuilt. + internal long RenderCallbacks => Interlocked.Read(ref renderCallbacks); internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; } internal void StartListening(VoiceCatClient owner) @@ -201,15 +206,24 @@ internal sealed class IosAudioEngine // This callback is the cadence iOS keeps exact while the app is backgrounded or the device // is locked, so it owns both managed 20 ms hands-offs: capture into the sender and one mix // cycle per 20 ms of render demand. Both run allocation-free and without locks or I/O. + Interlocked.Increment(ref renderCallbacks); + long now = System.Diagnostics.Stopwatch.GetTimestamp(), previous = lastRenderTimestamp; + lastRenderTimestamp = now; VoiceCatClient? owner = Volatile.Read(ref client); if (owner is null || !IsConnected) microphoneCredit = 0; else { + if (previous != 0 && (now - previous) * 1000.0 / System.Diagnostics.Stopwatch.Frequency > 100) + { + Volatile.Read(ref microphone)?.Ring.Resynchronize(); + playbackRing.Resynchronize(); owner.Audio.ResynchronizeInputs(); microphoneCredit = 0; + } microphoneCredit += frames; while (microphoneCredit >= 960) { microphoneCredit -= 960; PumpMicrophoneChunk(owner); } // Top the mix ring up to cover this callback plus its configured target so the read - // below never starves and the buffer keeps its chosen buffering latency. - int deficit = frames + playbackRing.TargetFrames - playbackRing.CountFrames; + // below never starves and the buffer keeps its chosen buffering latency. Catch-up is + // capped at one extra cycle so a refill cannot overrun this callback's deadline. + int deficit = Math.Min(frames + playbackRing.TargetFrames - playbackRing.CountFrames, frames + 960); for (int produced = 0; produced < deficit; produced += 960) owner.Audio.RunCycle(); } Span input = renderScratch.AsSpan(0, requested); @@ -245,7 +259,7 @@ internal sealed class IosAudioEngine if (tapInstalled) old.InputNode.RemoveTapOnBus(0); old.Stop(); if (source is not null) old.DetachNode(source); old.Dispose(); } - tapInstalled = false; pendingInput = null; inputProvider = null; + tapInstalled = false; pendingInput = null; inputProvider = null; lastRenderTimestamp = 0; microphoneCredit = 0; convertedMicrophone?.Dispose(); convertedMicrophone = null; microphoneConverter?.Dispose(); microphoneConverter = null; microphoneFormat?.Dispose(); microphoneFormat = null; diff --git a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs index 3a59006..6200171 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs @@ -18,6 +18,11 @@ internal sealed class IosAudioRouter internal static IosAudioRouter Shared { get; } = new(); private readonly NSUserDefaults defaults = NSUserDefaults.StandardUserDefaults; private bool applying; + private System.Threading.Timer? watchdog; + private long lastRenderCallbacks = -1; + private int watchdogMisses; + private bool watchdogTicking; + private bool interrupted; internal event Action? Changed; internal IosAudioPreset Preset { get; private set; } = IosAudioPreset.VoiceChat; internal IosBluetoothMode BluetoothMode { get; private set; } = IosBluetoothMode.HfpVoice; @@ -117,6 +122,7 @@ internal sealed class IosAudioRouter $"preferred={session.PreferredInput?.PortName ?? "default"} dataSource={session.InputDataSource?.DataSourceName ?? "default"} " + $"pattern={session.InputDataSource?.SelectedPolarPattern.ToString() ?? "default"}"); session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes(); + ResetWatchdog(); EnsureWatchdog(); } finally { applying = false; } } @@ -210,10 +216,14 @@ internal sealed class IosAudioRouter } private void Set(string key, string? value) { if (value is null) defaults.RemoveObject(key); else defaults.SetString(value, key); } - internal void Deactivate() => AVAudioSession.SharedInstance().SetActive(false, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out _); + internal void Deactivate() + { + watchdog?.Dispose(); watchdog = null; ResetWatchdog(); + AVAudioSession.SharedInstance().SetActive(false, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out _); + } internal void EnsureAudio(string reason) { - if (!IosAudioEngine.Shared.IsConnected) return; + if (!IosAudioEngine.Shared.IsConnected || interrupted) return; try { IosAudioEngine.Shared.EnsureRunning(); } catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); } } @@ -221,13 +231,47 @@ internal sealed class IosAudioRouter internal void Recover(string reason) { RefreshRoutes(); - if (!IosAudioEngine.Shared.IsConnected) return; + if (!IosAudioEngine.Shared.IsConnected || interrupted) return; UIApplication.SharedApplication.BeginInvokeOnMainThread(() => { try { IosAudioEngine.Shared.Reconfigure(); } catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); } }); } + + // The render callback is the only clock for the device-clocked pipeline, so a graph that + // stops while the app is backgrounded freezes capture, mix and send with no notification to + // recover from. Poll for that and rebuild; a failed rebuild is retried on the next tick. + private void EnsureWatchdog() + { + watchdog ??= new System.Threading.Timer(_ => UIApplication.SharedApplication.BeginInvokeOnMainThread(TickWatchdog), + null, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + } + + private void ResetWatchdog() { lastRenderCallbacks = -1; watchdogMisses = 0; } + + // An interruption that ends while the app is suspended never delivers its Ended notification. + internal void ResumeForeground() { interrupted = false; ResetWatchdog(); Recover("foreground"); } + + private void TickWatchdog() + { + if (watchdogTicking) return; + watchdogTicking = true; + try + { + if (!IosAudioEngine.Shared.IsConnected || interrupted || applying) { ResetWatchdog(); return; } + long callbacks = IosAudioEngine.Shared.RenderCallbacks; + bool stalled = !IosAudioEngine.Shared.IsRunning || callbacks == lastRenderCallbacks; + lastRenderCallbacks = callbacks; + if (!stalled) { watchdogMisses = 0; return; } + // One missed tick can be a route change already rebuilding the graph. + if (++watchdogMisses < 2) return; + ResetWatchdog(); + try { IosAudioEngine.Shared.Reconfigure(); } + catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio watchdog rebuild failed: {exception}"); } + } + finally { watchdogTicking = false; } + } private void HandleRouteChange(NSNotification note) { NSNumber? value = note.UserInfo?[new NSString("AVAudioSessionRouteChangeReasonKey")] as NSNumber; @@ -243,6 +287,10 @@ internal sealed class IosAudioRouter { NSNumber? type = note.UserInfo?[new NSString("AVAudioSessionInterruptionTypeKey")] as NSNumber; AVAudioSessionInterruptionType interruption = (AVAudioSessionInterruptionType)(type?.UInt32Value ?? 0); - if (interruption == AVAudioSessionInterruptionType.Ended) Recover("interruption ended"); + // The system stops the graph on Began and SetActive fails until the interruption clears, + // so suppress recovery until Ended and let the watchdog retry if that rebuild fails. + if (interruption == AVAudioSessionInterruptionType.Began) { interrupted = true; ResetWatchdog(); return; } + if (interruption != AVAudioSessionInterruptionType.Ended) return; + interrupted = false; ResetWatchdog(); Recover("interruption ended"); } } diff --git a/src/VoiceCat.Audio/AudioEngine.cs b/src/VoiceCat.Audio/AudioEngine.cs index e9e424c..7e4c550 100644 --- a/src/VoiceCat.Audio/AudioEngine.cs +++ b/src/VoiceCat.Audio/AudioEngine.cs @@ -177,6 +177,13 @@ public sealed class AudioEngine : IDisposable catch (Exception exception) { Failure = exception; stop.Cancel(); } } + // Drop each capture backlog to its buffer target. The consumer calls this after a stall it + // cannot catch up in place: one bounded gap instead of a queue that ratchets toward its edge. + public void ResynchronizeInputs() + { + foreach (LocalStream stream in Volatile.Read(ref routes).Local) stream.Input.Resynchronize(); + } + private void Work() { long deadline = Stopwatch.GetTimestamp(); @@ -189,12 +196,8 @@ public sealed class AudioEngine : IDisposable WaitUntil(deadline); if ((deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency < -100) { - // A stall this long cannot be caught up in place. Discarding only the schedule - // deficit would leave the producer backlog queued at a fixed offset forever and - // eventually overflow its ring, so drop back to the buffer targets instead: one - // bounded gap per stall rather than growing latency and feed drops. deadline = Stopwatch.GetTimestamp(); - foreach (LocalStream stream in Volatile.Read(ref routes).Local) stream.Input.Resynchronize(); + ResynchronizeInputs(); } } } diff --git a/tests/VoiceCat.Tests/AudioEngineTests.cs b/tests/VoiceCat.Tests/AudioEngineTests.cs index f9459d9..e58f068 100644 --- a/tests/VoiceCat.Tests/AudioEngineTests.cs +++ b/tests/VoiceCat.Tests/AudioEngineTests.cs @@ -446,6 +446,22 @@ public class AudioEngineTests Assert.Contains("public void Resynchronize()", buffer); } + [Fact] + public void ResynchronizeInputsDropsStalledBacklogToTheBufferTarget() + { + using var send = new AudioEngine((_, _, _, _) => true, false); + send.DeviceBufferMilliseconds = 40; send.AddLocalStream(Stream()); + short[] mono = Tone(); + for (int i = 0; i < 40; i++) send.FeedPcm(1, mono, 1); + Assert.True(send.GetLocalDiagnostics(1).BufferedFrames > 1920 * 2); + + send.ResynchronizeInputs(); + + Assert.Equal(1920, send.GetLocalDiagnostics(1).BufferedFrames); + send.ProcessCycle(); + Assert.Equal(0, send.GetLocalDiagnostics(1).StarvedCycles); + } + [Fact] public void RemotePlaybackSettingsAreIndependentForEachUserStream() { diff --git a/tests/VoiceCat.Tests/PublishServerScriptTests.cs b/tests/VoiceCat.Tests/PublishServerScriptTests.cs index e9a1452..71a62b8 100644 --- a/tests/VoiceCat.Tests/PublishServerScriptTests.cs +++ b/tests/VoiceCat.Tests/PublishServerScriptTests.cs @@ -164,6 +164,38 @@ public class PublishServerScriptTests Assert.Contains("deviceClockedAudio: true", model); } + [Fact] + public async Task IosRecoversBackgroundAudioWithoutForegrounding() + { + string root = FindRoot(); + string microphone = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs")); + string router = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "IosAudioRouter.cs")); + + // The render callback is the only clock for the device-clocked pipeline, so a graph that + // stops while backgrounded froze capture, mix and send until the app was foregrounded. + Assert.Contains("internal long RenderCallbacks", microphone); + Assert.Contains("internal bool IsRunning", microphone); + Assert.Contains("private void TickWatchdog()", router); + Assert.Contains("IosAudioEngine.Shared.RenderCallbacks", router); + Assert.Contains("IosAudioEngine.Shared.IsRunning", router); + Assert.Contains("IosAudioEngine.Shared.Reconfigure()", router); + + // SetActive fails for as long as an interruption is in force, so Began suppresses + // recovery and the watchdog retries the rebuild that Ended asks for. + Assert.Contains("AVAudioSessionInterruptionType.Began", router); + Assert.Contains("interrupted = true", router); + Assert.Contains("internal void ResumeForeground()", router); + + // A stalled render callback resynchronizes its own rings, and a refill is bounded so it + // cannot overrun the callback deadline that it is recovering from. + Assert.Contains("owner.Audio.ResynchronizeInputs()", microphone); + Assert.Contains("playbackRing.Resynchronize()", microphone); + Assert.Contains("Ring.Resynchronize()", microphone); + Assert.Contains("Math.Min(frames + playbackRing.TargetFrames - playbackRing.CountFrames, frames + 960)", microphone); + } + [Fact] public async Task IosAudioCommitsInputRouteBeforeChannelsAndVoiceProcessingBeforeConnections() {