Fix iOS voice capture and screen sharing

clients/apple/VoiceCat.iOS/IosAudioEngine.cs

@@ -1,5 +1,6 @@
 using System.Runtime.InteropServices;
 using AVFoundation;
+using UIKit;
 using VoiceCat.Audio;
 using VoiceCat.Core;
 
@@ -7,18 +8,29 @@ namespace VoiceCat.iOS;
 
 internal sealed class IosAudioEngine
 {
+    // InstallTapOnBus bufferSize is only a request. Physical iOS hardware can deliver 4,800
+    // frames per callback (observed with the voice-processing graph), so conversion storage
+    // must cover substantially more than the requested 960 frames without allocating in Capture.
+    private const int MaximumCaptureCallbackFrames = 16_384;
     internal static IosAudioEngine Shared { get; } = new();
     private readonly AdaptivePcmBuffer playbackRing = new(2, capacityFrames: 65_536);
     private readonly short[] renderScratch = new short[16_384];
     private AVAudioEngine? engine;
+    private Foundation.NSObject? engineConfigurationObserver;
     private AVAudioSourceNode? source;
     private AVAudioFormat? outputFormat;
     private VoiceCatClient? client;
-    private sealed record MicrophoneRoute(uint StreamId, int Channels, PcmRing Ring);
+    private sealed class MicrophoneRoute(uint streamId, int channels)
+    {
+        internal readonly uint StreamId = streamId;
+        internal readonly int Channels = channels;
+        internal readonly PcmRing Ring = new(131_072);
+        internal bool Primed;
+        internal int TargetFrames = 3;
+    }
     private MicrophoneRoute? microphone;
     private readonly short[] microphoneFrame = new short[960 * 2];
-    private readonly CancellationTokenSource microphoneStop = new();
-    private readonly Task microphoneWorker;
+    private readonly Thread microphonePump;
     private AVAudioFormat? microphoneFormat;
     private AVAudioConverter? microphoneConverter;
     private AVAudioPcmBuffer? convertedMicrophone;
@@ -26,10 +38,15 @@ internal sealed class IosAudioEngine
     private AVAudioConverterInputHandler? inputProvider;
     private bool inputProvided;
     private bool tapInstalled;
+    private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames;
     internal bool IsConnected { get; private set; }
     internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; }
 
-    private IosAudioEngine() { microphoneWorker = PumpMicrophoneAsync(microphoneStop.Token); }
+    private IosAudioEngine()
+    {
+        microphonePump = new Thread(PumpMicrophone) { IsBackground = true, Name = "VoiceCat iOS microphone pacer", Priority = ThreadPriority.Highest };
+        microphonePump.Start();
+    }
 
     internal void StartListening(VoiceCatClient owner)
     {
@@ -38,7 +55,7 @@ internal sealed class IosAudioEngine
 
     internal void StartMicrophone(uint streamId, int channels)
     {
-        Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2), new(131_072))); Rebuild();
+        Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2))); Rebuild();
     }
 
     internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); }
@@ -47,13 +64,13 @@ internal sealed class IosAudioEngine
         if (!IsConnected) return;
         MicrophoneRoute? current = Volatile.Read(ref microphone);
         int channels = IosAudioRouter.Shared.CaptureChannels;
-        if (current is not null && current.Channels != channels)
+        if (current is not null)
         {
             // Stop the old tap before publishing a route with a different sample width. Otherwise
             // an in-flight callback could interpret its old converter buffer using the new width.
             DestroyGraph();
-            client!.Audio.SetCaptureChannels(current.StreamId, channels);
-            Volatile.Write(ref microphone, new(current.StreamId, channels, new(131_072)));
+            if (current.Channels != channels) client!.Audio.SetCaptureChannels(current.StreamId, channels);
+            Volatile.Write(ref microphone, new(current.StreamId, channels));
         }
         Rebuild();
     }
@@ -67,6 +84,16 @@ internal sealed class IosAudioEngine
     {
         DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null);
         var next = new AVAudioEngine();
+        AVAudioInputNode? input = null;
+        if (route is not null)
+        {
+            // Enabling voice processing rebuilds both sides of AVAudioEngine. Do it before any
+            // formats are queried or nodes are connected so the graph is built from the final IO.
+            input = next.InputNode;
+            if (!input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out NSError? processingError))
+                throw new InvalidOperationException(processingError.LocalizedDescription);
+            if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl;
+        }
         outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false);
         source = new(outputFormat, Render);
         next.AttachNode(source);
@@ -76,13 +103,12 @@ internal sealed class IosAudioEngine
         if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription);
         if (route is not null)
         {
-            AVAudioInputNode input = next.InputNode;
-            input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out _);
-            if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl;
-            AVAudioFormat inputFormat = input.GetBusOutputFormat(0);
+            AVAudioFormat inputFormat = input!.GetBusOutputFormat(0);
+            Console.Error.WriteLine($"VC_GRAPH vpio={IosAudioRouter.Shared.UsesVoiceProcessing} requestedCh={route.Channels} " +
+                $"inputCh={inputFormat.ChannelCount} inputRate={inputFormat.SampleRate}");
             microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true);
             microphoneConverter = new(inputFormat, microphoneFormat);
-            uint capacity = checked((uint)Math.Ceiling(4_096 * 48_000 / inputFormat.SampleRate) + 64);
+            uint capacity = checked((uint)Math.Ceiling(MaximumCaptureCallbackFrames * 48_000 / inputFormat.SampleRate) + 64);
             convertedMicrophone = new(microphoneFormat, capacity);
             inputProvider = ProvideInput;
             NSError? tapError = null;
@@ -94,10 +120,21 @@ internal sealed class IosAudioEngine
         next.Prepare();
         if (!next.StartAndReturnError(out NSError? error)) { next.Dispose(); throw new InvalidOperationException(error.LocalizedDescription); }
         engine = next;
+        engineConfigurationObserver = Foundation.NSNotificationCenter.DefaultCenter.AddObserver(
+            AVAudioEngine.ConfigurationChangeNotification, notification =>
+            {
+                if (!ReferenceEquals(notification.Object, next)) return;
+                UIApplication.SharedApplication.BeginInvokeOnMainThread(() =>
+                {
+                    if (IsConnected && ReferenceEquals(engine, next) && !next.Running) Rebuild();
+                });
+            }, next);
     }
 
     private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time)
     {
+        Interlocked.Increment(ref captureCallbacks);
+        Interlocked.Add(ref capturedFrames, buffer.FrameLength);
         VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone);
         if (owner is null || route is null || buffer.FrameLength == 0) return;
         AVAudioConverter? converter = microphoneConverter;
@@ -105,34 +142,79 @@ internal sealed class IosAudioEngine
         AVAudioConverterInputHandler? provider = inputProvider;
         if (converter is null || converted is null || provider is null) return;
         pendingInput = buffer; inputProvided = false; converted.FrameLength = 0;
-        converter.ConvertToBuffer(converted, out _, provider);
+        converter.ConvertToBuffer(converted, out NSError? conversionError, provider);
+        if (conversionError is not null) Interlocked.Increment(ref converterFailures);
         if (converted.FrameLength == 0) return;
+        Interlocked.Add(ref convertedFrames, converted.FrameLength);
         nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData);
-        if (samples != 0) route.Ring.TryWrite(new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * route.Channels)));
+        if (samples != 0 && ReferenceEquals(route, Volatile.Read(ref microphone)))
+        {
+            var pcm = new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * route.Channels));
+            if (route.Channels == 2)
+            {
+                Interlocked.Add(ref stereoFrames, converted.FrameLength);
+                long different = 0;
+                for (int frame = 0; frame < converted.FrameLength; frame++)
+                    if (pcm[frame * 2] != pcm[frame * 2 + 1]) different++;
+                Interlocked.Add(ref stereoDifferentFrames, different);
+            }
+            if (!route.Ring.TryWrite(pcm)) Interlocked.Increment(ref rejectedFeeds);
+        }
         pendingInput = null;
     }
 
+    internal string CaptureDiagnostics()
+    {
+        return $"callbacks={Interlocked.Exchange(ref captureCallbacks, 0)} input={Interlocked.Exchange(ref capturedFrames, 0)} " +
+            $"converted={Interlocked.Exchange(ref convertedFrames, 0)} feedDrops={Interlocked.Exchange(ref rejectedFeeds, 0)} " +
+            $"converterErrors={Interlocked.Exchange(ref converterFailures, 0)} stereo={Interlocked.Exchange(ref stereoDifferentFrames, 0)}/" +
+            $"{Interlocked.Exchange(ref stereoFrames, 0)}";
+    }
+
     private AVAudioBuffer ProvideInput(uint _, out AVAudioConverterInputStatus status)
     {
         if (!inputProvided && pendingInput is { } input) { inputProvided = true; status = AVAudioConverterInputStatus.HaveData; return input; }
         status = AVAudioConverterInputStatus.NoDataNow; return null!;
     }
 
-    private async Task PumpMicrophoneAsync(CancellationToken token)
+    private void PumpMicrophone()
     {
-        using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(10));
-        while (await timer.WaitForNextTickAsync(token).ConfigureAwait(false))
+        long deadline = System.Diagnostics.Stopwatch.GetTimestamp();
+        while (true)
         {
             MicrophoneRoute? route = Volatile.Read(ref microphone);
-            if (route is null) continue;
-            int required = 960 * route.Channels;
-            while (route.Ring.Count >= required)
+            VoiceCatClient? owner = client;
+            if (route is not null && owner is not null)
             {
-                int read = route.Ring.Read(microphoneFrame.AsSpan(0, required));
-                VoiceCatClient? owner = client;
-                if (read == required && owner is not null && ReferenceEquals(route, Volatile.Read(ref microphone)))
-                    owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels);
+                int required = 960 * route.Channels;
+                int completeFrames = route.Ring.Count / required;
+                if (!route.Primed)
+                {
+                    if (completeFrames >= route.TargetFrames) route.Primed = true;
+                }
+                if (route.Primed)
+                {
+                    if (completeFrames == 0)
+                    {
+                        route.Primed = false;
+                        route.TargetFrames = Math.Min(route.TargetFrames + 1, 6);
+                    }
+                    else if (route.Ring.Read(microphoneFrame.AsSpan(0, required)) == required &&
+                        ReferenceEquals(route, Volatile.Read(ref microphone)) &&
+                        !owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels))
+                        Interlocked.Increment(ref rejectedFeeds);
+                }
             }
+            deadline += System.Diagnostics.Stopwatch.Frequency / 50;
+            while (true)
+            {
+                long remaining = deadline - System.Diagnostics.Stopwatch.GetTimestamp();
+                if (remaining <= 0) break;
+                double milliseconds = remaining * 1000.0 / System.Diagnostics.Stopwatch.Frequency;
+                if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1)); else Thread.SpinWait(64);
+            }
+            if ((deadline - System.Diagnostics.Stopwatch.GetTimestamp()) * 1000.0 / System.Diagnostics.Stopwatch.Frequency < -100)
+                deadline = System.Diagnostics.Stopwatch.GetTimestamp();
         }
     }
 
@@ -165,6 +247,11 @@ internal sealed class IosAudioEngine
 
     private void DestroyGraph()
     {
+        if (engineConfigurationObserver is { } observer)
+        {
+            Foundation.NSNotificationCenter.DefaultCenter.RemoveObserver(observer);
+            observer.Dispose(); engineConfigurationObserver = null;
+        }
         if (engine is { } old)
         {
             if (tapInstalled) old.InputNode.RemoveTapOnBus(0);