using System.Runtime.InteropServices; using AVFoundation; using UIKit; using VoiceCat.Audio; using VoiceCat.Core; namespace VoiceCat.iOS; internal sealed class IosAudioEngine { // InstallTapOnBus bufferSize is only a request. Physical iOS hardware can deliver 4,800 // frames per callback (observed with the voice-processing graph), so conversion storage // must cover substantially more than the requested 960 frames without allocating in Capture. private const int MaximumCaptureCallbackFrames = 16_384; internal static IosAudioEngine Shared { get; } = new(); private readonly AdaptivePcmBuffer playbackRing = new(2, capacityFrames: 65_536); private readonly short[] renderScratch = new short[16_384]; private AVAudioEngine? engine; private Foundation.NSObject? engineConfigurationObserver; private AVAudioSourceNode? source; private AVAudioFormat? outputFormat; private VoiceCatClient? client; private sealed class MicrophoneRoute(uint streamId, int channels) { internal readonly uint StreamId = streamId; internal readonly int Channels = channels; internal readonly PcmRing Ring = new(131_072); internal bool Primed; internal int TargetFrames = 3; } private MicrophoneRoute? microphone; private readonly short[] microphoneFrame = new short[960 * 2]; private readonly Thread microphonePump; private AVAudioFormat? microphoneFormat; private AVAudioConverter? microphoneConverter; private AVAudioPcmBuffer? convertedMicrophone; private AVAudioPcmBuffer? pendingInput; private AVAudioConverterInputHandler? inputProvider; private bool inputProvided; private bool tapInstalled; private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames; internal bool IsConnected { get; private set; } internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; } private IosAudioEngine() { microphonePump = new Thread(PumpMicrophone) { IsBackground = true, Name = "VoiceCat iOS microphone pacer", Priority = ThreadPriority.Highest }; microphonePump.Start(); } internal void StartListening(VoiceCatClient owner) { Stop(); client = owner; IsConnected = true; owner.Audio.MixedPcm += ReceiveMixedPcm; Rebuild(); } internal void StartMicrophone(uint streamId, int channels) { Volatile.Write(ref microphone, CreateMicrophoneRoute(streamId, channels)); Rebuild(); } internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); } internal void Reconfigure() { if (!IsConnected) return; MicrophoneRoute? current = Volatile.Read(ref microphone); int channels = IosAudioRouter.Shared.CaptureChannels; if (current is not null) { // Stop the old tap before publishing a route with a different sample width. Otherwise // an in-flight callback could interpret its old converter buffer using the new width. DestroyGraph(); if (current.Channels != channels) client!.Audio.SetCaptureChannels(current.StreamId, channels); Volatile.Write(ref microphone, CreateMicrophoneRoute(current.StreamId, channels)); } Rebuild(); } private static MicrophoneRoute CreateMicrophoneRoute(uint streamId, int channels) { var route = new MicrophoneRoute(streamId, Math.Clamp(channels, 1, 2)); // Physical RemoteIO capture is commonly delivered in 100 ms bursts. Starting its 20 ms // pacer with only the VPIO-oriented 60 ms cushion guarantees several underruns after every // mono/stereo rebuild before the adaptive path catches up. Prime one full burst plus one // frame for non-VPIO routes; VPIO remains at the proven low-latency three-frame cushion. route.TargetFrames = IosAudioRouter.Shared.UsesVoiceProcessing ? 3 : 6; return route; } internal bool EnsureRunning() { if (!IsConnected || engine?.Running == true) return true; Rebuild(); return engine?.Running == true; } private void Rebuild() { DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null); var next = new AVAudioEngine(); AVAudioInputNode? input = null; if (route is not null) { // Enabling voice processing rebuilds both sides of AVAudioEngine. Do it before any // formats are queried or nodes are connected so the graph is built from the final IO. input = next.InputNode; if (!input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out NSError? processingError)) throw new InvalidOperationException(processingError.LocalizedDescription); if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl; } outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false); source = new(outputFormat, Render); next.AttachNode(source); NSError? connectionError = null; if (OperatingSystem.IsIOSVersionAtLeast(27)) next.Connect(source, next.MainMixerNode, outputFormat, out connectionError); else next.Connect(source, next.MainMixerNode, outputFormat); if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription); if (route is not null) { AVAudioFormat inputFormat = input!.GetBusOutputFormat(0); Console.Error.WriteLine($"VC_GRAPH vpio={IosAudioRouter.Shared.UsesVoiceProcessing} requestedCh={route.Channels} " + $"inputCh={inputFormat.ChannelCount} inputRate={inputFormat.SampleRate}"); microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true); microphoneConverter = new(inputFormat, microphoneFormat); uint capacity = checked((uint)Math.Ceiling(MaximumCaptureCallbackFrames * 48_000 / inputFormat.SampleRate) + 64); convertedMicrophone = new(microphoneFormat, capacity); inputProvider = ProvideInput; NSError? tapError = null; if (OperatingSystem.IsIOSVersionAtLeast(27)) input.InstallTapOnBus(0, 960, inputFormat, out tapError, Capture); else input.InstallTapOnBus(0, 960, inputFormat, Capture); if (tapError is not null) throw new InvalidOperationException(tapError.LocalizedDescription); tapInstalled = true; } next.Prepare(); if (!next.StartAndReturnError(out NSError? error)) { next.Dispose(); throw new InvalidOperationException(error.LocalizedDescription); } engine = next; engineConfigurationObserver = Foundation.NSNotificationCenter.DefaultCenter.AddObserver( AVAudioEngine.ConfigurationChangeNotification, notification => { if (!ReferenceEquals(notification.Object, next)) return; UIApplication.SharedApplication.BeginInvokeOnMainThread(() => { if (IsConnected && ReferenceEquals(engine, next) && !next.Running) Rebuild(); }); }, next); } private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time) { Interlocked.Increment(ref captureCallbacks); Interlocked.Add(ref capturedFrames, buffer.FrameLength); VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone); if (owner is null || route is null || buffer.FrameLength == 0) return; AVAudioConverter? converter = microphoneConverter; AVAudioPcmBuffer? converted = convertedMicrophone; AVAudioConverterInputHandler? provider = inputProvider; if (converter is null || converted is null || provider is null) return; pendingInput = buffer; inputProvided = false; converted.FrameLength = 0; converter.ConvertToBuffer(converted, out NSError? conversionError, provider); if (conversionError is not null) Interlocked.Increment(ref converterFailures); if (converted.FrameLength == 0) return; Interlocked.Add(ref convertedFrames, converted.FrameLength); nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData); if (samples != 0 && ReferenceEquals(route, Volatile.Read(ref microphone))) { var pcm = new ReadOnlySpan((void*)samples, checked((int)converted.FrameLength * route.Channels)); if (route.Channels == 2) { Interlocked.Add(ref stereoFrames, converted.FrameLength); long different = 0; for (int frame = 0; frame < converted.FrameLength; frame++) if (pcm[frame * 2] != pcm[frame * 2 + 1]) different++; Interlocked.Add(ref stereoDifferentFrames, different); } if (!route.Ring.TryWrite(pcm)) Interlocked.Increment(ref rejectedFeeds); } pendingInput = null; } internal string CaptureDiagnostics() { return $"callbacks={Interlocked.Exchange(ref captureCallbacks, 0)} input={Interlocked.Exchange(ref capturedFrames, 0)} " + $"converted={Interlocked.Exchange(ref convertedFrames, 0)} feedDrops={Interlocked.Exchange(ref rejectedFeeds, 0)} " + $"converterErrors={Interlocked.Exchange(ref converterFailures, 0)} stereo={Interlocked.Exchange(ref stereoDifferentFrames, 0)}/" + $"{Interlocked.Exchange(ref stereoFrames, 0)}"; } private AVAudioBuffer ProvideInput(uint _, out AVAudioConverterInputStatus status) { if (!inputProvided && pendingInput is { } input) { inputProvided = true; status = AVAudioConverterInputStatus.HaveData; return input; } status = AVAudioConverterInputStatus.NoDataNow; return null!; } private void PumpMicrophone() { long deadline = System.Diagnostics.Stopwatch.GetTimestamp(); while (true) { MicrophoneRoute? route = Volatile.Read(ref microphone); VoiceCatClient? owner = client; if (route is not null && owner is not null) { int required = 960 * route.Channels; int completeFrames = route.Ring.Count / required; if (!route.Primed) { if (completeFrames >= route.TargetFrames) route.Primed = true; } if (route.Primed) { if (completeFrames == 0) { route.Primed = false; route.TargetFrames = Math.Min(route.TargetFrames + 1, 6); } else if (route.Ring.Read(microphoneFrame.AsSpan(0, required)) == required && ReferenceEquals(route, Volatile.Read(ref microphone)) && !owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels)) Interlocked.Increment(ref rejectedFeeds); } } deadline += System.Diagnostics.Stopwatch.Frequency / 50; while (true) { long remaining = deadline - System.Diagnostics.Stopwatch.GetTimestamp(); if (remaining <= 0) break; double milliseconds = remaining * 1000.0 / System.Diagnostics.Stopwatch.Frequency; if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1)); else Thread.SpinWait(64); } if ((deadline - System.Diagnostics.Stopwatch.GetTimestamp()) * 1000.0 / System.Diagnostics.Stopwatch.Frequency < -100) deadline = System.Diagnostics.Stopwatch.GetTimestamp(); } } private void ReceiveMixedPcm(ReadOnlySpan pcm) => playbackRing.TryWrite(pcm); private unsafe int Render(IntPtr isSilence, IntPtr timestamp, uint frameCount, IntPtr outputData) { int frames = checked((int)frameCount), requested = checked(frames * 2); if (requested > renderScratch.Length) return -1; Span input = renderScratch.AsSpan(0, requested); int read = playbackRing.Read(input); input[read..].Clear(); int count = Marshal.ReadInt32(outputData), first = IntPtr.Size == 8 ? 8 : 4, stride = IntPtr.Size == 8 ? 16 : 12; if (count != 2) return -1; for (int channel = 0; channel < 2; channel++) { nint data = Marshal.ReadIntPtr(outputData, first + channel * stride + 8); var output = new Span((void*)data, frames); for (int frame = 0; frame < frames; frame++) output[frame] = input[frame * 2 + channel] / 32768f; } if (isSilence != IntPtr.Zero) Marshal.WriteByte(isSilence, read == 0 ? (byte)1 : (byte)0); return 0; } internal void Stop() { IsConnected = false; Volatile.Write(ref microphone, null); if (client is { } owner) owner.Audio.MixedPcm -= ReceiveMixedPcm; DestroyGraph(); client = null; IosAudioRouter.Shared.Deactivate(); } private void DestroyGraph() { if (engineConfigurationObserver is { } observer) { Foundation.NSNotificationCenter.DefaultCenter.RemoveObserver(observer); observer.Dispose(); engineConfigurationObserver = null; } if (engine is { } old) { if (tapInstalled) old.InputNode.RemoveTapOnBus(0); old.Stop(); if (source is not null) old.DetachNode(source); old.Dispose(); } tapInstalled = false; pendingInput = null; inputProvider = null; convertedMicrophone?.Dispose(); convertedMicrophone = null; microphoneConverter?.Dispose(); microphoneConverter = null; microphoneFormat?.Dispose(); microphoneFormat = null; source?.Dispose(); source = null; outputFormat?.Dispose(); outputFormat = null; engine = null; } }