From 4aa0aa4fbdd5ee53a3d2b7489bbed44e1bbb69b5 Mon Sep 17 00:00:00 2001 From: Talon Date: Mon, 21 Sep 2026 18:04:00 +0200 Subject: [PATCH] Fix iOS stereo microphone capture --- PROGRESS.md | 13 +++-- clients/apple/VoiceCat.iOS/IosAudioEngine.cs | 15 ++++- clients/apple/VoiceCat.iOS/IosAudioRouter.cs | 56 +++++++++++++++++-- .../PublishServerScriptTests.cs | 5 ++ 4 files changed, 75 insertions(+), 14 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index c87bc77..faf3517 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -19,10 +19,11 @@ The source-of-truth layout is: - `native/media/` and `native/rnnoise/` — required Opus/RNNoise shim and vendored RNNoise. - `native/apple/broadcast/` — required ReplayKit upload extension and shared ring producer. -The physical-device iOS voice path now supports ReplayKit fallback and stable Apple VPIO -voice-chat capture using a paced 20 ms handoff. Stereo microphone routing remains an open device -gate: the stereo preset currently receives duplicated mono samples despite selecting the built-in -stereo capsule. +The physical-device iOS voice path now supports ReplayKit fallback, stable Apple VPIO voice-chat +capture using a paced 20 ms handoff, and true built-in stereo microphone capture. Stereo was +verified on an iPhone 16 Pro Max with a two-channel AVAudioEngine input and distinct left/right +samples; the managed Apple binding requires native use of its otherwise-unmapped stereo polar +pattern constant. ## Release gates @@ -30,8 +31,8 @@ stereo capsule. 20/40/60 ms buffering, duration-aware DRED/FEC, and mismatched input/output endpoints. - Complete NVDA and VoiceOver navigation/announcement passes. - Exercise iOS background/lock, interruption, Bluetooth, route-change, ReplayKit, and iOS 27 - ScreenCaptureKit paths on devices. Verify distinct left/right capture with the stereo microphone - preset, including changing mono/stereo while joined, and Windows desktop/per-app stereo sharing. + ScreenCaptureKit paths on devices. Complete extended mono/stereo/voice-chat switching while + joined, and verify Windows desktop/per-app stereo sharing. - Complete Developer ID signing/notarization. The iOS host and ReplayKit extension have been distribution-signed and packaged locally; upload the IPA for Apple's server-side validation. - Run the published Linux container and a 30-minute-or-longer server soak. diff --git a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs index 105189f..ba17072 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs @@ -55,7 +55,7 @@ internal sealed class IosAudioEngine internal void StartMicrophone(uint streamId, int channels) { - Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2))); Rebuild(); + Volatile.Write(ref microphone, CreateMicrophoneRoute(streamId, channels)); Rebuild(); } internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); } @@ -70,10 +70,21 @@ internal sealed class IosAudioEngine // an in-flight callback could interpret its old converter buffer using the new width. DestroyGraph(); if (current.Channels != channels) client!.Audio.SetCaptureChannels(current.StreamId, channels); - Volatile.Write(ref microphone, new(current.StreamId, channels)); + Volatile.Write(ref microphone, CreateMicrophoneRoute(current.StreamId, channels)); } Rebuild(); } + + private static MicrophoneRoute CreateMicrophoneRoute(uint streamId, int channels) + { + var route = new MicrophoneRoute(streamId, Math.Clamp(channels, 1, 2)); + // Physical RemoteIO capture is commonly delivered in 100 ms bursts. Starting its 20 ms + // pacer with only the VPIO-oriented 60 ms cushion guarantees several underruns after every + // mono/stereo rebuild before the adaptive path catches up. Prime one full burst plus one + // frame for non-VPIO routes; VPIO remains at the proven low-latency three-frame cushion. + route.TargetFrames = IosAudioRouter.Shared.UsesVoiceProcessing ? 3 : 6; + return route; + } internal bool EnsureRunning() { if (!IsConnected || engine?.Running == true) return true; diff --git a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs index 890b39a..3a59006 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs @@ -1,5 +1,6 @@ using AVFoundation; using Foundation; +using ObjCRuntime; using UIKit; namespace VoiceCat.iOS; @@ -12,6 +13,8 @@ internal sealed record IosAudioDataSource(string Id, string Name, IReadOnlyList< internal sealed class IosAudioRouter { + private static readonly NativeHandle SupportedPolarPatternsSelector = Selector.GetHandle("supportedPolarPatterns"); + private static readonly NativeHandle SetPreferredPolarPatternSelector = Selector.GetHandle("setPreferredPolarPattern:error:"); internal static IosAudioRouter Shared { get; } = new(); private readonly NSUserDefaults defaults = NSUserDefaults.StandardUserDefaults; private bool applying; @@ -123,17 +126,26 @@ internal sealed class IosAudioRouter AVAudioSessionPortDescription? port = session.AvailableInputs?.FirstOrDefault(value => value.UID == SelectedInputId); if (CaptureChannels == 2) port ??= session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic); if (port is null) { if (CaptureChannels == 1) ClearStereoPolarPattern(session); return; } + if (CaptureChannels == 2) + { + // Polar-pattern discovery alone is insufficient on current iPhones: until a stereo + // orientation is requested, the built-in port can expose only its mono Bottom source + // and AVAudioEngine consequently binds a one-channel input node. This is Apple's + // dedicated switch for built-in stereo recording. Portrait matches this portrait UI; + // it also gives Core Audio an unambiguous left/right mapping before graph creation. + if (!session.SetPreferredInputOrientation(AVAudioStereoOrientation.Portrait, out NSError? orientationError)) + throw new InvalidOperationException(orientationError?.LocalizedDescription ?? "Could not set the stereo microphone orientation."); + } AVAudioSessionDataSourceDescription? source = CaptureChannels == 2 - ? port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true) + ? port.DataSources?.FirstOrDefault(SupportsStereoPolarPattern) : port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId); if (CaptureChannels == 1 && source is null) ClearStereoPolarPattern(session); if (source is not null) { if (!port.SetPreferredDataSource(source, out NSError? portError)) throw new InvalidOperationException(portError.LocalizedDescription); - AVAudioDataSourcePolarPattern pattern = CaptureChannels == 2 - ? source.SupportedPolarPatterns?.FirstOrDefault(value => value.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) ?? AVAudioDataSourcePolarPattern.Unknown - : SelectedPolarPattern; - if (pattern != AVAudioDataSourcePolarPattern.Unknown && !source.SetPreferredPolarPattern(pattern, out NSError? patternError)) + if (CaptureChannels == 2) SetStereoPolarPattern(source); + else if (SelectedPolarPattern != AVAudioDataSourcePolarPattern.Unknown && + !source.SetPreferredPolarPattern(SelectedPolarPattern, out NSError? patternError)) throw new InvalidOperationException(patternError.LocalizedDescription); } if (!session.SetPreferredInput(port, out NSError? inputError)) throw new InvalidOperationException(inputError.LocalizedDescription); @@ -144,15 +156,47 @@ internal sealed class IosAudioRouter !session.SetInputDataSource(source, out NSError? sourceError)) throw new InvalidOperationException(sourceError.LocalizedDescription); Console.Error.WriteLine($"VC_ROUTE_SELECT port={port.PortName} source={source?.DataSourceName ?? "none"} " + + $"stereoPattern={source is not null && SupportsStereoPolarPattern(source)} " + $"patterns={string.Join(',', source?.SupportedPolarPatterns?.Select(value => value.ToString()) ?? [])}"); } + // Microsoft.iOS 27 exposes AVAudioSessionPolarPatternStereo as an NSString constant but its + // AVAudioDataSourcePolarPattern smart enum still has no Stereo member. Reading the native + // NSArray and setting the native NSString avoids silently mapping Stereo to Unknown. + private static bool SupportsStereoPolarPattern(AVAudioSessionDataSourceDescription source) + { + NativeHandle handle = NativeMethods.GetObject(source.Handle, SupportedPolarPatternsSelector); + NSArray? patterns = Runtime.GetNSObject(handle); + if (patterns is null) return false; + for (nuint index = 0; index < patterns.Count; index++) + if (patterns.GetItem(index)?.IsEqualTo(AVAudioSession.PolarPatternStereo.Handle) == true) return true; + return false; + } + + private static void SetStereoPolarPattern(AVAudioSessionDataSourceDescription source) + { + NativeHandle error = NativeHandle.Zero; + if (NativeMethods.SetObject(source.Handle, SetPreferredPolarPatternSelector, + AVAudioSession.PolarPatternStereo.Handle, ref error) == 0) + throw new InvalidOperationException(Runtime.GetNSObject(error)?.LocalizedDescription ?? "Could not enable stereo microphone capture."); + } + + private static class NativeMethods + { + [System.Runtime.InteropServices.DllImport("/usr/lib/libobjc.dylib", EntryPoint = "objc_msgSend")] + internal static extern NativeHandle GetObject(NativeHandle receiver, NativeHandle selector); + + [System.Runtime.InteropServices.DllImport("/usr/lib/libobjc.dylib", EntryPoint = "objc_msgSend")] + internal static extern byte SetObject(NativeHandle receiver, NativeHandle selector, NativeHandle value, ref NativeHandle error); + } + private static void ClearStereoPolarPattern(AVAudioSession session) { + session.SetPreferredInputOrientation(AVAudioStereoOrientation.None, out _); AVAudioSessionPortDescription? builtIn = session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic); if (builtIn is null) return; foreach (AVAudioSessionDataSourceDescription source in builtIn.DataSources ?? []) - if (source.SelectedPolarPattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) + if (source.SelectedPolarPattern == AVAudioDataSourcePolarPattern.Unknown && SupportsStereoPolarPattern(source)) source.SetPreferredPolarPattern(AVAudioDataSourcePolarPattern.Unknown, out _); } diff --git a/tests/VoiceCat.Tests/PublishServerScriptTests.cs b/tests/VoiceCat.Tests/PublishServerScriptTests.cs index 672afce..08280b3 100644 --- a/tests/VoiceCat.Tests/PublishServerScriptTests.cs +++ b/tests/VoiceCat.Tests/PublishServerScriptTests.cs @@ -124,6 +124,11 @@ public class PublishServerScriptTests router.IndexOf("SetActive(true", StringComparison.Ordinal)); Assert.DoesNotContain("session.SetPreferredInputNumberOfChannels", router); Assert.DoesNotContain("MaximumInputNumberOfChannels", router); + Assert.Contains("SetPreferredInputOrientation(AVAudioStereoOrientation.Portrait", router); + Assert.Contains("SetPreferredInputOrientation(AVAudioStereoOrientation.None", router); + Assert.Contains("AVAudioSession.PolarPatternStereo", router); + Assert.Contains("SupportsStereoPolarPattern", router); + Assert.Contains("route.TargetFrames = IosAudioRouter.Shared.UsesVoiceProcessing ? 3 : 6", engine); Assert.Contains("SelectedDataSourceId = null; SelectedPolarPattern = AVAudioDataSourcePolarPattern.Unknown", router); Assert.Contains("ClearStereoPolarPattern(session)", router); Assert.Contains("CaptureChannels == 2\n ? port.DataSources?.FirstOrDefault", router);