From 9be33f357d201c6297eb380dc451ea208fb6a0f3 Mon Sep 17 00:00:00 2001 From: Talon Date: Mon, 21 Sep 2026 02:11:33 +0200 Subject: [PATCH] Fix stereo capture paths --- PROGRESS.md | 3 +- clients/apple/VoiceCat.iOS/IosAudioEngine.cs | 52 ++++++++++++------- clients/apple/VoiceCat.iOS/IosAudioRouter.cs | 25 +++++++-- .../windows/VoiceCat.Windows/AudioControls.cs | 5 +- .../VoiceCat.Windows/VoiceCatClient.cs | 4 +- .../WindowsManagedClientTests.cs | 38 ++++++++++++++ 6 files changed, 100 insertions(+), 27 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 5fdc62e..1e5aaf8 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -25,7 +25,8 @@ The source-of-truth layout is: 20/40/60 ms buffering, duration-aware DRED/FEC, and mismatched input/output endpoints. - Complete NVDA and VoiceOver navigation/announcement passes. - Exercise iOS background/lock, interruption, Bluetooth, route-change, ReplayKit, and iOS 27 - ScreenCaptureKit paths on devices. + ScreenCaptureKit paths on devices. Verify distinct left/right capture with the stereo microphone + preset, including changing mono/stereo while joined, and Windows desktop/per-app stereo sharing. - Complete Developer ID signing/notarization and iOS distribution signing. - Run the published Linux container and a 30-minute-or-longer server soak. diff --git a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs index 33bcf91..bbc87b8 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs @@ -14,9 +14,8 @@ internal sealed class IosAudioEngine private AVAudioSourceNode? source; private AVAudioFormat? outputFormat; private VoiceCatClient? client; - private uint microphoneStream; - private int microphoneChannels = 1; - private readonly PcmRing microphoneRing = new(131_072); + private sealed record MicrophoneRoute(uint StreamId, int Channels, PcmRing Ring); + private MicrophoneRoute? microphone; private readonly short[] microphoneFrame = new short[960 * 2]; private readonly CancellationTokenSource microphoneStop = new(); private readonly Task microphoneWorker; @@ -39,11 +38,25 @@ internal sealed class IosAudioEngine internal void StartMicrophone(uint streamId, int channels) { - microphoneStream = streamId; microphoneChannels = Math.Clamp(channels, 1, 2); Rebuild(); + Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2), new(131_072))); Rebuild(); } - internal void StopMicrophone() { microphoneStream = 0; Rebuild(); } - internal void Reconfigure() { if (IsConnected) Rebuild(); } + internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); } + internal void Reconfigure() + { + if (!IsConnected) return; + MicrophoneRoute? current = Volatile.Read(ref microphone); + int channels = IosAudioRouter.Shared.CaptureChannels; + if (current is not null && current.Channels != channels) + { + // Stop the old tap before publishing a route with a different sample width. Otherwise + // an in-flight callback could interpret its old converter buffer using the new width. + DestroyGraph(); + client!.Audio.SetCaptureChannels(current.StreamId, channels); + Volatile.Write(ref microphone, new(current.StreamId, channels, new(131_072))); + } + Rebuild(); + } internal bool EnsureRunning() { if (!IsConnected || engine?.Running == true) return true; @@ -52,7 +65,7 @@ internal sealed class IosAudioEngine private void Rebuild() { - DestroyGraph(); IosAudioRouter.Shared.Apply(); + DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null); var next = new AVAudioEngine(); outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false); source = new(outputFormat, Render); @@ -61,13 +74,13 @@ internal sealed class IosAudioEngine if (OperatingSystem.IsIOSVersionAtLeast(27)) next.Connect(source, next.MainMixerNode, outputFormat, out connectionError); else next.Connect(source, next.MainMixerNode, outputFormat); if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription); - if (microphoneStream != 0) + if (route is not null) { AVAudioInputNode input = next.InputNode; input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out _); if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl; AVAudioFormat inputFormat = input.GetBusOutputFormat(0); - microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)microphoneChannels, true); + microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true); microphoneConverter = new(inputFormat, microphoneFormat); uint capacity = checked((uint)Math.Ceiling(4_096 * 48_000 / inputFormat.SampleRate) + 64); convertedMicrophone = new(microphoneFormat, capacity); @@ -85,8 +98,8 @@ internal sealed class IosAudioEngine private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time) { - VoiceCatClient? owner = client; uint stream = microphoneStream; - if (owner is null || stream == 0 || buffer.FrameLength == 0) return; + VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone); + if (owner is null || route is null || buffer.FrameLength == 0) return; AVAudioConverter? converter = microphoneConverter; AVAudioPcmBuffer? converted = convertedMicrophone; AVAudioConverterInputHandler? provider = inputProvider; @@ -95,7 +108,7 @@ internal sealed class IosAudioEngine converter.ConvertToBuffer(converted, out _, provider); if (converted.FrameLength == 0) return; nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData); - if (samples != 0) microphoneRing.TryWrite(new ReadOnlySpan((void*)samples, checked((int)converted.FrameLength * microphoneChannels))); + if (samples != 0) route.Ring.TryWrite(new ReadOnlySpan((void*)samples, checked((int)converted.FrameLength * route.Channels))); pendingInput = null; } @@ -110,12 +123,15 @@ internal sealed class IosAudioEngine using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(10)); while (await timer.WaitForNextTickAsync(token).ConfigureAwait(false)) { - int channels = microphoneChannels, required = 960 * channels; - while (microphoneRing.Count >= required) + MicrophoneRoute? route = Volatile.Read(ref microphone); + if (route is null) continue; + int required = 960 * route.Channels; + while (route.Ring.Count >= required) { - int read = microphoneRing.Read(microphoneFrame.AsSpan(0, required)); - VoiceCatClient? owner = client; uint stream = microphoneStream; - if (read == required && owner is not null && stream != 0) owner.Audio.FeedPcm(stream, microphoneFrame.AsSpan(0, required), channels); + int read = route.Ring.Read(microphoneFrame.AsSpan(0, required)); + VoiceCatClient? owner = client; + if (read == required && owner is not null && ReferenceEquals(route, Volatile.Read(ref microphone))) + owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels); } } } @@ -142,7 +158,7 @@ internal sealed class IosAudioEngine internal void Stop() { - IsConnected = false; microphoneStream = 0; + IsConnected = false; Volatile.Write(ref microphone, null); if (client is { } owner) owner.Audio.MixedPcm -= ReceiveMixedPcm; DestroyGraph(); client = null; IosAudioRouter.Shared.Deactivate(); } diff --git a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs index a413b66..cfa666a 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs @@ -90,7 +90,7 @@ internal sealed class IosAudioRouter SelectedInputId ??= session.PreferredInput?.UID; Changed?.Invoke(); } - internal void Apply() + internal void Apply(bool configureInput) { if (applying) return; applying = true; try @@ -102,8 +102,9 @@ internal sealed class IosAudioRouter AVAudioSessionMode mode = CaptureChannels == 2 ? AVAudioSessionMode.Default : MicMode == IosMicMode.Raw ? AVAudioSessionMode.Measurement : BluetoothMode == IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionMode.VideoRecording : AVAudioSessionMode.VoiceChat; if (!session.SetCategory(AVAudioSessionCategory.PlayAndRecord, mode, options, out NSError? categoryError)) throw new InvalidOperationException(categoryError.LocalizedDescription); - session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _); ApplyInput(session); + session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _); if (!session.SetActive(true, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out NSError? activeError)) throw new InvalidOperationException(activeError.LocalizedDescription); + if (configureInput) ApplyInput(session); session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes(); } finally { applying = false; } @@ -116,8 +117,24 @@ internal sealed class IosAudioRouter if (port is null) return; session.SetPreferredInput(port, out _); AVAudioSessionDataSourceDescription? source = port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId); if (CaptureChannels == 2) source ??= port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true); - if (source is null) return; port.SetPreferredDataSource(source, out _); session.SetInputDataSource(source, out _); - if (CaptureChannels != 2 && SelectedPolarPattern != AVAudioDataSourcePolarPattern.Unknown) source.SetPreferredPolarPattern(SelectedPolarPattern, out _); + if (source is not null) + { + if (!port.SetPreferredDataSource(source, out NSError? portError)) throw new InvalidOperationException(portError.LocalizedDescription); + if (!session.SetInputDataSource(source, out NSError? sourceError)) throw new InvalidOperationException(sourceError.LocalizedDescription); + } + if (session.MaximumInputNumberOfChannels < CaptureChannels) + throw new InvalidOperationException($"The selected input supports at most {session.MaximumInputNumberOfChannels} channel(s), not {CaptureChannels}."); + if (!session.SetPreferredInputNumberOfChannels(CaptureChannels, out NSError? channelError)) throw new InvalidOperationException(channelError.LocalizedDescription); + if (source is not null) + { + AVAudioDataSourcePolarPattern pattern = CaptureChannels == 2 + ? source.SupportedPolarPatterns?.FirstOrDefault(value => value.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) ?? AVAudioDataSourcePolarPattern.Unknown + : SelectedPolarPattern; + if (pattern != AVAudioDataSourcePolarPattern.Unknown && !source.SetPreferredPolarPattern(pattern, out NSError? patternError)) + throw new InvalidOperationException(patternError.LocalizedDescription); + } + if (session.InputNumberOfChannels != CaptureChannels) + throw new InvalidOperationException($"The selected input activated with {session.InputNumberOfChannels} channel(s), not {CaptureChannels}."); } private void SaveAndReconfigure() diff --git a/clients/windows/VoiceCat.Windows/AudioControls.cs b/clients/windows/VoiceCat.Windows/AudioControls.cs index 9990c4f..e862355 100644 --- a/clients/windows/VoiceCat.Windows/AudioControls.cs +++ b/clients/windows/VoiceCat.Windows/AudioControls.cs @@ -14,8 +14,9 @@ public sealed partial class VoiceCatClient StreamInfo? stream = null; try { - stream = core.StartStreamAsync((StreamKind)kind, label).GetAwaiter().GetResult(); - local[stream.StreamId] = new(stream, external); + int captureChannels = kind == VcStreamKind.Mic ? 1 : 2; + stream = core.StartStreamAsync((StreamKind)kind, label, captureChannels).GetAwaiter().GetResult(); + local[stream.StreamId] = new(stream, external, captureChannels); if (!external && backend is not null) captures[stream.StreamId] = Capture(stream); return (VcResult.Ok, stream.StreamId); } diff --git a/clients/windows/VoiceCat.Windows/VoiceCatClient.cs b/clients/windows/VoiceCat.Windows/VoiceCatClient.cs index 5881173..0d681dd 100644 --- a/clients/windows/VoiceCat.Windows/VoiceCatClient.cs +++ b/clients/windows/VoiceCat.Windows/VoiceCatClient.cs @@ -20,12 +20,12 @@ public sealed partial class VoiceCatClient : IDisposable private readonly Dictionary devices = []; private readonly Dictionary remoteStreams = []; private readonly Dictionary talkState = []; - private sealed class LocalStream(StreamInfo info, bool external) + private sealed class LocalStream(StreamInfo info, bool external, int captureChannels) { internal readonly uint Alias = info.StreamId; internal StreamInfo Info = info; internal readonly bool External = external; - internal int CaptureChannels = 1; + internal int CaptureChannels = captureChannels; internal int Restarting; } private readonly ConcurrentDictionary local = new(); diff --git a/tests/VoiceCat.Tests/WindowsManagedClientTests.cs b/tests/VoiceCat.Tests/WindowsManagedClientTests.cs index de86e25..68aac28 100644 --- a/tests/VoiceCat.Tests/WindowsManagedClientTests.cs +++ b/tests/VoiceCat.Tests/WindowsManagedClientTests.cs @@ -55,4 +55,42 @@ public class WindowsManagedClientTests Assert.Equal(VcResult.Ok, alice.StopStream(a.StreamId)); Assert.Empty(alice.ManagedClient.LocalStreams); } + + [Fact] + public async Task WindowsScreenAudioPreservesDistinctStereoChannels() + { + await using var fixture = new ServerFixture(); + using (var accounts = new AccountStore(Path.Combine(fixture.Directory, "voicecat.db"))) await accounts.CreateAccountAsync("Admin", "secret", true); + using var alice = new Client("Alice", "test", tofuStorePath: Path.Combine(fixture.Directory, "alice-screen.pins")); + using var bob = new Client("Bob", "test", tofuStorePath: Path.Combine(fixture.Directory, "bob-screen.pins")); + await Login(alice, fixture, true); await Login(bob, fixture); + await alice.ManagedClient.RequestAsync(new() { CreateChannel = new() { Channel = new() { Name = "Stereo screen", ParentId = 1, Audio = AudioEngineTests.Stream(20, true).Audio } } }); + await Until(alice, () => alice.ListChannels().Any(c => c.Name == "Stereo screen")); + uint channel = alice.ListChannels().Single(c => c.Name == "Stereo screen").Id; + Assert.Equal(VcResult.Ok, alice.JoinChannel(channel)); Assert.Equal(VcResult.Ok, bob.JoinChannel(channel)); + await Until(alice, () => alice.ListUsers().Count(u => u.ChannelId == channel) == 2); + Assert.Equal(VcResult.Ok, alice.JoinVoice()); Assert.Equal(VcResult.Ok, bob.JoinVoice()); + var screen = alice.StartStreamExternalFeed(VcStreamKind.ScreenAudio, "Screen audio"); + Assert.Equal(VcResult.Ok, screen.Result); + await Until(bob, () => bob.ManagedClient.Users.Any(u => u.Id != bob.ManagedClient.Authentication!.Self.Id && u.Streams.Any(s => s.Kind == StreamKind.StreamScreenAudio))); + long separation = 0; int receivedChannels = 0; + bob.ManagedClient.Audio.StreamPcm += (_, _, pcm, channels) => + { + receivedChannels = channels; + if (channels == 2) for (int i = 0; i < pcm.Length; i += 2) Interlocked.Add(ref separation, Math.Abs((int)pcm[i] - pcm[i + 1])); + }; + short[] stereo = new short[1920]; + for (int i = 0; i < 960; i++) + { + stereo[2 * i] = (short)(8000 * Math.Sin(i * Math.PI * 880 / 48000)); + stereo[2 * i + 1] = (short)(8000 * Math.Sin(i * Math.PI * 1320 / 48000)); + } + for (int i = 0; i < 40; i++) + { + Assert.Equal(VcResult.Ok, alice.StreamFeedPcm(screen.StreamId, stereo, 960, 2)); + alice.PumpEvents(); bob.PumpEvents(); await Task.Delay(20); + } + Assert.Equal(2, receivedChannels); + Assert.True(Interlocked.Read(ref separation) > 100_000); + } }