Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix stereo capture paths

Browse Source
This commit is contained in:
Talon
2026-09-21 02:11:33 +02:00
parent 08e6c5930a
commit 9be33f357d
6 changed files with 100 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PROGRESS.md
+2 -1
View File
@@ -25,7 +25,8 @@ The source-of-truth layout is:
20/40/60 ms buffering, duration-aware DRED/FEC, and mismatched input/output endpoints.
- Complete NVDA and VoiceOver navigation/announcement passes.
- Exercise iOS background/lock, interruption, Bluetooth, route-change, ReplayKit, and iOS 27
ScreenCaptureKit paths on devices.
ScreenCaptureKit paths on devices. Verify distinct left/right capture with the stereo microphone
preset, including changing mono/stereo while joined, and Windows desktop/per-app stereo sharing.
- Complete Developer ID signing/notarization and iOS distribution signing.
- Run the published Linux container and a 30-minute-or-longer server soak.
clients/apple/VoiceCat.iOS/IosAudioEngine.cs
+34 -18
View File
@@ -14,9 +14,8 @@ internal sealed class IosAudioEngine
private AVAudioSourceNode? source;
private AVAudioFormat? outputFormat;
private VoiceCatClient? client;
private uint microphoneStream;
private int microphoneChannels = 1;
private readonly PcmRing microphoneRing = new(131_072);
private sealed record MicrophoneRoute(uint StreamId, int Channels, PcmRing Ring);
private MicrophoneRoute? microphone;
private readonly short[] microphoneFrame = new short[960 * 2];
private readonly CancellationTokenSource microphoneStop = new();
private readonly Task microphoneWorker;
@@ -39,11 +38,25 @@ internal sealed class IosAudioEngine
internal void StartMicrophone(uint streamId, int channels)
{
microphoneStream = streamId; microphoneChannels = Math.Clamp(channels, 1, 2); Rebuild();
Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2), new(131_072))); Rebuild();
}
internal void StopMicrophone() { microphoneStream = 0; Rebuild(); }
internal void Reconfigure() { if (IsConnected) Rebuild(); }
internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); }
internal void Reconfigure()
{
if (!IsConnected) return;
MicrophoneRoute? current = Volatile.Read(ref microphone);
int channels = IosAudioRouter.Shared.CaptureChannels;
if (current is not null && current.Channels != channels)
{
// Stop the old tap before publishing a route with a different sample width. Otherwise
// an in-flight callback could interpret its old converter buffer using the new width.
DestroyGraph();
client!.Audio.SetCaptureChannels(current.StreamId, channels);
Volatile.Write(ref microphone, new(current.StreamId, channels, new(131_072)));
}
Rebuild();
}
internal bool EnsureRunning()
{
if (!IsConnected || engine?.Running == true) return true;
@@ -52,7 +65,7 @@ internal sealed class IosAudioEngine
private void Rebuild()
{
DestroyGraph(); IosAudioRouter.Shared.Apply();
DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null);
var next = new AVAudioEngine();
outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false);
source = new(outputFormat, Render);
@@ -61,13 +74,13 @@ internal sealed class IosAudioEngine
if (OperatingSystem.IsIOSVersionAtLeast(27)) next.Connect(source, next.MainMixerNode, outputFormat, out connectionError);
else next.Connect(source, next.MainMixerNode, outputFormat);
if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription);
if (microphoneStream != 0)
if (route is not null)
{
AVAudioInputNode input = next.InputNode;
input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out _);
if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl;
AVAudioFormat inputFormat = input.GetBusOutputFormat(0);
microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)microphoneChannels, true);
microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true);
microphoneConverter = new(inputFormat, microphoneFormat);
uint capacity = checked((uint)Math.Ceiling(4_096 * 48_000 / inputFormat.SampleRate) + 64);
convertedMicrophone = new(microphoneFormat, capacity);
@@ -85,8 +98,8 @@ internal sealed class IosAudioEngine
private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time)
{
VoiceCatClient? owner = client; uint stream = microphoneStream;
if (owner is null || stream == 0 || buffer.FrameLength == 0) return;
VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone);
if (owner is null || route is null || buffer.FrameLength == 0) return;
AVAudioConverter? converter = microphoneConverter;
AVAudioPcmBuffer? converted = convertedMicrophone;
AVAudioConverterInputHandler? provider = inputProvider;
@@ -95,7 +108,7 @@ internal sealed class IosAudioEngine
converter.ConvertToBuffer(converted, out _, provider);
if (converted.FrameLength == 0) return;
nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData);
if (samples != 0) microphoneRing.TryWrite(new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * microphoneChannels)));
if (samples != 0) route.Ring.TryWrite(new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * route.Channels)));
pendingInput = null;
}
@@ -110,12 +123,15 @@ internal sealed class IosAudioEngine
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(10));
while (await timer.WaitForNextTickAsync(token).ConfigureAwait(false))
{
int channels = microphoneChannels, required = 960 * channels;
while (microphoneRing.Count >= required)
MicrophoneRoute? route = Volatile.Read(ref microphone);
if (route is null) continue;
int required = 960 * route.Channels;
while (route.Ring.Count >= required)
{
int read = microphoneRing.Read(microphoneFrame.AsSpan(0, required));
VoiceCatClient? owner = client; uint stream = microphoneStream;
if (read == required && owner is not null && stream != 0) owner.Audio.FeedPcm(stream, microphoneFrame.AsSpan(0, required), channels);
int read = route.Ring.Read(microphoneFrame.AsSpan(0, required));
VoiceCatClient? owner = client;
if (read == required && owner is not null && ReferenceEquals(route, Volatile.Read(ref microphone)))
owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels);
}
}
}
@@ -142,7 +158,7 @@ internal sealed class IosAudioEngine
internal void Stop()
{
IsConnected = false; microphoneStream = 0;
IsConnected = false; Volatile.Write(ref microphone, null);
if (client is { } owner) owner.Audio.MixedPcm -= ReceiveMixedPcm;
DestroyGraph(); client = null; IosAudioRouter.Shared.Deactivate();
}
clients/apple/VoiceCat.iOS/IosAudioRouter.cs
+21 -4
View File
@@ -90,7 +90,7 @@ internal sealed class IosAudioRouter
SelectedInputId ??= session.PreferredInput?.UID; Changed?.Invoke();
}
internal void Apply()
internal void Apply(bool configureInput)
{
if (applying) return; applying = true;
try
@@ -102,8 +102,9 @@ internal sealed class IosAudioRouter
AVAudioSessionMode mode = CaptureChannels == 2 ? AVAudioSessionMode.Default : MicMode == IosMicMode.Raw ? AVAudioSessionMode.Measurement
: BluetoothMode == IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionMode.VideoRecording : AVAudioSessionMode.VoiceChat;
if (!session.SetCategory(AVAudioSessionCategory.PlayAndRecord, mode, options, out NSError? categoryError)) throw new InvalidOperationException(categoryError.LocalizedDescription);
session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _); ApplyInput(session);
session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _);
if (!session.SetActive(true, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out NSError? activeError)) throw new InvalidOperationException(activeError.LocalizedDescription);
if (configureInput) ApplyInput(session);
session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes();
}
finally { applying = false; }
@@ -116,8 +117,24 @@ internal sealed class IosAudioRouter
if (port is null) return; session.SetPreferredInput(port, out _);
AVAudioSessionDataSourceDescription? source = port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId);
if (CaptureChannels == 2) source ??= port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true);
if (source is null) return; port.SetPreferredDataSource(source, out _); session.SetInputDataSource(source, out _);
if (CaptureChannels != 2 && SelectedPolarPattern != AVAudioDataSourcePolarPattern.Unknown) source.SetPreferredPolarPattern(SelectedPolarPattern, out _);
if (source is not null)
{
if (!port.SetPreferredDataSource(source, out NSError? portError)) throw new InvalidOperationException(portError.LocalizedDescription);
if (!session.SetInputDataSource(source, out NSError? sourceError)) throw new InvalidOperationException(sourceError.LocalizedDescription);
}
if (session.MaximumInputNumberOfChannels < CaptureChannels)
throw new InvalidOperationException($"The selected input supports at most {session.MaximumInputNumberOfChannels} channel(s), not {CaptureChannels}.");
if (!session.SetPreferredInputNumberOfChannels(CaptureChannels, out NSError? channelError)) throw new InvalidOperationException(channelError.LocalizedDescription);
if (source is not null)
{
AVAudioDataSourcePolarPattern pattern = CaptureChannels == 2
? source.SupportedPolarPatterns?.FirstOrDefault(value => value.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) ?? AVAudioDataSourcePolarPattern.Unknown
: SelectedPolarPattern;
if (pattern != AVAudioDataSourcePolarPattern.Unknown && !source.SetPreferredPolarPattern(pattern, out NSError? patternError))
throw new InvalidOperationException(patternError.LocalizedDescription);
}
if (session.InputNumberOfChannels != CaptureChannels)
throw new InvalidOperationException($"The selected input activated with {session.InputNumberOfChannels} channel(s), not {CaptureChannels}.");
}
private void SaveAndReconfigure()
clients/windows/VoiceCat.Windows/AudioControls.cs
+3 -2
View File
@@ -14,8 +14,9 @@ public sealed partial class VoiceCatClient
StreamInfo? stream = null;
try
{
stream = core.StartStreamAsync((StreamKind)kind, label).GetAwaiter().GetResult();
local[stream.StreamId] = new(stream, external);
int captureChannels = kind == VcStreamKind.Mic ? 1 : 2;
stream = core.StartStreamAsync((StreamKind)kind, label, captureChannels).GetAwaiter().GetResult();
local[stream.StreamId] = new(stream, external, captureChannels);
if (!external && backend is not null) captures[stream.StreamId] = Capture(stream);
return (VcResult.Ok, stream.StreamId);
}
clients/windows/VoiceCat.Windows/VoiceCatClient.cs
+2 -2
View File
@@ -20,12 +20,12 @@ public sealed partial class VoiceCatClient : IDisposable
private readonly Dictionary<uint, string?> devices = [];
private readonly Dictionary<uint, StreamSummary[]> remoteStreams = [];
private readonly Dictionary<uint, bool> talkState = [];
private sealed class LocalStream(StreamInfo info, bool external)
private sealed class LocalStream(StreamInfo info, bool external, int captureChannels)
{
internal readonly uint Alias = info.StreamId;
internal StreamInfo Info = info;
internal readonly bool External = external;
internal int CaptureChannels = 1;
internal int CaptureChannels = captureChannels;
internal int Restarting;
}
private readonly ConcurrentDictionary<uint, LocalStream> local = new();
tests/VoiceCat.Tests/WindowsManagedClientTests.cs
+38
View File
@@ -55,4 +55,42 @@ public class WindowsManagedClientTests
Assert.Equal(VcResult.Ok, alice.StopStream(a.StreamId));
Assert.Empty(alice.ManagedClient.LocalStreams);
}
[Fact]
public async Task WindowsScreenAudioPreservesDistinctStereoChannels()
{
await using var fixture = new ServerFixture();
using (var accounts = new AccountStore(Path.Combine(fixture.Directory, "voicecat.db"))) await accounts.CreateAccountAsync("Admin", "secret", true);
using var alice = new Client("Alice", "test", tofuStorePath: Path.Combine(fixture.Directory, "alice-screen.pins"));
using var bob = new Client("Bob", "test", tofuStorePath: Path.Combine(fixture.Directory, "bob-screen.pins"));
await Login(alice, fixture, true); await Login(bob, fixture);
await alice.ManagedClient.RequestAsync(new() { CreateChannel = new() { Channel = new() { Name = "Stereo screen", ParentId = 1, Audio = AudioEngineTests.Stream(20, true).Audio } } });
await Until(alice, () => alice.ListChannels().Any(c => c.Name == "Stereo screen"));
uint channel = alice.ListChannels().Single(c => c.Name == "Stereo screen").Id;
Assert.Equal(VcResult.Ok, alice.JoinChannel(channel)); Assert.Equal(VcResult.Ok, bob.JoinChannel(channel));
await Until(alice, () => alice.ListUsers().Count(u => u.ChannelId == channel) == 2);
Assert.Equal(VcResult.Ok, alice.JoinVoice()); Assert.Equal(VcResult.Ok, bob.JoinVoice());
var screen = alice.StartStreamExternalFeed(VcStreamKind.ScreenAudio, "Screen audio");
Assert.Equal(VcResult.Ok, screen.Result);
await Until(bob, () => bob.ManagedClient.Users.Any(u => u.Id != bob.ManagedClient.Authentication!.Self.Id && u.Streams.Any(s => s.Kind == StreamKind.StreamScreenAudio)));
long separation = 0; int receivedChannels = 0;
bob.ManagedClient.Audio.StreamPcm += (_, _, pcm, channels) =>
{
receivedChannels = channels;
if (channels == 2) for (int i = 0; i < pcm.Length; i += 2) Interlocked.Add(ref separation, Math.Abs((int)pcm[i] - pcm[i + 1]));
};
short[] stereo = new short[1920];
for (int i = 0; i < 960; i++)
{
stereo[2 * i] = (short)(8000 * Math.Sin(i * Math.PI * 880 / 48000));
stereo[2 * i + 1] = (short)(8000 * Math.Sin(i * Math.PI * 1320 / 48000));
}
for (int i = 0; i < 40; i++)
{
Assert.Equal(VcResult.Ok, alice.StreamFeedPcm(screen.StreamId, stereo, 960, 2));
alice.PumpEvents(); bob.PumpEvents(); await Task.Delay(20);
}
Assert.Equal(2, receivedChannels);
Assert.True(Interlocked.Read(ref separation) > 100_000);
}
}