From 0372baf58973588bb9d96ddbfa9703d4a18ab25a Mon Sep 17 00:00:00 2001 From: Talon Date: Mon, 21 Sep 2026 14:14:15 +0200 Subject: [PATCH] Fix iOS voice capture and screen sharing --- PROGRESS.md | 5 + clients/apple/VoiceCat.iOS/AppModel.cs | 20 ++- .../apple/VoiceCat.iOS/BroadcastAudioPump.cs | 25 ++-- clients/apple/VoiceCat.iOS/Info.plist | 2 +- clients/apple/VoiceCat.iOS/IosAudioEngine.cs | 137 ++++++++++++++---- clients/apple/VoiceCat.iOS/IosAudioRouter.cs | 47 ++++-- .../apple/VoiceCat.iOS/SettingsController.cs | 5 +- src/VoiceCat.Audio/AdaptivePcmBuffer.cs | 19 ++- src/VoiceCat.Audio/AudioEngine.cs | 32 +++- src/VoiceCat.Audio/LocalStream.cs | 9 +- src/VoiceCat.Core/ClientMediaTransport.cs | 19 +-- tests/VoiceCat.Tests/AudioEngineTests.cs | 83 +++++++++++ .../PublishServerScriptTests.cs | 52 +++++++ 13 files changed, 389 insertions(+), 66 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 162755d..c87bc77 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -19,6 +19,11 @@ The source-of-truth layout is: - `native/media/` and `native/rnnoise/` — required Opus/RNNoise shim and vendored RNNoise. - `native/apple/broadcast/` — required ReplayKit upload extension and shared ring producer. +The physical-device iOS voice path now supports ReplayKit fallback and stable Apple VPIO +voice-chat capture using a paced 20 ms handoff. Stereo microphone routing remains an open device +gate: the stereo preset currently receives duplicated mono samples despite selecting the built-in +stereo capsule. + ## Release gates - Run real multi-person calls on Windows, macOS, and physical iOS hardware, including adaptive diff --git a/clients/apple/VoiceCat.iOS/AppModel.cs b/clients/apple/VoiceCat.iOS/AppModel.cs index 53ec62f..d434316 100644 --- a/clients/apple/VoiceCat.iOS/AppModel.cs +++ b/clients/apple/VoiceCat.iOS/AppModel.cs @@ -33,6 +33,7 @@ internal sealed class AppModel private bool restoreMuted; private bool restoreDeafened; private bool backgrounded; + private int diagnosticPolls; internal event Action? Changed; internal event Action? IdentityRequested; @@ -274,6 +275,13 @@ internal sealed class AppModel { VoiceCatClient? owner = client; uint stream = microphoneStream; if (owner is null || stream == 0) return; (float level, bool talking) = owner.Audio.GetLocalLevel(stream); MicrophoneLevel = level; + if (++diagnosticPolls >= 20) + { + diagnosticPolls = 0; + LocalAudioDiagnostics audio = owner.Audio.GetLocalDiagnostics(stream); + Console.Error.WriteLine($"VC_AUDIO {IosAudioEngine.Shared.CaptureDiagnostics()} cycles={audio.Cycles} starved={audio.StarvedCycles} " + + $"encoded={audio.EncodedPackets} packetDrops={audio.RejectedPackets} queued={audio.BufferedFrames}"); + } if (talking != lastTalking) { lastTalking = talking; owner.PublishStreamState(stream, talking); feedback.Play(talking ? SoundEvent.VoiceStart : SoundEvent.VoiceStop); } UIApplication.SharedApplication.BeginInvokeOnMainThread(Notify); } @@ -290,7 +298,17 @@ internal sealed class AppModel private void AddActivity(string text) { activity.Add(new(DateTime.Now, text)); if (activity.Count > 500) activity.RemoveAt(0); } private static string Name(VoiceCatClient owner, uint id) => owner.Users.FirstOrDefault(user => user.Id == id)?.Nickname ?? $"User {id}"; - private void BroadcastChanged() => UIApplication.SharedApplication.BeginInvokeOnMainThread(Notify); + private void BroadcastChanged() + { + UIApplication.SharedApplication.BeginInvokeOnMainThread(() => + { + // Presenting either system picker can replace or interrupt the app's audio session. + // Rebuild after the producer becomes active so playback and an existing mic tap are + // attached to the session that will remain in use for the broadcast. + if (ScreenSharing) IosAudioRouter.Shared.Recover("screen sharing started"); + Notify(); + }); + } private void CaptureRestoreState(VoiceCatClient owner) { diff --git a/clients/apple/VoiceCat.iOS/BroadcastAudioPump.cs b/clients/apple/VoiceCat.iOS/BroadcastAudioPump.cs index eb1385f..496084a 100644 --- a/clients/apple/VoiceCat.iOS/BroadcastAudioPump.cs +++ b/clients/apple/VoiceCat.iOS/BroadcastAudioPump.cs @@ -9,7 +9,7 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable private const uint Magic = 0x56434252, Version = 1; private const int Header = 64, Capacity = 96_000, Frame = 960; private readonly CancellationTokenSource stop = new(); - private Task worker = Task.CompletedTask; + private Thread? worker; private VoiceCatClient? client; private uint streamId; private bool active; @@ -19,15 +19,21 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable internal event Action? Changed; internal bool IsActive => active; - internal void Start(VoiceCatClient owner) { client = owner; worker = RunAsync(stop.Token); } - - private async Task RunAsync(CancellationToken token) + internal void Start(VoiceCatClient owner) { + client = owner; + worker = new Thread(Run) { IsBackground = true, Name = "VoiceCat iOS screen audio", Priority = ThreadPriority.AboveNormal }; + worker.Start(); + } + + private void Run() + { + CancellationToken token = stop.Token; while (!token.IsCancellationRequested) { - try { await DrainAsync(token); } - catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException) { } - await Task.Delay(10, token).ConfigureAwait(false); + try { DrainAsync(token).GetAwaiter().GetResult(); } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException or OperationCanceledException) { } + if (!token.IsCancellationRequested) Thread.Sleep(5); } } @@ -78,8 +84,9 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable private void SetActive(bool value) { if (active == value) return; active = value; Changed?.Invoke(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - stop.Cancel(); Interlocked.Increment(ref generation); try { await worker.ConfigureAwait(false); } catch (OperationCanceledException) { } StopStream(); SetActive(false); stop.Dispose(); + stop.Cancel(); Interlocked.Increment(ref generation); worker?.Join(); worker = null; + StopStream(); SetActive(false); stop.Dispose(); return ValueTask.CompletedTask; } } diff --git a/clients/apple/VoiceCat.iOS/Info.plist b/clients/apple/VoiceCat.iOS/Info.plist index 9faa79a..52017fd 100644 --- a/clients/apple/VoiceCat.iOS/Info.plist +++ b/clients/apple/VoiceCat.iOS/Info.plist @@ -5,7 +5,7 @@ CFBundleIdentifierme.iamtalon.voicecat LSRequiresIPhoneOS NSMicrophoneUsageDescriptionVoiceCat needs microphone access to transmit your voice in channels. - UIBackgroundModesaudio + UIBackgroundModesaudioscreen-capture UIRequiresFullScreen UIDeviceFamily12 UILaunchScreen diff --git a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs index bbc87b8..105189f 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioEngine.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioEngine.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using AVFoundation; +using UIKit; using VoiceCat.Audio; using VoiceCat.Core; @@ -7,18 +8,29 @@ namespace VoiceCat.iOS; internal sealed class IosAudioEngine { + // InstallTapOnBus bufferSize is only a request. Physical iOS hardware can deliver 4,800 + // frames per callback (observed with the voice-processing graph), so conversion storage + // must cover substantially more than the requested 960 frames without allocating in Capture. + private const int MaximumCaptureCallbackFrames = 16_384; internal static IosAudioEngine Shared { get; } = new(); private readonly AdaptivePcmBuffer playbackRing = new(2, capacityFrames: 65_536); private readonly short[] renderScratch = new short[16_384]; private AVAudioEngine? engine; + private Foundation.NSObject? engineConfigurationObserver; private AVAudioSourceNode? source; private AVAudioFormat? outputFormat; private VoiceCatClient? client; - private sealed record MicrophoneRoute(uint StreamId, int Channels, PcmRing Ring); + private sealed class MicrophoneRoute(uint streamId, int channels) + { + internal readonly uint StreamId = streamId; + internal readonly int Channels = channels; + internal readonly PcmRing Ring = new(131_072); + internal bool Primed; + internal int TargetFrames = 3; + } private MicrophoneRoute? microphone; private readonly short[] microphoneFrame = new short[960 * 2]; - private readonly CancellationTokenSource microphoneStop = new(); - private readonly Task microphoneWorker; + private readonly Thread microphonePump; private AVAudioFormat? microphoneFormat; private AVAudioConverter? microphoneConverter; private AVAudioPcmBuffer? convertedMicrophone; @@ -26,10 +38,15 @@ internal sealed class IosAudioEngine private AVAudioConverterInputHandler? inputProvider; private bool inputProvided; private bool tapInstalled; + private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames; internal bool IsConnected { get; private set; } internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; } - private IosAudioEngine() { microphoneWorker = PumpMicrophoneAsync(microphoneStop.Token); } + private IosAudioEngine() + { + microphonePump = new Thread(PumpMicrophone) { IsBackground = true, Name = "VoiceCat iOS microphone pacer", Priority = ThreadPriority.Highest }; + microphonePump.Start(); + } internal void StartListening(VoiceCatClient owner) { @@ -38,7 +55,7 @@ internal sealed class IosAudioEngine internal void StartMicrophone(uint streamId, int channels) { - Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2), new(131_072))); Rebuild(); + Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2))); Rebuild(); } internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); } @@ -47,13 +64,13 @@ internal sealed class IosAudioEngine if (!IsConnected) return; MicrophoneRoute? current = Volatile.Read(ref microphone); int channels = IosAudioRouter.Shared.CaptureChannels; - if (current is not null && current.Channels != channels) + if (current is not null) { // Stop the old tap before publishing a route with a different sample width. Otherwise // an in-flight callback could interpret its old converter buffer using the new width. DestroyGraph(); - client!.Audio.SetCaptureChannels(current.StreamId, channels); - Volatile.Write(ref microphone, new(current.StreamId, channels, new(131_072))); + if (current.Channels != channels) client!.Audio.SetCaptureChannels(current.StreamId, channels); + Volatile.Write(ref microphone, new(current.StreamId, channels)); } Rebuild(); } @@ -67,6 +84,16 @@ internal sealed class IosAudioEngine { DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null); var next = new AVAudioEngine(); + AVAudioInputNode? input = null; + if (route is not null) + { + // Enabling voice processing rebuilds both sides of AVAudioEngine. Do it before any + // formats are queried or nodes are connected so the graph is built from the final IO. + input = next.InputNode; + if (!input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out NSError? processingError)) + throw new InvalidOperationException(processingError.LocalizedDescription); + if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl; + } outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false); source = new(outputFormat, Render); next.AttachNode(source); @@ -76,13 +103,12 @@ internal sealed class IosAudioEngine if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription); if (route is not null) { - AVAudioInputNode input = next.InputNode; - input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out _); - if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl; - AVAudioFormat inputFormat = input.GetBusOutputFormat(0); + AVAudioFormat inputFormat = input!.GetBusOutputFormat(0); + Console.Error.WriteLine($"VC_GRAPH vpio={IosAudioRouter.Shared.UsesVoiceProcessing} requestedCh={route.Channels} " + + $"inputCh={inputFormat.ChannelCount} inputRate={inputFormat.SampleRate}"); microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true); microphoneConverter = new(inputFormat, microphoneFormat); - uint capacity = checked((uint)Math.Ceiling(4_096 * 48_000 / inputFormat.SampleRate) + 64); + uint capacity = checked((uint)Math.Ceiling(MaximumCaptureCallbackFrames * 48_000 / inputFormat.SampleRate) + 64); convertedMicrophone = new(microphoneFormat, capacity); inputProvider = ProvideInput; NSError? tapError = null; @@ -94,10 +120,21 @@ internal sealed class IosAudioEngine next.Prepare(); if (!next.StartAndReturnError(out NSError? error)) { next.Dispose(); throw new InvalidOperationException(error.LocalizedDescription); } engine = next; + engineConfigurationObserver = Foundation.NSNotificationCenter.DefaultCenter.AddObserver( + AVAudioEngine.ConfigurationChangeNotification, notification => + { + if (!ReferenceEquals(notification.Object, next)) return; + UIApplication.SharedApplication.BeginInvokeOnMainThread(() => + { + if (IsConnected && ReferenceEquals(engine, next) && !next.Running) Rebuild(); + }); + }, next); } private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time) { + Interlocked.Increment(ref captureCallbacks); + Interlocked.Add(ref capturedFrames, buffer.FrameLength); VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone); if (owner is null || route is null || buffer.FrameLength == 0) return; AVAudioConverter? converter = microphoneConverter; @@ -105,34 +142,79 @@ internal sealed class IosAudioEngine AVAudioConverterInputHandler? provider = inputProvider; if (converter is null || converted is null || provider is null) return; pendingInput = buffer; inputProvided = false; converted.FrameLength = 0; - converter.ConvertToBuffer(converted, out _, provider); + converter.ConvertToBuffer(converted, out NSError? conversionError, provider); + if (conversionError is not null) Interlocked.Increment(ref converterFailures); if (converted.FrameLength == 0) return; + Interlocked.Add(ref convertedFrames, converted.FrameLength); nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData); - if (samples != 0) route.Ring.TryWrite(new ReadOnlySpan((void*)samples, checked((int)converted.FrameLength * route.Channels))); + if (samples != 0 && ReferenceEquals(route, Volatile.Read(ref microphone))) + { + var pcm = new ReadOnlySpan((void*)samples, checked((int)converted.FrameLength * route.Channels)); + if (route.Channels == 2) + { + Interlocked.Add(ref stereoFrames, converted.FrameLength); + long different = 0; + for (int frame = 0; frame < converted.FrameLength; frame++) + if (pcm[frame * 2] != pcm[frame * 2 + 1]) different++; + Interlocked.Add(ref stereoDifferentFrames, different); + } + if (!route.Ring.TryWrite(pcm)) Interlocked.Increment(ref rejectedFeeds); + } pendingInput = null; } + internal string CaptureDiagnostics() + { + return $"callbacks={Interlocked.Exchange(ref captureCallbacks, 0)} input={Interlocked.Exchange(ref capturedFrames, 0)} " + + $"converted={Interlocked.Exchange(ref convertedFrames, 0)} feedDrops={Interlocked.Exchange(ref rejectedFeeds, 0)} " + + $"converterErrors={Interlocked.Exchange(ref converterFailures, 0)} stereo={Interlocked.Exchange(ref stereoDifferentFrames, 0)}/" + + $"{Interlocked.Exchange(ref stereoFrames, 0)}"; + } + private AVAudioBuffer ProvideInput(uint _, out AVAudioConverterInputStatus status) { if (!inputProvided && pendingInput is { } input) { inputProvided = true; status = AVAudioConverterInputStatus.HaveData; return input; } status = AVAudioConverterInputStatus.NoDataNow; return null!; } - private async Task PumpMicrophoneAsync(CancellationToken token) + private void PumpMicrophone() { - using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(10)); - while (await timer.WaitForNextTickAsync(token).ConfigureAwait(false)) + long deadline = System.Diagnostics.Stopwatch.GetTimestamp(); + while (true) { MicrophoneRoute? route = Volatile.Read(ref microphone); - if (route is null) continue; - int required = 960 * route.Channels; - while (route.Ring.Count >= required) + VoiceCatClient? owner = client; + if (route is not null && owner is not null) { - int read = route.Ring.Read(microphoneFrame.AsSpan(0, required)); - VoiceCatClient? owner = client; - if (read == required && owner is not null && ReferenceEquals(route, Volatile.Read(ref microphone))) - owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels); + int required = 960 * route.Channels; + int completeFrames = route.Ring.Count / required; + if (!route.Primed) + { + if (completeFrames >= route.TargetFrames) route.Primed = true; + } + if (route.Primed) + { + if (completeFrames == 0) + { + route.Primed = false; + route.TargetFrames = Math.Min(route.TargetFrames + 1, 6); + } + else if (route.Ring.Read(microphoneFrame.AsSpan(0, required)) == required && + ReferenceEquals(route, Volatile.Read(ref microphone)) && + !owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels)) + Interlocked.Increment(ref rejectedFeeds); + } } + deadline += System.Diagnostics.Stopwatch.Frequency / 50; + while (true) + { + long remaining = deadline - System.Diagnostics.Stopwatch.GetTimestamp(); + if (remaining <= 0) break; + double milliseconds = remaining * 1000.0 / System.Diagnostics.Stopwatch.Frequency; + if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1)); else Thread.SpinWait(64); + } + if ((deadline - System.Diagnostics.Stopwatch.GetTimestamp()) * 1000.0 / System.Diagnostics.Stopwatch.Frequency < -100) + deadline = System.Diagnostics.Stopwatch.GetTimestamp(); } } @@ -165,6 +247,11 @@ internal sealed class IosAudioEngine private void DestroyGraph() { + if (engineConfigurationObserver is { } observer) + { + Foundation.NSNotificationCenter.DefaultCenter.RemoveObserver(observer); + observer.Dispose(); engineConfigurationObserver = null; + } if (engine is { } old) { if (tapInstalled) old.InputNode.RemoveTapOnBus(0); diff --git a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs index cfa666a..890b39a 100644 --- a/clients/apple/VoiceCat.iOS/IosAudioRouter.cs +++ b/clients/apple/VoiceCat.iOS/IosAudioRouter.cs @@ -61,6 +61,11 @@ internal sealed class IosAudioRouter BluetoothMode = preset == IosAudioPreset.VoiceChat ? IosBluetoothMode.HfpVoice : IosBluetoothMode.BuiltInMicA2dp; if (preset is IosAudioPreset.StereoMicrophone or IosAudioPreset.MonoMicrophone) SelectedInputId = AVAudioSession.SharedInstance().AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic)?.UID; + else SelectedInputId = null; + // Named presets never carry an Advanced capsule selection across transitions. + // Stereo derives its data source below; mono/voice chat must clear a stale stereo one. + SelectedDataSourceId = null; SelectedPolarPattern = AVAudioDataSourcePolarPattern.Unknown; + if (preset == IosAudioPreset.VoiceChat) ForceSpeaker = true; } SaveAndReconfigure(); } @@ -103,38 +108,52 @@ internal sealed class IosAudioRouter : BluetoothMode == IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionMode.VideoRecording : AVAudioSessionMode.VoiceChat; if (!session.SetCategory(AVAudioSessionCategory.PlayAndRecord, mode, options, out NSError? categoryError)) throw new InvalidOperationException(categoryError.LocalizedDescription); session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _); + if (configureInput) ApplyInputSelection(session); if (!session.SetActive(true, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out NSError? activeError)) throw new InvalidOperationException(activeError.LocalizedDescription); - if (configureInput) ApplyInput(session); + Console.Error.WriteLine($"VC_ROUTE preset={Preset} requestedCh={CaptureChannels} sessionCh={session.InputNumberOfChannels} " + + $"preferred={session.PreferredInput?.PortName ?? "default"} dataSource={session.InputDataSource?.DataSourceName ?? "default"} " + + $"pattern={session.InputDataSource?.SelectedPolarPattern.ToString() ?? "default"}"); session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes(); } finally { applying = false; } } - private void ApplyInput(AVAudioSession session) + private void ApplyInputSelection(AVAudioSession session) { AVAudioSessionPortDescription? port = session.AvailableInputs?.FirstOrDefault(value => value.UID == SelectedInputId); if (CaptureChannels == 2) port ??= session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic); - if (port is null) return; session.SetPreferredInput(port, out _); - AVAudioSessionDataSourceDescription? source = port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId); - if (CaptureChannels == 2) source ??= port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true); + if (port is null) { if (CaptureChannels == 1) ClearStereoPolarPattern(session); return; } + AVAudioSessionDataSourceDescription? source = CaptureChannels == 2 + ? port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true) + : port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId); + if (CaptureChannels == 1 && source is null) ClearStereoPolarPattern(session); if (source is not null) { if (!port.SetPreferredDataSource(source, out NSError? portError)) throw new InvalidOperationException(portError.LocalizedDescription); - if (!session.SetInputDataSource(source, out NSError? sourceError)) throw new InvalidOperationException(sourceError.LocalizedDescription); - } - if (session.MaximumInputNumberOfChannels < CaptureChannels) - throw new InvalidOperationException($"The selected input supports at most {session.MaximumInputNumberOfChannels} channel(s), not {CaptureChannels}."); - if (!session.SetPreferredInputNumberOfChannels(CaptureChannels, out NSError? channelError)) throw new InvalidOperationException(channelError.LocalizedDescription); - if (source is not null) - { AVAudioDataSourcePolarPattern pattern = CaptureChannels == 2 ? source.SupportedPolarPatterns?.FirstOrDefault(value => value.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) ?? AVAudioDataSourcePolarPattern.Unknown : SelectedPolarPattern; if (pattern != AVAudioDataSourcePolarPattern.Unknown && !source.SetPreferredPolarPattern(pattern, out NSError? patternError)) throw new InvalidOperationException(patternError.LocalizedDescription); } - if (session.InputNumberOfChannels != CaptureChannels) - throw new InvalidOperationException($"The selected input activated with {session.InputNumberOfChannels} channel(s), not {CaptureChannels}."); + if (!session.SetPreferredInput(port, out NSError? inputError)) throw new InvalidOperationException(inputError.LocalizedDescription); + // The working Swift client deliberately does not call + // SetPreferredInputNumberOfChannels: doing so disrupts stereo + A2DP routing. The stereo + // capsule and polar pattern above cause the input node to expose its two-channel format. + if (CaptureChannels == 2 && source is not null && + !session.SetInputDataSource(source, out NSError? sourceError)) + throw new InvalidOperationException(sourceError.LocalizedDescription); + Console.Error.WriteLine($"VC_ROUTE_SELECT port={port.PortName} source={source?.DataSourceName ?? "none"} " + + $"patterns={string.Join(',', source?.SupportedPolarPatterns?.Select(value => value.ToString()) ?? [])}"); + } + + private static void ClearStereoPolarPattern(AVAudioSession session) + { + AVAudioSessionPortDescription? builtIn = session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic); + if (builtIn is null) return; + foreach (AVAudioSessionDataSourceDescription source in builtIn.DataSources ?? []) + if (source.SelectedPolarPattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) + source.SetPreferredPolarPattern(AVAudioDataSourcePolarPattern.Unknown, out _); } private void SaveAndReconfigure() diff --git a/clients/apple/VoiceCat.iOS/SettingsController.cs b/clients/apple/VoiceCat.iOS/SettingsController.cs index 8159e97..5875954 100644 --- a/clients/apple/VoiceCat.iOS/SettingsController.cs +++ b/clients/apple/VoiceCat.iOS/SettingsController.cs @@ -55,7 +55,10 @@ internal sealed class SettingsController : UITableViewController private void Slider(string title, float minimum, float maximum, float current, Action changed) { var slider = new UISlider(new CoreGraphics.CGRect(16, 48, 238, 28)) { MinValue = minimum, MaxValue = maximum, Value = current, AccessibilityLabel = title }; UIAlertController alert = UIAlertController.Create(title, null, UIAlertControllerStyle.Alert); alert.View!.AddSubview(slider); alert.AddAction(UIAlertAction.Create("Cancel", UIAlertActionStyle.Cancel, null)); alert.AddAction(UIAlertAction.Create("Apply", UIAlertActionStyle.Default, _ => changed(slider.Value))); PresentViewController(alert, true, null); } private void ShowScreenSharing() { - if (OperatingSystem.IsIOSVersionAtLeast(27)) { model.ToggleScreenAudio(); return; } + // ScreenCaptureKit is the preferred iOS 27 path, but `IsAvailable` also reflects + // policy restrictions and transient OS state. Keep the bundled ReplayKit provider as + // a compatibility path instead of reporting that capable hardware is unsupported. + if (IosScreenCapture.IsAvailable) { model.ToggleScreenAudio(); return; } #pragma warning disable CA1422 var picker = new RPSystemBroadcastPickerView(new CoreGraphics.CGRect(0, 0, 60, 60)) { PreferredExtension = IosConstants.BroadcastExtension, ShowsMicrophoneButton = false }; #pragma warning restore CA1422 diff --git a/src/VoiceCat.Audio/AdaptivePcmBuffer.cs b/src/VoiceCat.Audio/AdaptivePcmBuffer.cs index eef0acd..b3e1f6b 100644 --- a/src/VoiceCat.Audio/AdaptivePcmBuffer.cs +++ b/src/VoiceCat.Audio/AdaptivePcmBuffer.cs @@ -66,8 +66,25 @@ public sealed class AdaptivePcmBuffer } if (available <= 0) { primed = false; phase = 0; destination.Clear(); return 0; } - double correction = Math.Clamp((available - target) / (SampleRate * 2.0), -MaximumCorrection, MaximumCorrection); + // Reads observe a packetized producer. Occupancy naturally moves by roughly a callback + // or codec block even when both clocks are exactly 48 kHz; interpreting that sawtooth as + // clock drift biases the resampler fast until it periodically drains the ring. Only + // correct sustained movement outside two codec blocks of quantization headroom. + double error = available - target, positiveDeadband = requestedFrames * 2.0; + // Low occupancy is real underflow pressure and must be corrected immediately. The + // packetization sawtooth is above the target, so only the positive side needs a deadband. + double drift = error < 0 ? error : Math.Max(0, error - positiveDeadband); + double correction = Math.Clamp(drift / (SampleRate * 2.0), -MaximumCorrection, MaximumCorrection); double step = 1.0 + correction; + // A caller encoding fixed-size frames cannot use a partial read. Limit the correction + // to what the current occupancy can supply for the whole destination; if even the + // maximum slow-down cannot do that, preserve every queued sample and re-prime. + double maximumWholeReadStep = (available - phase) / requestedFrames; + if (maximumWholeReadStep < 1.0 - MaximumCorrection) + { + primed = false; phase = 0; destination.Clear(); return 0; + } + step = Math.Min(step, maximumWholeReadStep); int produced = 0, read = readFrame; for (int frame = 0; frame < requestedFrames; frame++) { diff --git a/src/VoiceCat.Audio/AudioEngine.cs b/src/VoiceCat.Audio/AudioEngine.cs index 4e2cc93..70a1fcc 100644 --- a/src/VoiceCat.Audio/AudioEngine.cs +++ b/src/VoiceCat.Audio/AudioEngine.cs @@ -4,6 +4,9 @@ using Voicecat.V1; namespace VoiceCat.Audio; +public readonly record struct LocalAudioDiagnostics(long Cycles, long StarvedCycles, long EncodedPackets, + long RejectedPackets, int BufferedFrames); + public sealed class AudioEngine : IDisposable { private readonly object gate = new(); @@ -42,7 +45,7 @@ public sealed class AudioEngine : IDisposable maintenance = MaintainAsync(); if (startWorker) { - worker = new Thread(Work) { IsBackground = true, Name = "VoiceCat managed audio" }; + worker = new Thread(Work) { IsBackground = true, Name = "VoiceCat managed audio", Priority = ThreadPriority.Highest }; worker.Start(); } } @@ -103,6 +106,12 @@ public sealed class AudioEngine : IDisposable foreach (LocalStream stream in Volatile.Read(ref routes).Local) if (stream.Info.StreamId == streamId) return (stream.Level, stream.Talking); return default; } + public LocalAudioDiagnostics GetLocalDiagnostics(uint streamId) + { + foreach (LocalStream stream in Volatile.Read(ref routes).Local) + if (stream.Info.StreamId == streamId) return stream.Diagnostics; + return default; + } public void SetLocalGain(uint streamId, float gain) { if (!float.IsFinite(gain) || gain < 0 || gain > 4) throw new ArgumentOutOfRangeException(nameof(gain)); @@ -153,13 +162,28 @@ public sealed class AudioEngine : IDisposable { ProcessCycle(); deadline += Stopwatch.Frequency / 50; - double remaining = (deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency; - if (remaining > 0) Thread.Sleep((int)Math.Ceiling(remaining)); - else if (remaining < -100) deadline = Stopwatch.GetTimestamp(); + WaitUntil(deadline); + if ((deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency < -100) + deadline = Stopwatch.GetTimestamp(); } } catch (Exception exception) { Failure = exception; stop.Cancel(); } } + + // Millisecond-rounded sleeps periodically overshoot Core Audio's hardware clock enough to + // empty its small handoff buffer. Sleep for the coarse portion, then hold the audio worker + // to the Stopwatch deadline for the final sub-millisecond interval. + private void WaitUntil(long deadline) + { + while (!stop.IsCancellationRequested) + { + long remaining = deadline - Stopwatch.GetTimestamp(); + if (remaining <= 0) return; + double milliseconds = remaining * 1000.0 / Stopwatch.Frequency; + if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1)); + else Thread.SpinWait(64); + } + } private async Task MaintainAsync() { try diff --git a/src/VoiceCat.Audio/LocalStream.cs b/src/VoiceCat.Audio/LocalStream.cs index 7d8683f..d7146fc 100644 --- a/src/VoiceCat.Audio/LocalStream.cs +++ b/src/VoiceCat.Audio/LocalStream.cs @@ -28,8 +28,11 @@ internal sealed class LocalStream : IDisposable private int feeding; private int buffered; private int starvedSamples; + private long cycles, starvedCycles, encodedPackets, rejectedPackets; private uint timestamp; private bool wasTransmitting, marker; + internal LocalAudioDiagnostics Diagnostics => new(Volatile.Read(ref cycles), Volatile.Read(ref starvedCycles), + Volatile.Read(ref encodedPackets), Volatile.Read(ref rejectedPackets), Input.CountFrames); internal bool Feed(ReadOnlySpan pcm, int channels) { @@ -67,9 +70,11 @@ internal sealed class LocalStream : IDisposable internal void Process(AudioEngine engine, EncodedVoiceSender sender) { + Interlocked.Increment(ref cycles); var input = capture.AsSpan(0, 960 * CaptureChannels); if (Input.Read(input) != input.Length) { + Interlocked.Increment(ref starvedCycles); Level = 0; Talking = false; starvedSamples += 960; if (starvedSamples >= 9600) { buffered = 0; wasTransmitting = false; } return; @@ -114,7 +119,9 @@ internal sealed class LocalStream : IDisposable { int length = encoder.Encode(wire.AsSpan(0, frame), packet); VoiceFrameFlags flags = (Info.Audio.Fec ? VoiceFrameFlags.FecPresent : VoiceFrameFlags.None) | (marker ? VoiceFrameFlags.Marker : VoiceFrameFlags.None); - sender(Info.Ssrc, timestamp, packet.AsSpan(0, length), flags); marker = false; + if (sender(Info.Ssrc, timestamp, packet.AsSpan(0, length), flags)) Interlocked.Increment(ref encodedPackets); + else Interlocked.Increment(ref rejectedPackets); + marker = false; timestamp = unchecked(timestamp + (uint)encoder.Options.SamplesPerChannel); buffered -= frame; wire.AsSpan(frame, buffered).CopyTo(wire); diff --git a/src/VoiceCat.Core/ClientMediaTransport.cs b/src/VoiceCat.Core/ClientMediaTransport.cs index 5ba91a1..70f309e 100644 --- a/src/VoiceCat.Core/ClientMediaTransport.cs +++ b/src/VoiceCat.Core/ClientMediaTransport.cs @@ -16,7 +16,7 @@ internal sealed class ClientMediaTransport : IAsyncDisposable private readonly byte[] binding = new byte[VoiceFrameHeader.Size + 16]; private readonly byte[] keepalive = new byte[VoiceFrameHeader.Size]; private readonly PacketQueue packets = new(); - private readonly Task sending; + private readonly Thread sending; private readonly Task receiving; private readonly TaskCompletionSource bound = new(TaskCreationOptions.RunContinuationsAsynchronously); internal event EncodedVoiceHandler? Received; @@ -34,12 +34,13 @@ internal sealed class ClientMediaTransport : IAsyncDisposable token.CopyTo(binding.AsSpan(VoiceFrameHeader.Size)); new VoiceFrameHeader(MediaFrameType.Keepalive, 0, 0, 0, 0, 0).Write(keepalive); receiving = ReceiveAsync(); - sending = SendAsync(); + sending = new Thread(Send) { IsBackground = true, Name = "VoiceCat UDP sender", Priority = ThreadPriority.AboveNormal }; + sending.Start(); } internal bool TrySend(VoiceFrameHeader header, ReadOnlySpan payload) => packets.TryWrite(header, payload); - private async Task SendAsync() + private void Send() { byte[] plain = new byte[1275], packet = new byte[1275 + VoiceFrameHeader.Size + MediaEncryptor.TagSize]; long nextKeepalive = 0; @@ -49,19 +50,19 @@ internal sealed class ClientMediaTransport : IAsyncDisposable { if (Environment.TickCount64 >= nextKeepalive) { - if (!bound.Task.IsCompleted) await socket.SendAsync(binding, SocketFlags.None, stop.Token).ConfigureAwait(false); - await socket.SendAsync(keepalive, SocketFlags.None, stop.Token).ConfigureAwait(false); + if (!bound.Task.IsCompleted) socket.Send(binding, SocketFlags.None); + socket.Send(keepalive, SocketFlags.None); nextKeepalive = Environment.TickCount64 + (bound.Task.IsCompleted ? 5000 : 250); } while (packets.TryRead(plain, out VoiceFrameHeader header, out int length)) { int size = crypto.Encryptor.Encrypt(header, plain.AsSpan(0, length), packet); - await socket.SendAsync(packet.AsMemory(0, size), SocketFlags.None, stop.Token).ConfigureAwait(false); + socket.Send(packet.AsSpan(0, size), SocketFlags.None); } - await Task.Delay(5, stop.Token).ConfigureAwait(false); + Thread.Sleep(1); } } - catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) + catch (Exception exception) when (exception is SocketException or ObjectDisposedException) { if (!stop.IsCancellationRequested) bound.TrySetException(exception); } finally { stop.Cancel(); } } @@ -91,7 +92,7 @@ internal sealed class ClientMediaTransport : IAsyncDisposable public async ValueTask DisposeAsync() { stop.Cancel(); socket.Dispose(); - try { await Task.WhenAll(sending, receiving).ConfigureAwait(false); } + try { sending.Join(); await receiving.ConfigureAwait(false); } finally { System.Security.Cryptography.CryptographicOperations.ZeroMemory(binding); stop.Dispose(); } } diff --git a/tests/VoiceCat.Tests/AudioEngineTests.cs b/tests/VoiceCat.Tests/AudioEngineTests.cs index 567a588..e20d635 100644 --- a/tests/VoiceCat.Tests/AudioEngineTests.cs +++ b/tests/VoiceCat.Tests/AudioEngineTests.cs @@ -7,6 +7,27 @@ namespace VoiceCat.Tests; public class AudioEngineTests { + [Fact] + public void ManagedAudioWorkerUsesDeadlineSchedulingAtRealtimePriority() + { + string source = File.ReadAllText(Path.Combine(FindRoot(), "src", "VoiceCat.Audio", "AudioEngine.cs")); + + Assert.Contains("Priority = ThreadPriority.Highest", source); + Assert.Contains("WaitUntil(deadline);", source); + Assert.DoesNotContain("Thread.Sleep((int)Math.Ceiling(remaining))", source); + } + + [Fact] + public void MediaSenderDoesNotDependOnCoalescedAsyncTimers() + { + string source = File.ReadAllText(Path.Combine(FindRoot(), "src", "VoiceCat.Core", "ClientMediaTransport.cs")); + + Assert.Contains("new Thread(Send)", source); + Assert.Contains("socket.Send(packet.AsSpan(0, size)", source); + Assert.DoesNotContain("Task.Delay(5", source); + Assert.DoesNotContain("SendAsync(packet.AsMemory", source); + } + [Theory] [InlineData(-1000)] [InlineData(1000)] @@ -29,6 +50,58 @@ public class AudioEngineTests Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); } + [Fact] + public void AdaptivePcmBufferPreservesPartialFrameAcrossCaptureCallbacks() + { + var buffer = new AdaptivePcmBuffer(1, 20); + short[] half = Enumerable.Range(0, 480).Select(value => (short)value).ToArray(); + short[] full = new short[960]; + + Assert.True(buffer.TryWrite(half)); + Assert.Equal(0, buffer.Read(full)); + Assert.Equal(480, buffer.CountFrames); + + Assert.True(buffer.TryWrite(half)); + Assert.Equal(960, buffer.Read(full)); + Assert.Equal(0, buffer.CountFrames); + Assert.Equal(half, full.AsSpan(0, 480).ToArray()); + Assert.Equal(half, full.AsSpan(480, 480).ToArray()); + } + + [Theory] + [InlineData(20)] + [InlineData(40)] + [InlineData(60)] + public void AdaptivePcmBufferSustainsIosSizedCaptureCallbacks(int bufferMilliseconds) + { + var buffer = new AdaptivePcmBuffer(1, bufferMilliseconds); + short[] callback = new short[1_024], encoded = new short[960]; + long producedFrames = 0; + int lateStarvation = 0, fullReads = 0; + + // AVAudioEngine commonly delivers 1,024 frames every 21 1/3 ms while the codec owner + // requests 960 frames every 20 ms. Replay those independent clocks for two minutes. + for (long consumerFrame = 0; consumerFrame < 48_000L * 120; consumerFrame += 960) + { + while (producedFrames <= consumerFrame) + { + for (int i = 0; i < callback.Length; i++) callback[i] = (short)(producedFrames + i); + Assert.True(buffer.TryWrite(callback)); + producedFrames += callback.Length; + } + + int read = buffer.Read(encoded); + if (read == encoded.Length) fullReads++; + // A 20 ms target initially contains one 1,024-frame callback, so the codec's + // second deadline precedes the next callback by 1.33 ms. Re-priming once during + // that first second is expected; recurring starvation is the audible defect. + else if (consumerFrame >= 48_000) lateStarvation++; + } + + Assert.Equal(0, lateStarvation); + Assert.True(fullReads > 5_000); + } + [Fact] public void OneCaptureMissDoesNotRestartTalkspurtButSustainedStarvationDoes() { @@ -40,6 +113,8 @@ public class AudioEngineTests engine.ProcessCycle(); engine.FeedPcm(1, tone, 1); engine.ProcessCycle(); Assert.Equal(2, sent.Count); Assert.True((sent[0].Flags & VoiceFrameFlags.Marker) != 0); Assert.Equal(VoiceFrameFlags.None, sent[1].Flags & VoiceFrameFlags.Marker); + LocalAudioDiagnostics diagnostic = engine.GetLocalDiagnostics(1); + Assert.Equal(3, diagnostic.Cycles); Assert.Equal(1, diagnostic.StarvedCycles); Assert.Equal(2, diagnostic.EncodedPackets); for (int i = 0; i < 10; i++) engine.ProcessCycle(); engine.FeedPcm(1, tone, 1); engine.ProcessCycle(); Assert.True((sent[^1].Flags & VoiceFrameFlags.Marker) != 0); @@ -193,6 +268,14 @@ public class AudioEngineTests Assert.True(energy > 100000); } + private static string FindRoot() + { + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "VoiceCat.slnx"))) + directory = directory.Parent; + return directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + } + private sealed class ManualAudioClock : TimeProvider { private long milliseconds; diff --git a/tests/VoiceCat.Tests/PublishServerScriptTests.cs b/tests/VoiceCat.Tests/PublishServerScriptTests.cs index 92be2e9..672afce 100644 --- a/tests/VoiceCat.Tests/PublishServerScriptTests.cs +++ b/tests/VoiceCat.Tests/PublishServerScriptTests.cs @@ -81,6 +81,58 @@ public class PublishServerScriptTests Assert.Contains("$(CURRENT_PROJECT_VERSION)", extensionManifest); } + [Fact] + public async Task IosScreenSharingDeclaresBackgroundCaptureAndKeepsReplayKitFallback() + { + string root = FindRoot(); + string manifest = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "Info.plist")); + string settings = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "SettingsController.cs")); + + Assert.Contains("screen-capture", manifest); + Assert.Contains("if (IosScreenCapture.IsAvailable)", settings); + Assert.Contains("new RPSystemBroadcastPickerView", settings); + Assert.DoesNotContain("if (OperatingSystem.IsIOSVersionAtLeast(27)) { model.ToggleScreenAudio();", settings); + } + + [Fact] + public async Task IosCaptureDoesNotDependOnCoalescedManagedTimers() + { + string root = FindRoot(); + string microphone = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs")); + string screen = await File.ReadAllTextAsync(Path.Combine( + root, "clients", "apple", "VoiceCat.iOS", "BroadcastAudioPump.cs")); + + Assert.DoesNotContain("PeriodicTimer", microphone); + Assert.Contains("owner.Audio.FeedPcm(route.StreamId", microphone); + Assert.Contains("new Thread(Run)", screen); + Assert.DoesNotContain("Task.Delay(10", screen); + Assert.Contains("MaximumCaptureCallbackFrames = 16_384", microphone); + Assert.DoesNotContain("Math.Ceiling(4_096 * 48_000", microphone); + } + + [Fact] + public async Task IosAudioCommitsInputRouteBeforeChannelsAndVoiceProcessingBeforeConnections() + { + string root = FindRoot(); + string router = await File.ReadAllTextAsync(Path.Combine(root, "clients", "apple", "VoiceCat.iOS", "IosAudioRouter.cs")); + string engine = await File.ReadAllTextAsync(Path.Combine(root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs")); + + Assert.True(router.IndexOf("ApplyInputSelection(session)", StringComparison.Ordinal) < + router.IndexOf("SetActive(true", StringComparison.Ordinal)); + Assert.DoesNotContain("session.SetPreferredInputNumberOfChannels", router); + Assert.DoesNotContain("MaximumInputNumberOfChannels", router); + Assert.Contains("SelectedDataSourceId = null; SelectedPolarPattern = AVAudioDataSourcePolarPattern.Unknown", router); + Assert.Contains("ClearStereoPolarPattern(session)", router); + Assert.Contains("CaptureChannels == 2\n ? port.DataSources?.FirstOrDefault", router); + Assert.True(engine.IndexOf("SetVoiceProcessingEnabled", StringComparison.Ordinal) < + engine.IndexOf("next.Connect(source", StringComparison.Ordinal)); + Assert.Contains("AVAudioEngine.ConfigurationChangeNotification", engine); + Assert.Contains("ReferenceEquals(engine, next) && !next.Running", engine); + } + [Fact] public async Task DefaultPublishTargetsWindowsAndLinuxWhileRuntimeCanSelectOne() {