Fix iOS voice capture and screen sharing
Build and test / test (macos-latest) (push) Canceled after 0s
Build and test / test (ubuntu-24.04) (push) Canceled after 0s
Build and test / test (windows-latest) (push) Canceled after 0s
Build and test / apple-client (push) Canceled after 0s

This commit is contained in:
2026-09-21 14:14:15 +02:00
parent 35617e9708
commit 0372baf589
13 changed files with 389 additions and 66 deletions
+5
View File
@@ -19,6 +19,11 @@ The source-of-truth layout is:
- `native/media/` and `native/rnnoise/` — required Opus/RNNoise shim and vendored RNNoise.
- `native/apple/broadcast/` — required ReplayKit upload extension and shared ring producer.
The physical-device iOS voice path now supports ReplayKit fallback and stable Apple VPIO
voice-chat capture using a paced 20 ms handoff. Stereo microphone routing remains an open device
gate: the stereo preset currently receives duplicated mono samples despite selecting the built-in
stereo capsule.
## Release gates
- Run real multi-person calls on Windows, macOS, and physical iOS hardware, including adaptive
+19 -1
View File
@@ -33,6 +33,7 @@ internal sealed class AppModel
private bool restoreMuted;
private bool restoreDeafened;
private bool backgrounded;
private int diagnosticPolls;
internal event Action? Changed;
internal event Action<ServerIdentityChallenge>? IdentityRequested;
@@ -274,6 +275,13 @@ internal sealed class AppModel
{
VoiceCatClient? owner = client; uint stream = microphoneStream; if (owner is null || stream == 0) return;
(float level, bool talking) = owner.Audio.GetLocalLevel(stream); MicrophoneLevel = level;
if (++diagnosticPolls >= 20)
{
diagnosticPolls = 0;
LocalAudioDiagnostics audio = owner.Audio.GetLocalDiagnostics(stream);
Console.Error.WriteLine($"VC_AUDIO {IosAudioEngine.Shared.CaptureDiagnostics()} cycles={audio.Cycles} starved={audio.StarvedCycles} " +
$"encoded={audio.EncodedPackets} packetDrops={audio.RejectedPackets} queued={audio.BufferedFrames}");
}
if (talking != lastTalking) { lastTalking = talking; owner.PublishStreamState(stream, talking); feedback.Play(talking ? SoundEvent.VoiceStart : SoundEvent.VoiceStop); }
UIApplication.SharedApplication.BeginInvokeOnMainThread(Notify);
}
@@ -290,7 +298,17 @@ internal sealed class AppModel
private void AddActivity(string text) { activity.Add(new(DateTime.Now, text)); if (activity.Count > 500) activity.RemoveAt(0); }
private static string Name(VoiceCatClient owner, uint id) => owner.Users.FirstOrDefault(user => user.Id == id)?.Nickname ?? $"User {id}";
private void BroadcastChanged() => UIApplication.SharedApplication.BeginInvokeOnMainThread(Notify);
private void BroadcastChanged()
{
UIApplication.SharedApplication.BeginInvokeOnMainThread(() =>
{
// Presenting either system picker can replace or interrupt the app's audio session.
// Rebuild after the producer becomes active so playback and an existing mic tap are
// attached to the session that will remain in use for the broadcast.
if (ScreenSharing) IosAudioRouter.Shared.Recover("screen sharing started");
Notify();
});
}
private void CaptureRestoreState(VoiceCatClient owner)
{
@@ -9,7 +9,7 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable
private const uint Magic = 0x56434252, Version = 1;
private const int Header = 64, Capacity = 96_000, Frame = 960;
private readonly CancellationTokenSource stop = new();
private Task worker = Task.CompletedTask;
private Thread? worker;
private VoiceCatClient? client;
private uint streamId;
private bool active;
@@ -19,15 +19,21 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable
internal event Action? Changed;
internal bool IsActive => active;
internal void Start(VoiceCatClient owner) { client = owner; worker = RunAsync(stop.Token); }
private async Task RunAsync(CancellationToken token)
internal void Start(VoiceCatClient owner)
{
client = owner;
worker = new Thread(Run) { IsBackground = true, Name = "VoiceCat iOS screen audio", Priority = ThreadPriority.AboveNormal };
worker.Start();
}
private void Run()
{
CancellationToken token = stop.Token;
while (!token.IsCancellationRequested)
{
try { await DrainAsync(token); }
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException) { }
await Task.Delay(10, token).ConfigureAwait(false);
try { DrainAsync(token).GetAwaiter().GetResult(); }
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException or OperationCanceledException) { }
if (!token.IsCancellationRequested) Thread.Sleep(5);
}
}
@@ -78,8 +84,9 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable
private void SetActive(bool value) { if (active == value) return; active = value; Changed?.Invoke(); }
public async ValueTask DisposeAsync()
public ValueTask DisposeAsync()
{
stop.Cancel(); Interlocked.Increment(ref generation); try { await worker.ConfigureAwait(false); } catch (OperationCanceledException) { } StopStream(); SetActive(false); stop.Dispose();
stop.Cancel(); Interlocked.Increment(ref generation); worker?.Join(); worker = null;
StopStream(); SetActive(false); stop.Dispose(); return ValueTask.CompletedTask;
}
}
+1 -1
View File
@@ -5,7 +5,7 @@
<key>CFBundleIdentifier</key><string>me.iamtalon.voicecat</string>
<key>LSRequiresIPhoneOS</key><true/>
<key>NSMicrophoneUsageDescription</key><string>VoiceCat needs microphone access to transmit your voice in channels.</string>
<key>UIBackgroundModes</key><array><string>audio</string></array>
<key>UIBackgroundModes</key><array><string>audio</string><string>screen-capture</string></array>
<key>UIRequiresFullScreen</key><false/>
<key>UIDeviceFamily</key><array><integer>1</integer><integer>2</integer></array>
<key>UILaunchScreen</key><dict/>
+112 -25
View File
@@ -1,5 +1,6 @@
using System.Runtime.InteropServices;
using AVFoundation;
using UIKit;
using VoiceCat.Audio;
using VoiceCat.Core;
@@ -7,18 +8,29 @@ namespace VoiceCat.iOS;
internal sealed class IosAudioEngine
{
// InstallTapOnBus bufferSize is only a request. Physical iOS hardware can deliver 4,800
// frames per callback (observed with the voice-processing graph), so conversion storage
// must cover substantially more than the requested 960 frames without allocating in Capture.
private const int MaximumCaptureCallbackFrames = 16_384;
internal static IosAudioEngine Shared { get; } = new();
private readonly AdaptivePcmBuffer playbackRing = new(2, capacityFrames: 65_536);
private readonly short[] renderScratch = new short[16_384];
private AVAudioEngine? engine;
private Foundation.NSObject? engineConfigurationObserver;
private AVAudioSourceNode? source;
private AVAudioFormat? outputFormat;
private VoiceCatClient? client;
private sealed record MicrophoneRoute(uint StreamId, int Channels, PcmRing Ring);
private sealed class MicrophoneRoute(uint streamId, int channels)
{
internal readonly uint StreamId = streamId;
internal readonly int Channels = channels;
internal readonly PcmRing Ring = new(131_072);
internal bool Primed;
internal int TargetFrames = 3;
}
private MicrophoneRoute? microphone;
private readonly short[] microphoneFrame = new short[960 * 2];
private readonly CancellationTokenSource microphoneStop = new();
private readonly Task microphoneWorker;
private readonly Thread microphonePump;
private AVAudioFormat? microphoneFormat;
private AVAudioConverter? microphoneConverter;
private AVAudioPcmBuffer? convertedMicrophone;
@@ -26,10 +38,15 @@ internal sealed class IosAudioEngine
private AVAudioConverterInputHandler? inputProvider;
private bool inputProvided;
private bool tapInstalled;
private long captureCallbacks, capturedFrames, convertedFrames, rejectedFeeds, converterFailures, stereoFrames, stereoDifferentFrames;
internal bool IsConnected { get; private set; }
internal int BufferMilliseconds { get => playbackRing.BufferMilliseconds; set => playbackRing.BufferMilliseconds = value; }
private IosAudioEngine() { microphoneWorker = PumpMicrophoneAsync(microphoneStop.Token); }
private IosAudioEngine()
{
microphonePump = new Thread(PumpMicrophone) { IsBackground = true, Name = "VoiceCat iOS microphone pacer", Priority = ThreadPriority.Highest };
microphonePump.Start();
}
internal void StartListening(VoiceCatClient owner)
{
@@ -38,7 +55,7 @@ internal sealed class IosAudioEngine
internal void StartMicrophone(uint streamId, int channels)
{
Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2), new(131_072))); Rebuild();
Volatile.Write(ref microphone, new(streamId, Math.Clamp(channels, 1, 2))); Rebuild();
}
internal void StopMicrophone() { Volatile.Write(ref microphone, null); Rebuild(); }
@@ -47,13 +64,13 @@ internal sealed class IosAudioEngine
if (!IsConnected) return;
MicrophoneRoute? current = Volatile.Read(ref microphone);
int channels = IosAudioRouter.Shared.CaptureChannels;
if (current is not null && current.Channels != channels)
if (current is not null)
{
// Stop the old tap before publishing a route with a different sample width. Otherwise
// an in-flight callback could interpret its old converter buffer using the new width.
DestroyGraph();
client!.Audio.SetCaptureChannels(current.StreamId, channels);
Volatile.Write(ref microphone, new(current.StreamId, channels, new(131_072)));
if (current.Channels != channels) client!.Audio.SetCaptureChannels(current.StreamId, channels);
Volatile.Write(ref microphone, new(current.StreamId, channels));
}
Rebuild();
}
@@ -67,6 +84,16 @@ internal sealed class IosAudioEngine
{
DestroyGraph(); MicrophoneRoute? route = Volatile.Read(ref microphone); IosAudioRouter.Shared.Apply(route is not null);
var next = new AVAudioEngine();
AVAudioInputNode? input = null;
if (route is not null)
{
// Enabling voice processing rebuilds both sides of AVAudioEngine. Do it before any
// formats are queried or nodes are connected so the graph is built from the final IO.
input = next.InputNode;
if (!input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out NSError? processingError))
throw new InvalidOperationException(processingError.LocalizedDescription);
if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl;
}
outputFormat = new(AVAudioCommonFormat.PCMFloat32, 48_000, 2, false);
source = new(outputFormat, Render);
next.AttachNode(source);
@@ -76,13 +103,12 @@ internal sealed class IosAudioEngine
if (connectionError is not null) throw new InvalidOperationException(connectionError.LocalizedDescription);
if (route is not null)
{
AVAudioInputNode input = next.InputNode;
input.SetVoiceProcessingEnabled(IosAudioRouter.Shared.UsesVoiceProcessing, out _);
if (IosAudioRouter.Shared.UsesVoiceProcessing) input.VoiceProcessingAgcEnabled = IosAudioRouter.Shared.AutomaticGainControl;
AVAudioFormat inputFormat = input.GetBusOutputFormat(0);
AVAudioFormat inputFormat = input!.GetBusOutputFormat(0);
Console.Error.WriteLine($"VC_GRAPH vpio={IosAudioRouter.Shared.UsesVoiceProcessing} requestedCh={route.Channels} " +
$"inputCh={inputFormat.ChannelCount} inputRate={inputFormat.SampleRate}");
microphoneFormat = new(AVAudioCommonFormat.PCMInt16, 48_000, (uint)route.Channels, true);
microphoneConverter = new(inputFormat, microphoneFormat);
uint capacity = checked((uint)Math.Ceiling(4_096 * 48_000 / inputFormat.SampleRate) + 64);
uint capacity = checked((uint)Math.Ceiling(MaximumCaptureCallbackFrames * 48_000 / inputFormat.SampleRate) + 64);
convertedMicrophone = new(microphoneFormat, capacity);
inputProvider = ProvideInput;
NSError? tapError = null;
@@ -94,10 +120,21 @@ internal sealed class IosAudioEngine
next.Prepare();
if (!next.StartAndReturnError(out NSError? error)) { next.Dispose(); throw new InvalidOperationException(error.LocalizedDescription); }
engine = next;
engineConfigurationObserver = Foundation.NSNotificationCenter.DefaultCenter.AddObserver(
AVAudioEngine.ConfigurationChangeNotification, notification =>
{
if (!ReferenceEquals(notification.Object, next)) return;
UIApplication.SharedApplication.BeginInvokeOnMainThread(() =>
{
if (IsConnected && ReferenceEquals(engine, next) && !next.Running) Rebuild();
});
}, next);
}
private unsafe void Capture(AVAudioPcmBuffer buffer, AVAudioTime time)
{
Interlocked.Increment(ref captureCallbacks);
Interlocked.Add(ref capturedFrames, buffer.FrameLength);
VoiceCatClient? owner = client; MicrophoneRoute? route = Volatile.Read(ref microphone);
if (owner is null || route is null || buffer.FrameLength == 0) return;
AVAudioConverter? converter = microphoneConverter;
@@ -105,34 +142,79 @@ internal sealed class IosAudioEngine
AVAudioConverterInputHandler? provider = inputProvider;
if (converter is null || converted is null || provider is null) return;
pendingInput = buffer; inputProvided = false; converted.FrameLength = 0;
converter.ConvertToBuffer(converted, out _, provider);
converter.ConvertToBuffer(converted, out NSError? conversionError, provider);
if (conversionError is not null) Interlocked.Increment(ref converterFailures);
if (converted.FrameLength == 0) return;
Interlocked.Add(ref convertedFrames, converted.FrameLength);
nint samples = Marshal.ReadIntPtr(converted.Int16ChannelData);
if (samples != 0) route.Ring.TryWrite(new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * route.Channels)));
if (samples != 0 && ReferenceEquals(route, Volatile.Read(ref microphone)))
{
var pcm = new ReadOnlySpan<short>((void*)samples, checked((int)converted.FrameLength * route.Channels));
if (route.Channels == 2)
{
Interlocked.Add(ref stereoFrames, converted.FrameLength);
long different = 0;
for (int frame = 0; frame < converted.FrameLength; frame++)
if (pcm[frame * 2] != pcm[frame * 2 + 1]) different++;
Interlocked.Add(ref stereoDifferentFrames, different);
}
if (!route.Ring.TryWrite(pcm)) Interlocked.Increment(ref rejectedFeeds);
}
pendingInput = null;
}
internal string CaptureDiagnostics()
{
return $"callbacks={Interlocked.Exchange(ref captureCallbacks, 0)} input={Interlocked.Exchange(ref capturedFrames, 0)} " +
$"converted={Interlocked.Exchange(ref convertedFrames, 0)} feedDrops={Interlocked.Exchange(ref rejectedFeeds, 0)} " +
$"converterErrors={Interlocked.Exchange(ref converterFailures, 0)} stereo={Interlocked.Exchange(ref stereoDifferentFrames, 0)}/" +
$"{Interlocked.Exchange(ref stereoFrames, 0)}";
}
private AVAudioBuffer ProvideInput(uint _, out AVAudioConverterInputStatus status)
{
if (!inputProvided && pendingInput is { } input) { inputProvided = true; status = AVAudioConverterInputStatus.HaveData; return input; }
status = AVAudioConverterInputStatus.NoDataNow; return null!;
}
private async Task PumpMicrophoneAsync(CancellationToken token)
private void PumpMicrophone()
{
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(10));
while (await timer.WaitForNextTickAsync(token).ConfigureAwait(false))
long deadline = System.Diagnostics.Stopwatch.GetTimestamp();
while (true)
{
MicrophoneRoute? route = Volatile.Read(ref microphone);
if (route is null) continue;
int required = 960 * route.Channels;
while (route.Ring.Count >= required)
VoiceCatClient? owner = client;
if (route is not null && owner is not null)
{
int read = route.Ring.Read(microphoneFrame.AsSpan(0, required));
VoiceCatClient? owner = client;
if (read == required && owner is not null && ReferenceEquals(route, Volatile.Read(ref microphone)))
owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels);
int required = 960 * route.Channels;
int completeFrames = route.Ring.Count / required;
if (!route.Primed)
{
if (completeFrames >= route.TargetFrames) route.Primed = true;
}
if (route.Primed)
{
if (completeFrames == 0)
{
route.Primed = false;
route.TargetFrames = Math.Min(route.TargetFrames + 1, 6);
}
else if (route.Ring.Read(microphoneFrame.AsSpan(0, required)) == required &&
ReferenceEquals(route, Volatile.Read(ref microphone)) &&
!owner.Audio.FeedPcm(route.StreamId, microphoneFrame.AsSpan(0, required), route.Channels))
Interlocked.Increment(ref rejectedFeeds);
}
}
deadline += System.Diagnostics.Stopwatch.Frequency / 50;
while (true)
{
long remaining = deadline - System.Diagnostics.Stopwatch.GetTimestamp();
if (remaining <= 0) break;
double milliseconds = remaining * 1000.0 / System.Diagnostics.Stopwatch.Frequency;
if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1)); else Thread.SpinWait(64);
}
if ((deadline - System.Diagnostics.Stopwatch.GetTimestamp()) * 1000.0 / System.Diagnostics.Stopwatch.Frequency < -100)
deadline = System.Diagnostics.Stopwatch.GetTimestamp();
}
}
@@ -165,6 +247,11 @@ internal sealed class IosAudioEngine
private void DestroyGraph()
{
if (engineConfigurationObserver is { } observer)
{
Foundation.NSNotificationCenter.DefaultCenter.RemoveObserver(observer);
observer.Dispose(); engineConfigurationObserver = null;
}
if (engine is { } old)
{
if (tapInstalled) old.InputNode.RemoveTapOnBus(0);
+33 -14
View File
@@ -61,6 +61,11 @@ internal sealed class IosAudioRouter
BluetoothMode = preset == IosAudioPreset.VoiceChat ? IosBluetoothMode.HfpVoice : IosBluetoothMode.BuiltInMicA2dp;
if (preset is IosAudioPreset.StereoMicrophone or IosAudioPreset.MonoMicrophone)
SelectedInputId = AVAudioSession.SharedInstance().AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic)?.UID;
else SelectedInputId = null;
// Named presets never carry an Advanced capsule selection across transitions.
// Stereo derives its data source below; mono/voice chat must clear a stale stereo one.
SelectedDataSourceId = null; SelectedPolarPattern = AVAudioDataSourcePolarPattern.Unknown;
if (preset == IosAudioPreset.VoiceChat) ForceSpeaker = true;
}
SaveAndReconfigure();
}
@@ -103,38 +108,52 @@ internal sealed class IosAudioRouter
: BluetoothMode == IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionMode.VideoRecording : AVAudioSessionMode.VoiceChat;
if (!session.SetCategory(AVAudioSessionCategory.PlayAndRecord, mode, options, out NSError? categoryError)) throw new InvalidOperationException(categoryError.LocalizedDescription);
session.SetPreferredSampleRate(48_000, out _); session.SetPreferredIOBufferDuration(0.02, out _);
if (configureInput) ApplyInputSelection(session);
if (!session.SetActive(true, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out NSError? activeError)) throw new InvalidOperationException(activeError.LocalizedDescription);
if (configureInput) ApplyInput(session);
Console.Error.WriteLine($"VC_ROUTE preset={Preset} requestedCh={CaptureChannels} sessionCh={session.InputNumberOfChannels} " +
$"preferred={session.PreferredInput?.PortName ?? "default"} dataSource={session.InputDataSource?.DataSourceName ?? "default"} " +
$"pattern={session.InputDataSource?.SelectedPolarPattern.ToString() ?? "default"}");
session.OverrideOutputAudioPort(ForceSpeaker && BluetoothMode != IosBluetoothMode.BuiltInMicA2dp ? AVAudioSessionPortOverride.Speaker : AVAudioSessionPortOverride.None, out _); RefreshRoutes();
}
finally { applying = false; }
}
private void ApplyInput(AVAudioSession session)
private void ApplyInputSelection(AVAudioSession session)
{
AVAudioSessionPortDescription? port = session.AvailableInputs?.FirstOrDefault(value => value.UID == SelectedInputId);
if (CaptureChannels == 2) port ??= session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic);
if (port is null) return; session.SetPreferredInput(port, out _);
AVAudioSessionDataSourceDescription? source = port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId);
if (CaptureChannels == 2) source ??= port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true);
if (port is null) { if (CaptureChannels == 1) ClearStereoPolarPattern(session); return; }
AVAudioSessionDataSourceDescription? source = CaptureChannels == 2
? port.DataSources?.FirstOrDefault(value => value.SupportedPolarPatterns?.Any(pattern => pattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) == true)
: port.DataSources?.FirstOrDefault(value => value.DataSourceID.ToString() == SelectedDataSourceId);
if (CaptureChannels == 1 && source is null) ClearStereoPolarPattern(session);
if (source is not null)
{
if (!port.SetPreferredDataSource(source, out NSError? portError)) throw new InvalidOperationException(portError.LocalizedDescription);
if (!session.SetInputDataSource(source, out NSError? sourceError)) throw new InvalidOperationException(sourceError.LocalizedDescription);
}
if (session.MaximumInputNumberOfChannels < CaptureChannels)
throw new InvalidOperationException($"The selected input supports at most {session.MaximumInputNumberOfChannels} channel(s), not {CaptureChannels}.");
if (!session.SetPreferredInputNumberOfChannels(CaptureChannels, out NSError? channelError)) throw new InvalidOperationException(channelError.LocalizedDescription);
if (source is not null)
{
AVAudioDataSourcePolarPattern pattern = CaptureChannels == 2
? source.SupportedPolarPatterns?.FirstOrDefault(value => value.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase)) ?? AVAudioDataSourcePolarPattern.Unknown
: SelectedPolarPattern;
if (pattern != AVAudioDataSourcePolarPattern.Unknown && !source.SetPreferredPolarPattern(pattern, out NSError? patternError))
throw new InvalidOperationException(patternError.LocalizedDescription);
}
if (session.InputNumberOfChannels != CaptureChannels)
throw new InvalidOperationException($"The selected input activated with {session.InputNumberOfChannels} channel(s), not {CaptureChannels}.");
if (!session.SetPreferredInput(port, out NSError? inputError)) throw new InvalidOperationException(inputError.LocalizedDescription);
// The working Swift client deliberately does not call
// SetPreferredInputNumberOfChannels: doing so disrupts stereo + A2DP routing. The stereo
// capsule and polar pattern above cause the input node to expose its two-channel format.
if (CaptureChannels == 2 && source is not null &&
!session.SetInputDataSource(source, out NSError? sourceError))
throw new InvalidOperationException(sourceError.LocalizedDescription);
Console.Error.WriteLine($"VC_ROUTE_SELECT port={port.PortName} source={source?.DataSourceName ?? "none"} " +
$"patterns={string.Join(',', source?.SupportedPolarPatterns?.Select(value => value.ToString()) ?? [])}");
}
private static void ClearStereoPolarPattern(AVAudioSession session)
{
AVAudioSessionPortDescription? builtIn = session.AvailableInputs?.FirstOrDefault(value => value.PortType == AVAudioSession.PortBuiltInMic);
if (builtIn is null) return;
foreach (AVAudioSessionDataSourceDescription source in builtIn.DataSources ?? [])
if (source.SelectedPolarPattern.ToString().Contains("Stereo", StringComparison.OrdinalIgnoreCase))
source.SetPreferredPolarPattern(AVAudioDataSourcePolarPattern.Unknown, out _);
}
private void SaveAndReconfigure()
@@ -55,7 +55,10 @@ internal sealed class SettingsController : UITableViewController
private void Slider(string title, float minimum, float maximum, float current, Action<float> changed) { var slider = new UISlider(new CoreGraphics.CGRect(16, 48, 238, 28)) { MinValue = minimum, MaxValue = maximum, Value = current, AccessibilityLabel = title }; UIAlertController alert = UIAlertController.Create(title, null, UIAlertControllerStyle.Alert); alert.View!.AddSubview(slider); alert.AddAction(UIAlertAction.Create("Cancel", UIAlertActionStyle.Cancel, null)); alert.AddAction(UIAlertAction.Create("Apply", UIAlertActionStyle.Default, _ => changed(slider.Value))); PresentViewController(alert, true, null); }
private void ShowScreenSharing()
{
if (OperatingSystem.IsIOSVersionAtLeast(27)) { model.ToggleScreenAudio(); return; }
// ScreenCaptureKit is the preferred iOS 27 path, but `IsAvailable` also reflects
// policy restrictions and transient OS state. Keep the bundled ReplayKit provider as
// a compatibility path instead of reporting that capable hardware is unsupported.
if (IosScreenCapture.IsAvailable) { model.ToggleScreenAudio(); return; }
#pragma warning disable CA1422
var picker = new RPSystemBroadcastPickerView(new CoreGraphics.CGRect(0, 0, 60, 60)) { PreferredExtension = IosConstants.BroadcastExtension, ShowsMicrophoneButton = false };
#pragma warning restore CA1422
+18 -1
View File
@@ -66,8 +66,25 @@ public sealed class AdaptivePcmBuffer
}
if (available <= 0) { primed = false; phase = 0; destination.Clear(); return 0; }
double correction = Math.Clamp((available - target) / (SampleRate * 2.0), -MaximumCorrection, MaximumCorrection);
// Reads observe a packetized producer. Occupancy naturally moves by roughly a callback
// or codec block even when both clocks are exactly 48 kHz; interpreting that sawtooth as
// clock drift biases the resampler fast until it periodically drains the ring. Only
// correct sustained movement outside two codec blocks of quantization headroom.
double error = available - target, positiveDeadband = requestedFrames * 2.0;
// Low occupancy is real underflow pressure and must be corrected immediately. The
// packetization sawtooth is above the target, so only the positive side needs a deadband.
double drift = error < 0 ? error : Math.Max(0, error - positiveDeadband);
double correction = Math.Clamp(drift / (SampleRate * 2.0), -MaximumCorrection, MaximumCorrection);
double step = 1.0 + correction;
// A caller encoding fixed-size frames cannot use a partial read. Limit the correction
// to what the current occupancy can supply for the whole destination; if even the
// maximum slow-down cannot do that, preserve every queued sample and re-prime.
double maximumWholeReadStep = (available - phase) / requestedFrames;
if (maximumWholeReadStep < 1.0 - MaximumCorrection)
{
primed = false; phase = 0; destination.Clear(); return 0;
}
step = Math.Min(step, maximumWholeReadStep);
int produced = 0, read = readFrame;
for (int frame = 0; frame < requestedFrames; frame++)
{
+28 -4
View File
@@ -4,6 +4,9 @@ using Voicecat.V1;
namespace VoiceCat.Audio;
public readonly record struct LocalAudioDiagnostics(long Cycles, long StarvedCycles, long EncodedPackets,
long RejectedPackets, int BufferedFrames);
public sealed class AudioEngine : IDisposable
{
private readonly object gate = new();
@@ -42,7 +45,7 @@ public sealed class AudioEngine : IDisposable
maintenance = MaintainAsync();
if (startWorker)
{
worker = new Thread(Work) { IsBackground = true, Name = "VoiceCat managed audio" };
worker = new Thread(Work) { IsBackground = true, Name = "VoiceCat managed audio", Priority = ThreadPriority.Highest };
worker.Start();
}
}
@@ -103,6 +106,12 @@ public sealed class AudioEngine : IDisposable
foreach (LocalStream stream in Volatile.Read(ref routes).Local) if (stream.Info.StreamId == streamId) return (stream.Level, stream.Talking);
return default;
}
public LocalAudioDiagnostics GetLocalDiagnostics(uint streamId)
{
foreach (LocalStream stream in Volatile.Read(ref routes).Local)
if (stream.Info.StreamId == streamId) return stream.Diagnostics;
return default;
}
public void SetLocalGain(uint streamId, float gain)
{
if (!float.IsFinite(gain) || gain < 0 || gain > 4) throw new ArgumentOutOfRangeException(nameof(gain));
@@ -153,13 +162,28 @@ public sealed class AudioEngine : IDisposable
{
ProcessCycle();
deadline += Stopwatch.Frequency / 50;
double remaining = (deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency;
if (remaining > 0) Thread.Sleep((int)Math.Ceiling(remaining));
else if (remaining < -100) deadline = Stopwatch.GetTimestamp();
WaitUntil(deadline);
if ((deadline - Stopwatch.GetTimestamp()) * 1000.0 / Stopwatch.Frequency < -100)
deadline = Stopwatch.GetTimestamp();
}
}
catch (Exception exception) { Failure = exception; stop.Cancel(); }
}
// Millisecond-rounded sleeps periodically overshoot Core Audio's hardware clock enough to
// empty its small handoff buffer. Sleep for the coarse portion, then hold the audio worker
// to the Stopwatch deadline for the final sub-millisecond interval.
private void WaitUntil(long deadline)
{
while (!stop.IsCancellationRequested)
{
long remaining = deadline - Stopwatch.GetTimestamp();
if (remaining <= 0) return;
double milliseconds = remaining * 1000.0 / Stopwatch.Frequency;
if (milliseconds > 2) Thread.Sleep(Math.Max(1, (int)milliseconds - 1));
else Thread.SpinWait(64);
}
}
private async Task MaintainAsync()
{
try
+8 -1
View File
@@ -28,8 +28,11 @@ internal sealed class LocalStream : IDisposable
private int feeding;
private int buffered;
private int starvedSamples;
private long cycles, starvedCycles, encodedPackets, rejectedPackets;
private uint timestamp;
private bool wasTransmitting, marker;
internal LocalAudioDiagnostics Diagnostics => new(Volatile.Read(ref cycles), Volatile.Read(ref starvedCycles),
Volatile.Read(ref encodedPackets), Volatile.Read(ref rejectedPackets), Input.CountFrames);
internal bool Feed(ReadOnlySpan<short> pcm, int channels)
{
@@ -67,9 +70,11 @@ internal sealed class LocalStream : IDisposable
internal void Process(AudioEngine engine, EncodedVoiceSender sender)
{
Interlocked.Increment(ref cycles);
var input = capture.AsSpan(0, 960 * CaptureChannels);
if (Input.Read(input) != input.Length)
{
Interlocked.Increment(ref starvedCycles);
Level = 0; Talking = false; starvedSamples += 960;
if (starvedSamples >= 9600) { buffered = 0; wasTransmitting = false; }
return;
@@ -114,7 +119,9 @@ internal sealed class LocalStream : IDisposable
{
int length = encoder.Encode(wire.AsSpan(0, frame), packet);
VoiceFrameFlags flags = (Info.Audio.Fec ? VoiceFrameFlags.FecPresent : VoiceFrameFlags.None) | (marker ? VoiceFrameFlags.Marker : VoiceFrameFlags.None);
sender(Info.Ssrc, timestamp, packet.AsSpan(0, length), flags); marker = false;
if (sender(Info.Ssrc, timestamp, packet.AsSpan(0, length), flags)) Interlocked.Increment(ref encodedPackets);
else Interlocked.Increment(ref rejectedPackets);
marker = false;
timestamp = unchecked(timestamp + (uint)encoder.Options.SamplesPerChannel);
buffered -= frame;
wire.AsSpan(frame, buffered).CopyTo(wire);
+10 -9
View File
@@ -16,7 +16,7 @@ internal sealed class ClientMediaTransport : IAsyncDisposable
private readonly byte[] binding = new byte[VoiceFrameHeader.Size + 16];
private readonly byte[] keepalive = new byte[VoiceFrameHeader.Size];
private readonly PacketQueue packets = new();
private readonly Task sending;
private readonly Thread sending;
private readonly Task receiving;
private readonly TaskCompletionSource bound = new(TaskCreationOptions.RunContinuationsAsynchronously);
internal event EncodedVoiceHandler? Received;
@@ -34,12 +34,13 @@ internal sealed class ClientMediaTransport : IAsyncDisposable
token.CopyTo(binding.AsSpan(VoiceFrameHeader.Size));
new VoiceFrameHeader(MediaFrameType.Keepalive, 0, 0, 0, 0, 0).Write(keepalive);
receiving = ReceiveAsync();
sending = SendAsync();
sending = new Thread(Send) { IsBackground = true, Name = "VoiceCat UDP sender", Priority = ThreadPriority.AboveNormal };
sending.Start();
}
internal bool TrySend(VoiceFrameHeader header, ReadOnlySpan<byte> payload) => packets.TryWrite(header, payload);
private async Task SendAsync()
private void Send()
{
byte[] plain = new byte[1275], packet = new byte[1275 + VoiceFrameHeader.Size + MediaEncryptor.TagSize];
long nextKeepalive = 0;
@@ -49,19 +50,19 @@ internal sealed class ClientMediaTransport : IAsyncDisposable
{
if (Environment.TickCount64 >= nextKeepalive)
{
if (!bound.Task.IsCompleted) await socket.SendAsync(binding, SocketFlags.None, stop.Token).ConfigureAwait(false);
await socket.SendAsync(keepalive, SocketFlags.None, stop.Token).ConfigureAwait(false);
if (!bound.Task.IsCompleted) socket.Send(binding, SocketFlags.None);
socket.Send(keepalive, SocketFlags.None);
nextKeepalive = Environment.TickCount64 + (bound.Task.IsCompleted ? 5000 : 250);
}
while (packets.TryRead(plain, out VoiceFrameHeader header, out int length))
{
int size = crypto.Encryptor.Encrypt(header, plain.AsSpan(0, length), packet);
await socket.SendAsync(packet.AsMemory(0, size), SocketFlags.None, stop.Token).ConfigureAwait(false);
socket.Send(packet.AsSpan(0, size), SocketFlags.None);
}
await Task.Delay(5, stop.Token).ConfigureAwait(false);
Thread.Sleep(1);
}
}
catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException)
catch (Exception exception) when (exception is SocketException or ObjectDisposedException)
{ if (!stop.IsCancellationRequested) bound.TrySetException(exception); }
finally { stop.Cancel(); }
}
@@ -91,7 +92,7 @@ internal sealed class ClientMediaTransport : IAsyncDisposable
public async ValueTask DisposeAsync()
{
stop.Cancel(); socket.Dispose();
try { await Task.WhenAll(sending, receiving).ConfigureAwait(false); }
try { sending.Join(); await receiving.ConfigureAwait(false); }
finally { System.Security.Cryptography.CryptographicOperations.ZeroMemory(binding); stop.Dispose(); }
}
+83
View File
@@ -7,6 +7,27 @@ namespace VoiceCat.Tests;
public class AudioEngineTests
{
[Fact]
public void ManagedAudioWorkerUsesDeadlineSchedulingAtRealtimePriority()
{
string source = File.ReadAllText(Path.Combine(FindRoot(), "src", "VoiceCat.Audio", "AudioEngine.cs"));
Assert.Contains("Priority = ThreadPriority.Highest", source);
Assert.Contains("WaitUntil(deadline);", source);
Assert.DoesNotContain("Thread.Sleep((int)Math.Ceiling(remaining))", source);
}
[Fact]
public void MediaSenderDoesNotDependOnCoalescedAsyncTimers()
{
string source = File.ReadAllText(Path.Combine(FindRoot(), "src", "VoiceCat.Core", "ClientMediaTransport.cs"));
Assert.Contains("new Thread(Send)", source);
Assert.Contains("socket.Send(packet.AsSpan(0, size)", source);
Assert.DoesNotContain("Task.Delay(5", source);
Assert.DoesNotContain("SendAsync(packet.AsMemory", source);
}
[Theory]
[InlineData(-1000)]
[InlineData(1000)]
@@ -29,6 +50,58 @@ public class AudioEngineTests
Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before);
}
[Fact]
public void AdaptivePcmBufferPreservesPartialFrameAcrossCaptureCallbacks()
{
var buffer = new AdaptivePcmBuffer(1, 20);
short[] half = Enumerable.Range(0, 480).Select(value => (short)value).ToArray();
short[] full = new short[960];
Assert.True(buffer.TryWrite(half));
Assert.Equal(0, buffer.Read(full));
Assert.Equal(480, buffer.CountFrames);
Assert.True(buffer.TryWrite(half));
Assert.Equal(960, buffer.Read(full));
Assert.Equal(0, buffer.CountFrames);
Assert.Equal(half, full.AsSpan(0, 480).ToArray());
Assert.Equal(half, full.AsSpan(480, 480).ToArray());
}
[Theory]
[InlineData(20)]
[InlineData(40)]
[InlineData(60)]
public void AdaptivePcmBufferSustainsIosSizedCaptureCallbacks(int bufferMilliseconds)
{
var buffer = new AdaptivePcmBuffer(1, bufferMilliseconds);
short[] callback = new short[1_024], encoded = new short[960];
long producedFrames = 0;
int lateStarvation = 0, fullReads = 0;
// AVAudioEngine commonly delivers 1,024 frames every 21 1/3 ms while the codec owner
// requests 960 frames every 20 ms. Replay those independent clocks for two minutes.
for (long consumerFrame = 0; consumerFrame < 48_000L * 120; consumerFrame += 960)
{
while (producedFrames <= consumerFrame)
{
for (int i = 0; i < callback.Length; i++) callback[i] = (short)(producedFrames + i);
Assert.True(buffer.TryWrite(callback));
producedFrames += callback.Length;
}
int read = buffer.Read(encoded);
if (read == encoded.Length) fullReads++;
// A 20 ms target initially contains one 1,024-frame callback, so the codec's
// second deadline precedes the next callback by 1.33 ms. Re-priming once during
// that first second is expected; recurring starvation is the audible defect.
else if (consumerFrame >= 48_000) lateStarvation++;
}
Assert.Equal(0, lateStarvation);
Assert.True(fullReads > 5_000);
}
[Fact]
public void OneCaptureMissDoesNotRestartTalkspurtButSustainedStarvationDoes()
{
@@ -40,6 +113,8 @@ public class AudioEngineTests
engine.ProcessCycle();
engine.FeedPcm(1, tone, 1); engine.ProcessCycle();
Assert.Equal(2, sent.Count); Assert.True((sent[0].Flags & VoiceFrameFlags.Marker) != 0); Assert.Equal(VoiceFrameFlags.None, sent[1].Flags & VoiceFrameFlags.Marker);
LocalAudioDiagnostics diagnostic = engine.GetLocalDiagnostics(1);
Assert.Equal(3, diagnostic.Cycles); Assert.Equal(1, diagnostic.StarvedCycles); Assert.Equal(2, diagnostic.EncodedPackets);
for (int i = 0; i < 10; i++) engine.ProcessCycle();
engine.FeedPcm(1, tone, 1); engine.ProcessCycle();
Assert.True((sent[^1].Flags & VoiceFrameFlags.Marker) != 0);
@@ -193,6 +268,14 @@ public class AudioEngineTests
Assert.True(energy > 100000);
}
private static string FindRoot()
{
DirectoryInfo? directory = new(AppContext.BaseDirectory);
while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "VoiceCat.slnx")))
directory = directory.Parent;
return directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found.");
}
private sealed class ManualAudioClock : TimeProvider
{
private long milliseconds;
@@ -81,6 +81,58 @@ public class PublishServerScriptTests
Assert.Contains("<string>$(CURRENT_PROJECT_VERSION)</string>", extensionManifest);
}
[Fact]
public async Task IosScreenSharingDeclaresBackgroundCaptureAndKeepsReplayKitFallback()
{
string root = FindRoot();
string manifest = await File.ReadAllTextAsync(Path.Combine(
root, "clients", "apple", "VoiceCat.iOS", "Info.plist"));
string settings = await File.ReadAllTextAsync(Path.Combine(
root, "clients", "apple", "VoiceCat.iOS", "SettingsController.cs"));
Assert.Contains("<string>screen-capture</string>", manifest);
Assert.Contains("if (IosScreenCapture.IsAvailable)", settings);
Assert.Contains("new RPSystemBroadcastPickerView", settings);
Assert.DoesNotContain("if (OperatingSystem.IsIOSVersionAtLeast(27)) { model.ToggleScreenAudio();", settings);
}
[Fact]
public async Task IosCaptureDoesNotDependOnCoalescedManagedTimers()
{
string root = FindRoot();
string microphone = await File.ReadAllTextAsync(Path.Combine(
root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs"));
string screen = await File.ReadAllTextAsync(Path.Combine(
root, "clients", "apple", "VoiceCat.iOS", "BroadcastAudioPump.cs"));
Assert.DoesNotContain("PeriodicTimer", microphone);
Assert.Contains("owner.Audio.FeedPcm(route.StreamId", microphone);
Assert.Contains("new Thread(Run)", screen);
Assert.DoesNotContain("Task.Delay(10", screen);
Assert.Contains("MaximumCaptureCallbackFrames = 16_384", microphone);
Assert.DoesNotContain("Math.Ceiling(4_096 * 48_000", microphone);
}
[Fact]
public async Task IosAudioCommitsInputRouteBeforeChannelsAndVoiceProcessingBeforeConnections()
{
string root = FindRoot();
string router = await File.ReadAllTextAsync(Path.Combine(root, "clients", "apple", "VoiceCat.iOS", "IosAudioRouter.cs"));
string engine = await File.ReadAllTextAsync(Path.Combine(root, "clients", "apple", "VoiceCat.iOS", "IosAudioEngine.cs"));
Assert.True(router.IndexOf("ApplyInputSelection(session)", StringComparison.Ordinal) <
router.IndexOf("SetActive(true", StringComparison.Ordinal));
Assert.DoesNotContain("session.SetPreferredInputNumberOfChannels", router);
Assert.DoesNotContain("MaximumInputNumberOfChannels", router);
Assert.Contains("SelectedDataSourceId = null; SelectedPolarPattern = AVAudioDataSourcePolarPattern.Unknown", router);
Assert.Contains("ClearStereoPolarPattern(session)", router);
Assert.Contains("CaptureChannels == 2\n ? port.DataSources?.FirstOrDefault", router);
Assert.True(engine.IndexOf("SetVoiceProcessingEnabled", StringComparison.Ordinal) <
engine.IndexOf("next.Connect(source", StringComparison.Ordinal));
Assert.Contains("AVAudioEngine.ConfigurationChangeNotification", engine);
Assert.Contains("ReferenceEquals(engine, next) && !next.Running", engine);
}
[Fact]
public async Task DefaultPublishTargetsWindowsAndLinuxWhileRuntimeCanSelectOne()
{