From 42e3bbe14c45f1cb0a435d4b3ac5d2f15de52180 Mon Sep 17 00:00:00 2001 From: Talon Date: Sat, 19 Sep 2026 20:09:00 +0200 Subject: [PATCH] Keep managed iOS audio active in background --- PROGRESS.md | 8 +++-- clients/apple/dotnet/README.md | 12 +++++-- clients/apple/dotnet/VoiceCat.iOS/AppModel.cs | 34 ++++++++++++++++++ .../dotnet/VoiceCat.iOS/BroadcastAudioPump.cs | 11 +++++- .../dotnet/VoiceCat.iOS/IosAudioEngine.cs | 5 +++ .../dotnet/VoiceCat.iOS/IosAudioRouter.cs | 36 ++++++++++++++++--- .../dotnet/VoiceCat.iOS/IosScreenCapture.cs | 4 ++- .../dotnet/VoiceCat.iOS/SceneDelegate.cs | 8 ++++- .../dotnet/VoiceCat.iOS/SettingsController.cs | 4 +-- .../VoiceCat.iOS/native/ios_screen_capture.m | 5 +-- 10 files changed, 112 insertions(+), 15 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index c852c54..b1d0ee0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -23,8 +23,12 @@ up instantly. Newest status at the top. administration, persistent VAD/PTT/always-on voice controls, event feedback, advanced iOS audio routing, reconnect restoration, and an always-visible voice bar. iOS 27 uses a small dynamically loaded ScreenCaptureKit audio bridge; iOS 18–26 retain ReplayKit, with both producers - feeding the frozen ring ABI. Added physical-device build/deploy wrappers and expanded the - managed administration round-trip test. Corrected the historical App Group and extension + feeding the frozen ring ABI. Scene lifecycle handling now preserves the active PlayAndRecord + graph in the background and recovers it on foreground activation, interruption, hardware-route + changes and media-service resets. The iOS 27 Settings action now switches ScreenCaptureKit audio + on and off, including cancellation of stale stream negotiation after rapid switching; the native + bridge also tears down its picker observer and active state. Added physical-device build/deploy + wrappers and expanded the managed administration round-trip test. Corrected the historical App Group and extension bundle IDs. CI installs the iOS workload and builds both managed Apple clients. **Next:** run the physical-device VoiceOver, route-change, background/lock and real multi-human call matrix; retain the Swift app as release oracle until those observable gates pass. diff --git a/clients/apple/dotnet/README.md b/clients/apple/dotnet/README.md index d30075d..4e934f5 100644 --- a/clients/apple/dotnet/README.md +++ b/clients/apple/dotnet/README.md @@ -40,8 +40,16 @@ clients/apple/dotnet/deploy-ios-device.sh --device "My iPhone" --configuration D The build stages the verified app at `dist/ios-managed-device/VoiceCat.iOS.app`. Pass `--no-build` to the deployment script for quick reinstall cycles. On iOS 18–26, screen audio uses the retained ReplayKit extension. On iOS 27 and newer, the host uses a small dynamically -loaded ScreenCaptureKit bridge and writes the same versioned ring; this keeps one managed consumer and -allows the app to remain launchable on older supported systems. +loaded ScreenCaptureKit bridge and writes the same versioned ring; the Settings row toggles +that capture on and off. This keeps one managed consumer and allows the app to remain launchable +on older supported systems. + +While joined to voice, the active `PlayAndRecord` session and audio engine remain running when +the scene backgrounds or the device locks, which keeps microphone capture, peer playback and the +screen-audio pump eligible for the declared `audio` background mode. Foreground activation, +hardware route changes, audio interruptions and media-service resets revalidate or rebuild the +audio graph. Validate this behavior on hardware: iOS simulator lifecycle transitions do not prove +background execution, lock-screen routing or Bluetooth recovery. Profiles, TOFU state, passwords and the ReplayKit ring use the signed App Group `group.me.iamtalon.voicecat`; the managed client migrates the old app-private profile files on first use. The shared ring ABI is frozen in [`docs/broadcast-ring-format.md`](../../../docs/broadcast-ring-format.md). diff --git a/clients/apple/dotnet/VoiceCat.iOS/AppModel.cs b/clients/apple/dotnet/VoiceCat.iOS/AppModel.cs index cba8dd2..77a7d98 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/AppModel.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/AppModel.cs @@ -32,6 +32,7 @@ internal sealed class AppModel private uint restoreChannel; private bool restoreMuted; private bool restoreDeafened; + private bool backgrounded; internal event Action? Changed; internal event Action? IdentityRequested; @@ -44,6 +45,7 @@ internal sealed class AppModel internal bool IsConnecting { get; private set; } internal bool VoiceJoined => microphoneStream != 0; internal bool ScreenSharing => broadcast?.IsActive == true; + internal bool IsBackgrounded => backgrounded; internal float MicrophoneLevel { get; private set; } internal string Status { get; private set; } = "Not connected"; internal uint CurrentChannelId { get; private set; } @@ -55,6 +57,27 @@ internal sealed class AppModel internal void Load() { profiles.Clear(); profiles.AddRange(storage.LoadProfiles()); settings.Load(); IosAudioRouter.Shared.Load(); Notify(); } internal void Save() { storage.SaveProfiles(profiles); settings.Save(); } + internal void DidEnterBackground() + { + backgrounded = true; Save(); + // Do not stop or rebuild AVAudioEngine here. With the `audio` background mode and an + // active PlayAndRecord session, capture, playback, media UDP, and screen-ring draining + // remain live while the scene is backgrounded or the device is locked. + } + + internal void WillEnterForeground() + { + backgrounded = false; + IosAudioRouter.Shared.Recover("foreground"); + } + + internal void DidBecomeActive() + { + backgrounded = false; + IosAudioRouter.Shared.EnsureAudio("active scene"); + Notify(); + } + internal void UpsertProfile(ServerProfile profile, string? password) { int index = profiles.FindIndex(item => item.Id == profile.Id); @@ -186,6 +209,17 @@ internal sealed class AppModel microphoneStream = stream.StreamId; IosAudioEngine.Shared.StartMicrophone(stream.StreamId, channels); feedback.Play(SoundEvent.VoiceOn); Notify(); } + internal void ToggleScreenAudio() + { + if (!OperatingSystem.IsIOSVersionAtLeast(27)) throw new PlatformNotSupportedException("Use the ReplayKit broadcast picker on this iOS version."); + if (ScreenSharing) { IosScreenCapture.Stop(); broadcast?.RequestStop(); } + else + { + if (!IsConnected || CurrentChannelId == 0) throw new InvalidOperationException("Connect and join a channel before sharing screen audio."); + IosScreenCapture.Present(); + } + } + internal void SetSelfAudio(bool muted, bool deafened) { client?.SetSelfAudioState(muted, deafened); Notify(); } internal void SetPushToTalk(bool active) { diff --git a/clients/apple/dotnet/VoiceCat.iOS/BroadcastAudioPump.cs b/clients/apple/dotnet/VoiceCat.iOS/BroadcastAudioPump.cs index f564914..eb1385f 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/BroadcastAudioPump.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/BroadcastAudioPump.cs @@ -13,6 +13,7 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable private VoiceCatClient? client; private uint streamId; private bool active; + private int generation; private readonly short[] scratch = new short[Frame * 2]; internal event Action? Changed; @@ -43,7 +44,13 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable VoiceCatClient owner = client ?? throw new IOException("Client disconnected."); if (streamId == 0) { + int startGeneration = Volatile.Read(ref generation); StreamInfo stream = await owner.StartStreamAsync(StreamKind.StreamScreenAudio, "Screen audio", 2, token).ConfigureAwait(false); + if (startGeneration != Volatile.Read(ref generation) || view.ReadUInt32(16) == 0 || !ReferenceEquals(client, owner)) + { + try { owner.StopStream(stream.StreamId); } catch (Exception exception) when (exception is IOException or InvalidOperationException) { } + return; + } streamId = stream.StreamId; view.Write(32, view.ReadUInt64(24)); SetActive(true); } ulong write = view.ReadUInt64(24), read = view.ReadUInt64(32); @@ -67,10 +74,12 @@ internal sealed class BroadcastAudioPump : IAsyncDisposable try { client.StopStream(id); } catch (Exception exception) when (exception is IOException or InvalidOperationException) { } } + internal void RequestStop() { Interlocked.Increment(ref generation); SetActive(false); } + private void SetActive(bool value) { if (active == value) return; active = value; Changed?.Invoke(); } public async ValueTask DisposeAsync() { - stop.Cancel(); try { await worker.ConfigureAwait(false); } catch (OperationCanceledException) { } StopStream(); SetActive(false); stop.Dispose(); + stop.Cancel(); Interlocked.Increment(ref generation); try { await worker.ConfigureAwait(false); } catch (OperationCanceledException) { } StopStream(); SetActive(false); stop.Dispose(); } } diff --git a/clients/apple/dotnet/VoiceCat.iOS/IosAudioEngine.cs b/clients/apple/dotnet/VoiceCat.iOS/IosAudioEngine.cs index 23a3732..fbfbcaf 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/IosAudioEngine.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/IosAudioEngine.cs @@ -43,6 +43,11 @@ internal sealed class IosAudioEngine internal void StopMicrophone() { microphoneStream = 0; Rebuild(); } internal void Reconfigure() { if (IsConnected) Rebuild(); } + internal bool EnsureRunning() + { + if (!IsConnected || engine?.Running == true) return true; + Rebuild(); return engine?.Running == true; + } private void Rebuild() { diff --git a/clients/apple/dotnet/VoiceCat.iOS/IosAudioRouter.cs b/clients/apple/dotnet/VoiceCat.iOS/IosAudioRouter.cs index ff7ee61..a413b66 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/IosAudioRouter.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/IosAudioRouter.cs @@ -33,9 +33,9 @@ internal sealed class IosAudioRouter private IosAudioRouter() { - NSNotificationCenter.DefaultCenter.AddObserver(AVAudioSession.RouteChangeNotification, _ => Recover()); + NSNotificationCenter.DefaultCenter.AddObserver(AVAudioSession.RouteChangeNotification, HandleRouteChange); NSNotificationCenter.DefaultCenter.AddObserver(AVAudioSession.InterruptionNotification, HandleInterruption); - NSNotificationCenter.DefaultCenter.AddObserver(AVAudioSession.MediaServicesWereResetNotification, _ => Recover()); + NSNotificationCenter.DefaultCenter.AddObserver(AVAudioSession.MediaServicesWereResetNotification, _ => Recover("media services reset")); } internal void Load() @@ -131,10 +131,38 @@ internal sealed class IosAudioRouter private void Set(string key, string? value) { if (value is null) defaults.RemoveObject(key); else defaults.SetString(value, key); } internal void Deactivate() => AVAudioSession.SharedInstance().SetActive(false, AVAudioSessionSetActiveOptions.NotifyOthersOnDeactivation, out _); - private void Recover() { RefreshRoutes(); if (IosAudioEngine.Shared.IsConnected) UIApplication.SharedApplication.BeginInvokeOnMainThread(IosAudioEngine.Shared.Reconfigure); } + internal void EnsureAudio(string reason) + { + if (!IosAudioEngine.Shared.IsConnected) return; + try { IosAudioEngine.Shared.EnsureRunning(); } + catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); } + } + + internal void Recover(string reason) + { + RefreshRoutes(); + if (!IosAudioEngine.Shared.IsConnected) return; + UIApplication.SharedApplication.BeginInvokeOnMainThread(() => + { + try { IosAudioEngine.Shared.Reconfigure(); } + catch (Exception exception) { System.Diagnostics.Debug.WriteLine($"Audio recovery ({reason}) failed: {exception}"); } + }); + } + private void HandleRouteChange(NSNotification note) + { + NSNumber? value = note.UserInfo?[new NSString("AVAudioSessionRouteChangeReasonKey")] as NSNumber; + AVAudioSessionRouteChangeReason reason = (AVAudioSessionRouteChangeReason)(value?.UInt32Value ?? 0); + RefreshRoutes(); + if (reason is AVAudioSessionRouteChangeReason.CategoryChange + or AVAudioSessionRouteChangeReason.Override + or AVAudioSessionRouteChangeReason.RouteConfigurationChange) + return; + Recover($"route change ({reason})"); + } private void HandleInterruption(NSNotification note) { NSNumber? type = note.UserInfo?[new NSString("AVAudioSessionInterruptionTypeKey")] as NSNumber; - if ((AVAudioSessionInterruptionType)(type?.UInt32Value ?? 0) == AVAudioSessionInterruptionType.Ended) Recover(); + AVAudioSessionInterruptionType interruption = (AVAudioSessionInterruptionType)(type?.UInt32Value ?? 0); + if (interruption == AVAudioSessionInterruptionType.Ended) Recover("interruption ended"); } } diff --git a/clients/apple/dotnet/VoiceCat.iOS/IosScreenCapture.cs b/clients/apple/dotnet/VoiceCat.iOS/IosScreenCapture.cs index c90271b..a458555 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/IosScreenCapture.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/IosScreenCapture.cs @@ -11,9 +11,11 @@ internal static partial class IosScreenCapture NSUrl? root = NSFileManager.DefaultManager.GetContainerUrl(IosConstants.AppGroup); if (root?.Path is null) throw new InvalidOperationException("The VoiceCat App Group is unavailable."); string path = Path.Combine(root.Path, "voicecat", "broadcast_audio.ring"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); - if (Available() == 0) throw new InvalidOperationException("Screen audio sharing is unavailable on this device."); PresentNative(path); + if (!IsAvailable) throw new InvalidOperationException("Screen audio sharing is unavailable on this device."); PresentNative(path); } + internal static bool IsAvailable => OperatingSystem.IsIOSVersionAtLeast(27) && Available() != 0; + [LibraryImport("__Internal", EntryPoint = "vc_ios_screen_capture_available")] private static partial int Available(); [LibraryImport("__Internal", EntryPoint = "vc_ios_screen_capture_present", StringMarshalling = StringMarshalling.Utf8)] diff --git a/clients/apple/dotnet/VoiceCat.iOS/SceneDelegate.cs b/clients/apple/dotnet/VoiceCat.iOS/SceneDelegate.cs index 1fe119a..395319c 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/SceneDelegate.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/SceneDelegate.cs @@ -18,5 +18,11 @@ internal sealed class SceneDelegate : UIResponder, IUIWindowSceneDelegate } [Export("sceneDidEnterBackground:")] - public void DidEnterBackground(UIScene scene) => AppModel.Shared.Save(); + public void DidEnterBackground(UIScene scene) => AppModel.Shared.DidEnterBackground(); + + [Export("sceneWillEnterForeground:")] + public void WillEnterForeground(UIScene scene) => AppModel.Shared.WillEnterForeground(); + + [Export("sceneDidBecomeActive:")] + public void DidBecomeActive(UIScene scene) => AppModel.Shared.DidBecomeActive(); } diff --git a/clients/apple/dotnet/VoiceCat.iOS/SettingsController.cs b/clients/apple/dotnet/VoiceCat.iOS/SettingsController.cs index 5fb52fc..3095526 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/SettingsController.cs +++ b/clients/apple/dotnet/VoiceCat.iOS/SettingsController.cs @@ -21,7 +21,7 @@ internal sealed class SettingsController : UITableViewController UITableViewCell cell = tableView.DequeueReusableCell("setting", path); cell.AccessoryView = null; cell.Accessory = UITableViewCellAccessory.None; string title = path.Section switch { - 0 => path.Row switch { 0 => "Audio preset", 1 => "Speaker output", 2 => "Advanced audio", _ => "Share screen audio" }, + 0 => path.Row switch { 0 => "Audio preset", 1 => "Speaker output", 2 => "Advanced audio", _ => model.ScreenSharing ? "Stop sharing screen audio" : "Share screen audio" }, 1 => path.Row switch { 0 => "Input mode", 1 => $"VAD threshold: {model.Settings.VadThreshold:0.000}", 2 => $"Microphone volume: {model.Settings.InputGain:P0}", _ => "Microphone noise reduction" }, 2 => path.Row switch { 0 => "Event sounds", 1 => $"Sound volume: {model.Settings.EventVolume:P0}", 2 => "Speak events", 3 => "Own voice activity sounds", _ => "Push-to-talk cue" }, 3 => "Manage accounts", _ => path.Row == 0 ? "Disconnect" : "VoiceCat 0.0.1" @@ -54,7 +54,7 @@ internal sealed class SettingsController : UITableViewController private void Slider(string title, float minimum, float maximum, float current, Action changed) { var slider = new UISlider(new CoreGraphics.CGRect(16, 48, 238, 28)) { MinValue = minimum, MaxValue = maximum, Value = current, AccessibilityLabel = title }; UIAlertController alert = UIAlertController.Create(title, null, UIAlertControllerStyle.Alert); alert.View!.AddSubview(slider); alert.AddAction(UIAlertAction.Create("Cancel", UIAlertActionStyle.Cancel, null)); alert.AddAction(UIAlertAction.Create("Apply", UIAlertActionStyle.Default, _ => changed(slider.Value))); PresentViewController(alert, true, null); } private void ShowScreenSharing() { - if (OperatingSystem.IsIOSVersionAtLeast(27)) { IosScreenCapture.Present(); return; } + if (OperatingSystem.IsIOSVersionAtLeast(27)) { model.ToggleScreenAudio(); return; } #pragma warning disable CA1422 var picker = new RPSystemBroadcastPickerView(new CoreGraphics.CGRect(0, 0, 60, 60)) { PreferredExtension = IosConstants.BroadcastExtension, ShowsMicrophoneButton = false }; #pragma warning restore CA1422 diff --git a/clients/apple/dotnet/VoiceCat.iOS/native/ios_screen_capture.m b/clients/apple/dotnet/VoiceCat.iOS/native/ios_screen_capture.m index 4b17073..ffa2414 100644 --- a/clients/apple/dotnet/VoiceCat.iOS/native/ios_screen_capture.m +++ b/clients/apple/dotnet/VoiceCat.iOS/native/ios_screen_capture.m @@ -70,11 +70,12 @@ static SCContentSharingPicker *VCSharedPicker(void) { SCContentSharingPickerConfiguration *configuration = [configurationClass new]; if (!picker || !configuration) return; configuration.showsMicrophoneControl = NO; configuration.showsCameraControl = NO; picker.defaultConfiguration = configuration; - [picker addObserver:self]; picker.active = YES; [picker presentPickerUsingContentStyle:SCShareableContentStyleDisplay]; + [picker removeObserver:self]; [picker addObserver:self]; picker.active = YES; [picker presentPickerUsingContentStyle:SCShareableContentStyleDisplay]; } - (void)stop { - [self.stream stopCaptureWithCompletionHandler:^(__unused NSError *error) {}]; self.stream = nil; [self setRingActive:NO]; + [self.stream stopCaptureWithCompletionHandler:^(__unused NSError *error) {}]; self.stream = nil; + SCContentSharingPicker *picker = VCSharedPicker(); [picker removeObserver:self]; picker.active = NO; [self setRingActive:NO]; } - (void)contentSharingPicker:(SCContentSharingPicker *)picker didUpdateWithFilter:(SCContentFilter *)filter forStream:(SCStream *)stream API_AVAILABLE(ios(27.0)) {