From fb73b694d0fd9cfca04d131f06baf9109f249487 Mon Sep 17 00:00:00 2001 From: Talon Date: Mon, 22 Jun 2026 03:43:00 +0200 Subject: [PATCH] fix(ios): VPIO silent playback + quiet speaker in stereo/studio presets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two on-device bugs in the native iOS Voice-Processing path (Swift-only; no core/ABI change). 1. Voice Chat (VPIO) silent playback: doStartMicStream() called audioRestart() BEFORE startStream, so when the engine was already running (a remote stream had started it) it reopened with external_capture=false and opened a hardware miniaudio capture device. The announce-result restart then early-returns (engine already running) so that device was never dropped and fought the AVAudioEngine VPIO unit, silencing playback. Now: setExternalPlayback first, then startStream (stores external_feed synchronously), THEN audioRestart() — the core reopens in full external mode (no hardware devices). Added VPIO diagnostics: graph/route formats at start, ring written/read totals at teardown. 2. Stereo Mic / Studio quiet earpiece: the .builtInMicBtA2dp presets omit .defaultToSpeaker (it breaks A2DP) and skip forceSpeaker, so with no Bluetooth connected output pinned to the quiet receiver. New IOSAudioRouter.applyA2dpSpeakerFallback() overrides to the built-in speaker when no external (A2DP/wired/AirPlay) output is present and clears the override when one is — called after activation and on device-change route changes. --- PROGRESS.md | 20 ++++++++++-- .../iOS/VoiceCatiOS/AudioSessionManager.swift | 15 +++++---- .../iOS/VoiceCatiOS/IOSAudioRouter.swift | 32 +++++++++++++++++-- .../IOSVoiceProcessingEngine.swift | 24 +++++++++++++- .../apple/iOS/VoiceCatiOS/SessionState.swift | 19 +++++++---- 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index f5d22e3..6ffe878 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -30,10 +30,24 @@ up instantly. Newest status at the top. to speaker; SettingsView shows AEC/NS state. **Rebuild the xcframework** before building the app: `clients/apple/scripts/build-xcframework.sh --all` (new ABI symbols). `xcodebuild` iOS sim Debug BUILD SUCCEEDED. + - **Post-verification fixes (2026-06-22, Swift-only — no core/ABI change):** two on-device bugs fixed. + - **Voice Chat (VPIO) silent playback:** `SessionState.doStartMicStream()` called `audioRestart()` + BEFORE `startStream`, so when the engine was already running (a remote stream had started it) it + reopened with `external_capture=false` and opened a hardware miniaudio capture device; the + announce-result restart then early-returned (engine already running) so that device was never + dropped and fought the `AVAudioEngine` VPIO unit, silencing playback. Fix: set + `setExternalPlayback` first, then `startStream` (which stores `external_feed` synchronously), THEN + `audioRestart()` — the core reopens in full external mode (no hardware devices). Added VPIO + diagnostics (graph/route formats at start; ring written/read totals at teardown). + - **Stereo Mic / Studio quiet earpiece:** the `.builtInMicBtA2dp` presets omit `.defaultToSpeaker` + (it breaks A2DP) and skip `forceSpeaker`, so with no Bluetooth connected output pinned to the quiet + receiver. New `IOSAudioRouter.applyA2dpSpeakerFallback()` overrides to the built-in speaker when no + external (A2DP/wired/AirPlay) output is present, clears the override when one is — called after + activation and on device-change route changes (`AudioSessionManager`). - **Next (user, on device):** two iPhones on speaker, Voice Chat preset → confirm (a) no echo, (b) - background noise suppressed, (c) speaker output by default; then Studio preset still works (raw, no - AEC) — i.e. VPIO is scoped to the AEC presets. Tune the mixer-timer/ring sizing if there's - under/overrun. + background noise suppressed, (c) speaker output by default AND remote audio is now audible; then + Stereo Mic / Studio with no BT → confirm loud speaker (not earpiece), and A2DP takes over when a BT + headset connects. Tune the mixer-timer/ring sizing if there's under/overrun. - **Done (2026-06-21):** **Docker + Linux deployment + GitHub Actions cross-build.** Added the complete Linux server deployment story (the only missing platform — Windows and macOS already have native diff --git a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift index 25aa16a..6296582 100644 --- a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift +++ b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift @@ -56,12 +56,10 @@ final class AudioSessionManager { let session = AVAudioSession.sharedInstance() try session.setActive(true, options: []) isSessionActive = true - // For the A2DP output presets, make sure output isn't pinned to the built-in speaker. - // A2DP routing in .playAndRecord is fragile; clearing any speaker override after the - // session is live nudges iOS to honor the Bluetooth output route. - if IOSAudioRouter.shared.wantsA2dpOutput { - try? session.overrideOutputAudioPort(.none) - } + // For the A2DP output presets, pick the right output once the session is live: defer to + // a connected A2DP/wired/AirPlay route, but fall back to the loud built-in speaker (not + // the quiet earpiece) when nothing external is connected. See applyA2dpSpeakerFallback(). + IOSAudioRouter.shared.applyA2dpSpeakerFallback() let route = AVAudioSession.sharedInstance().currentRoute let outputNames = route.outputs.map { $0.portName }.joined(separator: ", ") let inputNames = route.inputs.map { $0.portName }.joined(separator: ", ") @@ -160,6 +158,11 @@ final class AudioSessionManager { if reason == .oldDeviceUnavailable || reason == .newDeviceAvailable { logger.info("routeChange — external device change, re-applying config") IOSAudioRouter.shared.applyConfiguration() + // Re-evaluate the A2DP-mode speaker fallback: a Bluetooth unplug should drop us onto + // the loud speaker (not the earpiece), and a replug should hand output back to A2DP. + if isSessionActive { + IOSAudioRouter.shared.applyA2dpSpeakerFallback() + } } IOSAudioRouter.shared.refreshRoutes() diff --git a/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift index 20a2cf7..c9dc7d7 100644 --- a/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift +++ b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift @@ -689,9 +689,35 @@ final class IOSAudioRouter: ObservableObject { showsA2dpNoAecWarning = (bluetoothMode == .builtInMicBtA2dp) } - /// Whether the current configuration wants Bluetooth A2DP output. Used after session - /// activation to clear any lingering speaker override that would pin output to the speaker. - var wantsA2dpOutput: Bool { bluetoothMode == .builtInMicBtA2dp } + /// Route fallback for the A2DP-output presets (Stereo Mic / Studio / BT Headphones + Mono + /// Mic, all `.builtInMicBtA2dp`). These presets deliberately omit `.defaultToSpeaker` (it + /// breaks A2DP routing) and skip the `forceSpeaker` override, so when NO external output + /// (Bluetooth A2DP / wired / AirPlay) is connected `.playAndRecord` pins output to the quiet + /// built-in receiver (earpiece). This routes to the loud built-in speaker instead via a + /// post-activation `overrideOutputAudioPort(.speaker)` — the documented "A2DP if connected, + /// else speaker" behavior. When an external output IS present we clear the override so A2DP / + /// headphones / AirPlay are honored. No-op outside `.builtInMicBtA2dp` mode (other modes pick + /// their route via category options). Must be called AFTER the session is active. + func applyA2dpSpeakerFallback() { + guard bluetoothMode == .builtInMicBtA2dp else { return } + let session = AVAudioSession.sharedInstance() + // Treat the built-in receiver and speaker as "internal"; anything else (A2DP, headphones, + // USB, AirPlay) is an external output we should defer to. + let hasExternalOutput = session.currentRoute.outputs.contains { + $0.portType != .builtInReceiver && $0.portType != .builtInSpeaker + } + do { + if hasExternalOutput { + try session.overrideOutputAudioPort(.none) + logger.info("A2DP mode — external output present, clearing speaker override") + } else { + try session.overrideOutputAudioPort(.speaker) + logger.info("A2DP mode — no external output, routing to built-in speaker") + } + } catch { + logger.error("A2DP speaker fallback failed: \(error.localizedDescription)") + } + } /// The selected input port object, if any. var selectedPort: IOSAudioInputPort? { diff --git a/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift b/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift index 6b24aea..7bc7471 100644 --- a/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift +++ b/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift @@ -75,6 +75,14 @@ final class PCMRing { /// Discard everything buffered — call before (re)starting so stale pre-roll isn't played. func reset() { OSMemoryBarrier(); readIdx = writeIdx } + + /// Diagnostics: monotonic total samples written / read since the ring was created. The + /// indices are already cumulative, so these are free. Only read them when both threads are + /// quiesced (e.g. at teardown after the engine + mixer sink are stopped) — they are not + /// synchronized for live cross-thread reads. Lets us tell "core never delivered PCM" (Bug 1 + /// core path) apart from "PCM arrived but produced no sound" (AVAudioEngine output graph). + var debugTotalWritten: UInt64 { writeIdx } + var debugTotalRead: UInt64 { readIdx } } /// Native iOS voice-processing audio path (docs/voice.md §8 "iOS voice processing"). @@ -215,7 +223,17 @@ final class IOSVoiceProcessingEngine { do { try engine.start() isRunning = true - logger.info("VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh)") + // Diagnostics: capture the negotiated graph formats and the live output route so a + // silent-playback report can be triaged (format/rate mismatch vs. routing vs. the + // core not delivering PCM — see the ring stats logged in teardown()). + let outFmt = engine.outputNode.outputFormat(forBus: 0) + let mixFmt = engine.mainMixerNode.outputFormat(forBus: 0) + let route = AVAudioSession.sharedInstance().currentRoute.outputs + .map { "\($0.portName)[\($0.portType.rawValue)]" }.joined(separator: ", ") + logger.info(""" + VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh), \ + outputNode=\(outFmt), mainMixer=\(mixFmt), outputRoute=[\(route)] + """) } catch { logger.error("VPIO engine start failed: \(error.localizedDescription)") teardown() @@ -234,6 +252,10 @@ final class IOSVoiceProcessingEngine { client?.setMixedOutputSink(nil, user: nil) engine.inputNode.removeTap(onBus: 0) if engine.isRunning { engine.stop() } + // Diagnostics (threads now quiesced): how much mixed PCM the core delivered into the ring + // vs. how much the render thread consumed. written==0 ⇒ the core never delivered (Bug 1 + // core/lifecycle path); written>0 with no audible output ⇒ the AVAudioEngine output graph. + logger.info("VPIO ring stats — written=\(self.ring.debugTotalWritten) read=\(self.ring.debugTotalRead) samples") try? engine.inputNode.setVoiceProcessingEnabled(false) if let src = sourceNode { engine.detach(src) diff --git a/clients/apple/iOS/VoiceCatiOS/SessionState.swift b/clients/apple/iOS/VoiceCatiOS/SessionState.swift index 21a3cbe..cddb569 100644 --- a/clients/apple/iOS/VoiceCatiOS/SessionState.swift +++ b/clients/apple/iOS/VoiceCatiOS/SessionState.swift @@ -230,13 +230,16 @@ final class SessionState { // runs in external mode (no hardware mic/playback). The mic stream is started with // externalFeed so the core skips the hardware capture device; setExternalPlayback makes // it skip the hardware playback device and deliver the mix to IOSVoiceProcessingEngine. + // + // ORDER MATTERS: set the external-playback flag now, but defer audioRestart() until + // AFTER startStream (below) so the MIC LocalStream — which carries external_feed=true — + // already exists when ensure_audio_running() derives external_capture. Restarting before + // the stream exists makes the core reopen a hardware capture device that is never dropped + // (the announce-result restart early-returns because the engine is already running); that + // lingering miniaudio capture unit then fights the AVAudioEngine VPIO unit on the same + // .voiceChat session and silences VPIO playback. let useVPIO = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing - if useVPIO { - client.setExternalPlayback(true) - client.audioRestart() // flip any already-running (pre-join) engine into external mode - } else { - client.setExternalPlayback(false) - } + client.setExternalPlayback(useVPIO) let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic", externalFeed: useVPIO) @@ -259,6 +262,10 @@ final class SessionState { client.setCaptureChannels(streamId: streamId, channels: channels) } if useVPIO { + // The external-feed MIC stream now exists, so restart the core into full + // external mode (no hardware capture/playback, mixer-timer only) — mic and + // speaker are owned entirely by the VPIO engine, which we start right after. + client.audioRestart() IOSVoiceProcessingEngine.shared.start( client: client, micStreamId: streamId, captureChannels: channels) }