diff --git a/CLAUDE.md b/CLAUDE.md index 4efc9d5..d90ddf6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ and what's next* read [`PROGRESS.md`](PROGRESS.md); for *design* read [`docs/`]( > server-mute, channel CRUD, in-app account management, disconnect/keepalive/reaper. Windows > WinForms C# client shipped (M4). **macOS AppKit client shipped** — `VoiceCatMac.xcodeproj` > at `clients/apple/macOS/`. **iOS SwiftUI client shipped** — `VoiceCatiOS.xcodeproj` at -> `clients/apple/iOS/`. `ctest --preset dev` green — 23/23 tests. +> `clients/apple/iOS/`. `ctest --preset dev` green — 24/24 tests. > External PCM feed/tap API (`vc_stream_feed_pcm` + `vc_set_pcm_sink`) shipped. > **Screen-audio sharing shipped on macOS (ScreenCaptureKit) and iOS (ReplayKit Broadcast > Upload Extension → host App Group ring → `vc_stream_feed_pcm`).** See [`PROGRESS.md`](PROGRESS.md). diff --git a/PROGRESS.md b/PROGRESS.md index 85d8829..f5d22e3 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -10,6 +10,31 @@ up instantly. Newest status at the top. ## ▶ Where we left off / next action +- **Awaiting on-device verification (2026-06-22):** **iOS real echo cancellation / noise suppression + via native VPIO.** Root cause of "voice chat doesn't sound like a call" (echo + no NR): real iOS + AEC/NS/AGC come only from Apple's Voice-Processing I/O unit (VPIO), but the core uses miniaudio's + plain RemoteIO units — so `.voiceChat` mode alone never engaged AEC. Fix moves both mic capture and + playback to a native Swift `AVAudioEngine` (`setVoiceProcessingEnabled`) on the AEC presets, with the + core in external mode. + - **Core (done, builds + tests green):** new ABI `vc_set_mixed_output_sink` + `vc_set_external_playback` + (voicecat.h PATCH→2). `AudioEngine` gains a mixer-timer thread that drives `on_playback` (decode+mix) + on a ~20 ms cadence with NO hardware playback device and ships the final mix to the mixed-output + sink; `start()` also skips the hardware capture device when the MIC stream is `external_feed` + (`AudioParams.external_capture`). New white-box test `test_external_playback` (23/24; + pre-existing `external_pcm` teardown crash on Darwin 25.5 is UNRELATED — original tree crashes too). + - **Swift (done, builds):** `VoiceCatCore` wrappers (`externalFeed` on `StreamDescriptor`, + `setMixedOutputSink`, `setExternalPlayback`); new `IOSVoiceProcessingEngine.swift` (VPIO + `AVAudioEngine`: mic tap→`feedPcm`, mixed-sink lock-free ring→`AVAudioSourceNode`); + `IOSAudioRouter.currentConfigUsesVoiceProcessing` gates the path per preset; `SessionState` + join/leave + `reconcileVoicePath()` switch between VPIO and the miniaudio path; Voice Chat defaults + to speaker; SettingsView shows AEC/NS state. **Rebuild the xcframework** before building the app: + `clients/apple/scripts/build-xcframework.sh --all` (new ABI symbols). `xcodebuild` iOS sim Debug + BUILD SUCCEEDED. + - **Next (user, on device):** two iPhones on speaker, Voice Chat preset → confirm (a) no echo, (b) + background noise suppressed, (c) speaker output by default; then Studio preset still works (raw, no + AEC) — i.e. VPIO is scoped to the AEC presets. Tune the mixer-timer/ring sizing if there's + under/overrun. + - **Done (2026-06-21):** **Docker + Linux deployment + GitHub Actions cross-build.** Added the complete Linux server deployment story (the only missing platform — Windows and macOS already have native binaries): diff --git a/clients/apple/Sources/VoiceCatCore/Models.swift b/clients/apple/Sources/VoiceCatCore/Models.swift index 6527938..1dc614d 100644 --- a/clients/apple/Sources/VoiceCatCore/Models.swift +++ b/clients/apple/Sources/VoiceCatCore/Models.swift @@ -212,9 +212,14 @@ public struct StreamDescriptor: Sendable, Equatable { /// nil = default device for this kind. public let deviceId: String? public let label: String + /// When true the caller feeds PCM via `feedPcm` (e.g. the iOS VPIO mic path) and the core + /// skips opening a hardware capture device for this stream. Mirrors `vc_stream_desc.external_feed`. + public let externalFeed: Bool - public init(kind: VoiceCatStreamKind, deviceId: String? = nil, label: String) { + public init(kind: VoiceCatStreamKind, deviceId: String? = nil, label: String, + externalFeed: Bool = false) { self.kind = kind; self.deviceId = deviceId; self.label = label + self.externalFeed = externalFeed } } diff --git a/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift b/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift index d285bbe..a1d9c1d 100644 --- a/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift +++ b/clients/apple/Sources/VoiceCatCore/VoiceCatClient.swift @@ -41,6 +41,10 @@ import Foundation /// `VcPcmSinkCallback` delegate. public typealias VoiceCatPcmSinkCallback = vc_pcm_sink_cb +/// Swift-idiomatic alias for the C `vc_mixed_output_cb` function-pointer type from `voicecat.h` +/// — the external mixed-output sink used by the iOS VPIO path (see `setMixedOutputSink`). +public typealias VoiceCatMixedOutputCallback = vc_mixed_output_cb + /// The Swift wrapper around `vc_client*`. Owns the native handle for its entire lifetime; /// `deinit` destroys it. Events and level meters are delivered on the main queue via the /// `onEvent` / `onLevel` closures. @@ -292,6 +296,7 @@ public final class VoiceCatClient { desc.kind = descriptor.kind.cValue desc.device_id = deviceIdPtr.map { UnsafePointer($0) } desc.label = UnsafePointer(labelPtr) + desc.external_feed = descriptor.externalFeed ? 1 : 0 let r = vc_stream_start(handle, &desc, &streamId) return (VoiceCatResult(r), streamId) @@ -355,6 +360,26 @@ public final class VoiceCatClient { VoiceCatResult(vc_set_pcm_sink(handle, cb, user)) } + /// External mixed-output sink (iOS VPIO) — receives the FINAL mixed remote audio as int16 + /// PCM on the core's mixer-timer thread when external playback is enabled. The Swift VPIO + /// renderer copies this into its ring and plays it through the voice-processing output so + /// echo cancellation has its reference signal. Pass `nil` to disable. Mirrors + /// `vc_set_mixed_output_sink`. The callback MUST NOT block or allocate. + @discardableResult + public func setMixedOutputSink(_ cb: VoiceCatMixedOutputCallback?, + user: UnsafeMutableRawPointer?) -> VoiceCatResult { + VoiceCatResult(vc_set_mixed_output_sink(handle, cb, user)) + } + + /// Enable/disable external-playback mode (iOS VPIO). When enabled, the core opens NO hardware + /// playback device; it drives decode+mix on a timer and delivers the final mix via + /// `setMixedOutputSink`. Apply before the engine starts, or follow with `audioRestart()` to + /// apply to a running engine. Mirrors `vc_set_external_playback`. + @discardableResult + public func setExternalPlayback(_ enabled: Bool) -> VoiceCatResult { + VoiceCatResult(vc_set_external_playback(handle, enabled ? 1 : 0)) + } + @discardableResult public func setInputMode(_ mode: VoiceCatInputMode) -> VoiceCatResult { VoiceCatResult(vc_set_input_mode(handle, mode.cValue)) diff --git a/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj b/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj index 4ab0a04..b04d350 100644 --- a/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj +++ b/clients/apple/iOS/VoiceCatiOS.xcodeproj/project.pbxproj @@ -33,6 +33,7 @@ BBBB00000000000000000046 /* AccountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002D /* AccountsView.swift */; }; BBBB00000000000000000047 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002E /* SettingsView.swift */; }; BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002F /* IOSAudioRouter.swift */; }; + BBBB00000000000000000F02 /* IOSVoiceProcessingEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */; }; BBBB00000000000000000048 /* VoiceCatCore in Frameworks */ = {isa = PBXBuildFile; productRef = BBBB0000000000000000004A /* VoiceCatCore */; }; CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000002 /* BroadcastAudioPump.swift */; }; CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; }; @@ -89,6 +90,7 @@ BBBB0000000000000000002D /* AccountsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountsView.swift; sourceTree = ""; }; BBBB0000000000000000002E /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; BBBB0000000000000000002F /* IOSAudioRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSAudioRouter.swift; sourceTree = ""; }; + BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSVoiceProcessingEngine.swift; sourceTree = ""; }; CCCC00000000000000000001 /* BroadcastAudioRing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioRing.swift; sourceTree = ""; }; CCCC00000000000000000002 /* BroadcastAudioPump.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioPump.swift; sourceTree = ""; }; CCCC00000000000000000003 /* SampleHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SampleHandler.swift; sourceTree = ""; }; @@ -143,6 +145,7 @@ BBBB00000000000000000019 /* SessionState.swift */, BBBB0000000000000000001A /* AudioSessionManager.swift */, BBBB0000000000000000002F /* IOSAudioRouter.swift */, + BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */, BBBB0000000000000000001B /* ServerListStore.swift */, BBBB0000000000000000001C /* SavedServer.swift */, CCCC00000000000000000002 /* BroadcastAudioPump.swift */, @@ -296,6 +299,7 @@ BBBB00000000000000000032 /* SessionState.swift in Sources */, BBBB00000000000000000033 /* AudioSessionManager.swift in Sources */, BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */, + BBBB00000000000000000F02 /* IOSVoiceProcessingEngine.swift in Sources */, BBBB00000000000000000034 /* ServerListStore.swift in Sources */, BBBB00000000000000000035 /* SavedServer.swift in Sources */, CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */, diff --git a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift index 342a653..25aa16a 100644 --- a/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift +++ b/clients/apple/iOS/VoiceCatiOS/AudioSessionManager.swift @@ -15,6 +15,12 @@ final class AudioSessionManager { /// channel count (e.g. when switching stereo → mono) without going through `SessionState`. var activeMicStreamId: UInt32? + /// Set by `SessionState`. Invoked by `IOSAudioRouter` after an audio-config change so the + /// voice path (native VPIO vs the core's miniaudio path) can be restarted to match the new + /// preset/route when the mic is active. No-op when not in voice. See + /// `SessionState.reconcileVoicePath()` and `IOSVoiceProcessingEngine`. + var reconcileVoicePath: (() -> Void)? + /// Tracks whether WE activated the session. The session must be active whenever the /// AudioEngine is running (for capture OR playback), so it is activated when any audio /// needs to play (a remote stream started OR the user joins voice) and only deactivated diff --git a/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift index 10fc50f..20a2cf7 100644 --- a/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift +++ b/clients/apple/iOS/VoiceCatiOS/IOSAudioRouter.swift @@ -307,6 +307,14 @@ final class IOSAudioRouter: ObservableObject { return .custom } + /// Whether the current configuration should use the native iOS Voice-Processing path (VPIO: + /// real AEC/NS/AGC via `IOSVoiceProcessingEngine`). True exactly when `applyConfiguration` + /// selects the `.voiceChat` AVAudioSession mode — mono + standard processing + not A2DP + /// (A2DP / stereo / raw modes can't use VPIO, so they keep the core's miniaudio path). + var currentConfigUsesVoiceProcessing: Bool { + captureChannels == .mono && micMode == .standard && bluetoothMode != .builtInMicBtA2dp + } + // MARK: - Apply configuration /// Apply the full audio configuration to AVAudioSession. Call this before the core @@ -573,6 +581,8 @@ final class IOSAudioRouter: ObservableObject { savePreferences() applyConfiguration() refreshRoutes() + // VPIO class or route may have changed — restart the voice path if mic is active. + AudioSessionManager.shared.reconcileVoicePath?() } func setForceSpeaker(_ on: Bool) { @@ -580,6 +590,8 @@ final class IOSAudioRouter: ObservableObject { savePreferences() applyConfiguration() refreshRoutes() + // Route changed under a possibly-running VPIO engine — reconcile if mic is active. + AudioSessionManager.shared.reconcileVoicePath?() } func selectMicMode(_ mode: MicMode) { @@ -587,6 +599,8 @@ final class IOSAudioRouter: ObservableObject { savePreferences() applyConfiguration() updateWarnings() + // Standard↔Raw flips the VPIO class — reconcile if mic is active. + AudioSessionManager.shared.reconcileVoicePath?() } func selectCaptureChannels(_ channels: CaptureChannels) { @@ -603,6 +617,8 @@ final class IOSAudioRouter: ObservableObject { // route), then capture — avoiding the race where stereo capture activation drops // A2DP before the playback device has a chance to claim the route. _ = AudioSessionManager.shared.client?.audioRestart() + // Mono↔stereo flips the VPIO class (stereo can't use VPIO) — reconcile if mic is active. + AudioSessionManager.shared.reconcileVoicePath?() } // MARK: - Presets @@ -617,6 +633,10 @@ final class IOSAudioRouter: ObservableObject { micMode = preset.micMode captureChannels = preset.captureChannels + // Voice Chat is a phone-call experience — default to the loud speaker so output doesn't + // land on the quiet earpiece (receiver). Still yields to connected BT/wired output. + if preset == .voiceChat { forceSpeaker = true } + if preset.usesBuiltInMic { // Find the built-in mic port from available inputs and select it. let session = AVAudioSession.sharedInstance() @@ -648,6 +668,9 @@ final class IOSAudioRouter: ObservableObject { // count is stored. Playback opens first (commits A2DP route), then capture. _ = AudioSessionManager.shared.client?.audioRestart() refreshRoutes() + // The preset may have flipped the VPIO class (and/or the route) — restart the voice path + // if the mic is active so AEC/NS engage (or disengage) to match the new preset. + AudioSessionManager.shared.reconcileVoicePath?() logger.info("applyPreset — \(preset.rawValue)") } diff --git a/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift b/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift new file mode 100644 index 0000000..6b24aea --- /dev/null +++ b/clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift @@ -0,0 +1,247 @@ +import AVFoundation +import Darwin +import os +import VoiceCatCore + +private let logger = Logger(subsystem: "cat.voice.VoiceCatiOS", category: "IOSVoiceProcessingEngine") + +/// In-process single-producer/single-consumer int16 PCM ring for the VPIO playback path. +/// +/// producer = the core's mixer-timer thread (the `vc_set_mixed_output_sink` callback) +/// consumer = the `AVAudioSourceNode` render thread +/// +/// Heap-backed (not shared memory like `BroadcastAudioRing`), but the same discipline: aligned +/// 64-bit monotonic indices with `OSMemoryBarrier` for acquire/release ordering. Both the C +/// callback and the render block are real-time — they only do index math + a memcpy here, never +/// lock or allocate. +final class PCMRing { + private let data: UnsafeMutablePointer + private let capacity: Int + private var writeIdx: UInt64 = 0 + private var readIdx: UInt64 = 0 + + init(capacitySamples: Int) { + capacity = capacitySamples + data = UnsafeMutablePointer.allocate(capacity: capacitySamples) + data.initialize(repeating: 0, count: capacitySamples) + } + deinit { data.deallocate() } + + /// Producer: append `count` interleaved int16 samples. Drops the chunk if it doesn't fit + /// (better to skip than tear). Single producer only (the core mixer-timer thread). + func write(_ src: UnsafePointer, count: Int) { + guard count > 0, count <= capacity else { return } + let w = writeIdx + OSMemoryBarrier() + let r = readIdx + if capacity - Int(w &- r) < count { return } // full: drop + var idx = Int(w % UInt64(capacity)) + var off = 0 + var rem = count + while rem > 0 { + let chunk = min(rem, capacity - idx) + (data + idx).update(from: src + off, count: chunk) + idx = (idx + chunk) % capacity + off += chunk + rem -= chunk + } + OSMemoryBarrier() + writeIdx = w &+ UInt64(count) + } + + /// Consumer: read up to `count` interleaved int16 samples into `dst`; returns the number + /// read (the rest is the caller's to silence-fill). Single consumer only (render thread). + func read(into dst: UnsafeMutablePointer, count: Int) -> Int { + let r = readIdx + OSMemoryBarrier() + let w = writeIdx + let available = Int(w &- r) + if available <= 0 { return 0 } + let n = min(available, count) + var idx = Int(r % UInt64(capacity)) + var off = 0 + var rem = n + while rem > 0 { + let chunk = min(rem, capacity - idx) + (dst + off).update(from: data + idx, count: chunk) + idx = (idx + chunk) % capacity + off += chunk + rem -= chunk + } + OSMemoryBarrier() + readIdx = r &+ UInt64(n) + return n + } + + /// Discard everything buffered — call before (re)starting so stale pre-roll isn't played. + func reset() { OSMemoryBarrier(); readIdx = writeIdx } +} + +/// Native iOS voice-processing audio path (docs/voice.md §8 "iOS voice processing"). +/// +/// Real iOS echo cancellation / noise suppression / AGC come ONLY from Apple's Voice-Processing +/// I/O unit (VPIO), which `AVAudioEngine.setVoiceProcessingEnabled(true)` enables. For VPIO to +/// cancel echo it must own BOTH the mic capture and the remote-audio playback (it subtracts the +/// played-back signal from the mic), so on the AEC presets this engine drives both directions and +/// the core runs in external mode (no hardware devices): +/// - **Mic → core:** a tap on the VPIO input node → 48 kHz int16 → `client.feedPcm(micStreamId)`. +/// - **core → speaker:** the core's mixed-output sink fills `ring`; an `AVAudioSourceNode` pulls +/// from it and renders through the VPIO output, giving AEC its reference signal. +/// +/// Lifecycle is driven by `SessionState` join/leave. The Stereo Mic / Studio / A2DP presets keep +/// the core's miniaudio path instead (they want raw / stereo / no-AEC routing VPIO can't provide). +@MainActor +final class IOSVoiceProcessingEngine { + static let shared = IOSVoiceProcessingEngine() + + private(set) var isRunning = false + + private let engine = AVAudioEngine() + private var sourceNode: AVAudioSourceNode? + private weak var client: VoiceCatClient? + private var micStreamId: UInt32 = 0 + + // 48 kHz stereo Float32 (deinterleaved) — the format the source node renders and the engine + // processes in. The core delivers 48 kHz stereo int16 via the mixed-output sink. + private let outFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false)! + + // Playback ring (mixed remote audio): ~0.5 s of 48 kHz stereo int16. Filled by the core's + // mixer-timer thread, drained by the source-node render thread. + private let ring = PCMRing(capacitySamples: 48000 * 2 / 2) + // Render-thread scratch for deinterleaving — pre-allocated so the render block never allocates. + private let renderScratchFrames = 8192 + private let renderScratch: UnsafeMutablePointer + + // Mic-feed converter (input-node format → 48 kHz int16) and its target buffer. Owned here so + // the (background) tap block reuses them instead of allocating per callback. + private var micConverter: AVAudioConverter? + private var micTargetFormat: AVAudioFormat? + + private init() { + renderScratch = UnsafeMutablePointer.allocate(capacity: renderScratchFrames * 2) + renderScratch.initialize(repeating: 0, count: renderScratchFrames * 2) + } + + /// Start the VPIO engine for an active mic stream. The caller must have already enabled + /// external playback on the core (`client.setExternalPlayback(true)` + `audioRestart()`) and + /// started the MIC stream with `externalFeed: true`. + func start(client: VoiceCatClient, micStreamId: UInt32, captureChannels: UInt32) { + guard !isRunning else { return } + self.client = client + self.micStreamId = micStreamId + ring.reset() + + // Enable the voice-processing I/O unit (AEC/NS/AGC) on the shared input+output unit. + do { + try engine.inputNode.setVoiceProcessingEnabled(true) + } catch { + logger.error("setVoiceProcessingEnabled failed: \(error.localizedDescription) — AEC unavailable") + } + + // ── Playback: source node pulls mixed PCM from the ring through the VPIO output. ── + let ring = self.ring + let scratch = self.renderScratch + let scratchFrames = self.renderScratchFrames + let src = AVAudioSourceNode(format: outFormat) { _, _, frameCount, ablPtr in + let frames = Int(frameCount) + let abl = UnsafeMutableAudioBufferListPointer(ablPtr) + let n = min(frames, scratchFrames) + let got = ring.read(into: scratch, count: n * 2) / 2 // interleaved stereo → frames + // Deinterleave int16 → Float32 per channel; silence-fill any underrun tail. + let scale: Float = 1.0 / 32768.0 + for ch in 0.. 0) + ? AVAudioConverter(from: inFormat, to: target!) : nil + if micConverter == nil { + logger.error("mic converter unavailable (in=\(inFormat)) — mic will not transmit") + } + + let c = client + let sid = micStreamId + let converter = micConverter + let tgt = micTargetFormat + engine.inputNode.installTap(onBus: 0, bufferSize: 960, format: inFormat) { buffer, _ in + guard let converter, let tgt else { return } + // Convert this tap buffer to 48 kHz int16. Output capacity scaled for any upsample. + let ratio = tgt.sampleRate / buffer.format.sampleRate + let outCap = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 16) + guard let outBuf = AVAudioPCMBuffer(pcmFormat: tgt, frameCapacity: outCap) else { return } + var fed = false + let status = converter.convert(to: outBuf, error: nil) { _, outStatus in + if fed { outStatus.pointee = .noDataNow; return nil } + fed = true + outStatus.pointee = .haveData + return buffer + } + guard status != .error, outBuf.frameLength > 0, + let chData = outBuf.int16ChannelData else { return } + let spc = Int(outBuf.frameLength) + // int16 interleaved → channelData[0] is the interleaved buffer for interleaved formats. + c.feedPcm(streamId: sid, pcm: chData[0], samplesPerChannel: spc, channels: targetCh) + } + + // ── Wire the core's mixed-output sink into the ring (C function pointer, no captures). ── + let ringPtr = Unmanaged.passUnretained(self.ring).toOpaque() + client.setMixedOutputSink({ user, pcm, spc, ch, _ in + guard let user, let pcm else { return } + let ring = Unmanaged.fromOpaque(user).takeUnretainedValue() + ring.write(pcm, count: spc * Int(ch)) + }, user: ringPtr) + + engine.prepare() + do { + try engine.start() + isRunning = true + logger.info("VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh)") + } catch { + logger.error("VPIO engine start failed: \(error.localizedDescription)") + teardown() + } + } + + /// Stop the VPIO engine. The caller is responsible for restoring the core's hardware playback + /// afterwards (`client.setExternalPlayback(false)` + `audioRestart()`). + func stop() { + guard isRunning else { return } + teardown() + logger.info("VPIO engine stopped") + } + + private func teardown() { + client?.setMixedOutputSink(nil, user: nil) + engine.inputNode.removeTap(onBus: 0) + if engine.isRunning { engine.stop() } + try? engine.inputNode.setVoiceProcessingEnabled(false) + if let src = sourceNode { + engine.detach(src) + sourceNode = nil + } + micConverter = nil + micTargetFormat = nil + ring.reset() + isRunning = false + } +} diff --git a/clients/apple/iOS/VoiceCatiOS/SessionState.swift b/clients/apple/iOS/VoiceCatiOS/SessionState.swift index aecc30f..21a3cbe 100644 --- a/clients/apple/iOS/VoiceCatiOS/SessionState.swift +++ b/clients/apple/iOS/VoiceCatiOS/SessionState.swift @@ -73,6 +73,9 @@ final class SessionState { broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() } broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() } broadcastPump.start() + // When IOSAudioRouter changes the audio config, restart the voice path if needed so the + // native VPIO engine (AEC/NS/AGC) engages or disengages to match the new preset/route. + AudioSessionManager.shared.reconcileVoicePath = { [weak self] in self?.reconcileVoicePath() } } deinit { @@ -222,7 +225,21 @@ final class SessionState { addActivity("AVAudioSession activate failed: \(error)") return } - let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic") + + // VPIO path: on the AEC presets, the native AVAudioEngine does AEC/NS/AGC and the core + // runs in external mode (no hardware mic/playback). The mic stream is started with + // externalFeed so the core skips the hardware capture device; setExternalPlayback makes + // it skip the hardware playback device and deliver the mix to IOSVoiceProcessingEngine. + let useVPIO = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing + if useVPIO { + client.setExternalPlayback(true) + client.audioRestart() // flip any already-running (pre-join) engine into external mode + } else { + client.setExternalPlayback(false) + } + + let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic", + externalFeed: useVPIO) let (result, streamId) = client.startStream(desc) if result == .ok { voiceState.micActive = true @@ -241,17 +258,37 @@ final class SessionState { if channels != 1 { client.setCaptureChannels(streamId: streamId, channels: channels) } + if useVPIO { + IOSVoiceProcessingEngine.shared.start( + client: client, micStreamId: streamId, captureChannels: channels) + } } else { addActivity("Failed to start mic: \(result.description)") + if useVPIO { // revert external-playback mode so remote audio still plays + client.setExternalPlayback(false) + client.audioRestart() + } } } func stopMicStream() { + // Tear down the VPIO engine first (removes the mic tap + unregisters the mixed sink), + // then stop the mic stream, then restore the core's hardware playback for any remaining + // remote audio. Order matters: the mic stream must be gone before audioRestart so the + // core opens a normal playback device (and no capture device — there's no mic stream). + let wasVPIO = IOSVoiceProcessingEngine.shared.isRunning + if wasVPIO { + IOSVoiceProcessingEngine.shared.stop() + } if voiceState.localStreamId != 0 { client.stopStream(voiceState.localStreamId) voiceState.localStreamId = 0 AudioSessionManager.shared.activeMicStreamId = nil } + if wasVPIO { + client.setExternalPlayback(false) + client.audioRestart() // reopen hardware playback (no mic stream → no hw capture) + } voiceState.micActive = false voiceState.level = 0 // Do NOT deactivate the AVAudioSession here — the user may still want to hear @@ -259,6 +296,19 @@ final class SessionState { // disconnecting from the server (see AppState.disconnect / .disconnected event). } + /// Restart the voice path when the audio config changes mid-call (driven by IOSAudioRouter). + /// If VPIO is involved on either the current or desired side, restart the mic so the native + /// voice-processing engine engages/disengages and re-binds to the new route. Pure miniaudio + /// config tweaks need no restart — the core's own audioRestart (already issued) handles them. + private func reconcileVoicePath() { + guard voiceState.micActive else { return } + let want = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing + let have = IOSVoiceProcessingEngine.shared.isRunning + guard want || have else { return } + stopMicStream() + doStartMicStream() + } + // MARK: - Screen audio share /// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream; diff --git a/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift b/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift index ee1fd74..577b03c 100644 --- a/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift +++ b/clients/apple/iOS/VoiceCatiOS/Views/SettingsView.swift @@ -30,6 +30,23 @@ struct SettingsView: View { .accessibilityLabel("Speaker output") .accessibilityHint("Routes audio to the speaker instead of the earpiece when no headphones are connected.") + // Surface the voice-processing state. On the AEC presets the native iOS + // Voice-Processing unit (VPIO) does echo cancellation, noise suppression and + // automatic gain control; the other presets (stereo/studio/A2DP) can't use it. + if router.currentConfigUsesVoiceProcessing { + Label("Echo cancellation & noise suppression on (iOS voice processing)", + systemImage: "waveform.badge.mic") + .font(.caption) + .foregroundStyle(.secondary) + .accessibilityLabel("Echo cancellation and noise suppression are on") + } else { + Label("No echo cancellation in this preset (stereo / studio / A2DP)", + systemImage: "waveform.slash") + .font(.caption) + .foregroundStyle(.secondary) + .accessibilityLabel("Echo cancellation is off in this preset") + } + if !router.hasBluetoothDevice && !router.hasWiredHeadset { Text("Connect Bluetooth headphones or a wired headset for more presets.") .font(.caption) diff --git a/core/include/voicecat.h b/core/include/voicecat.h index 8af5a1f..0e962d2 100644 --- a/core/include/voicecat.h +++ b/core/include/voicecat.h @@ -46,7 +46,7 @@ extern "C" { /* ── Version ──────────────────────────────────────────────────────────────── */ #define VOICECAT_VERSION_MAJOR 0 #define VOICECAT_VERSION_MINOR 0 -#define VOICECAT_VERSION_PATCH 1 +#define VOICECAT_VERSION_PATCH 2 /* +vc_set_mixed_output_sink / vc_set_external_playback (iOS VPIO) */ /* The control-protocol version this build speaks (docs/protocol.md §4). * v2 widened the UDP voice frame seq field u16 → u64 (docs/voice.md §2); a v2 server @@ -460,6 +460,43 @@ typedef void (*vc_pcm_sink_cb)(void* user, uint32_t user_id, uint32_t stream_id, uint32_t channels, uint32_t sample_rate); VC_API vc_result vc_set_pcm_sink(vc_client* c, vc_pcm_sink_cb cb, void* user); +/* ── External playback (iOS VPIO / echo cancellation) ─────────────────────────── + * On iOS, real echo cancellation + noise suppression + AGC are provided ONLY by Apple's + * Voice-Processing I/O audio unit (VPIO), which the Swift AVAudioEngine layer owns. For VPIO + * to cancel echo, the remote-audio playback must go through the SAME VPIO unit as the mic + * capture (VPIO subtracts the played-back signal from the mic). So in that topology the core + * must NOT open/drive its own hardware playback device — its output would bypass VPIO, giving + * it no reference signal and producing echo. Instead, enable external playback: the core keeps + * decoding + mixing every remote stream on a steady ~20 ms cadence and delivers the FINAL + * MIXED PCM (post output-volume, all streams summed) to this sink, which the Swift layer + * renders through the VPIO output. + * + * cb(user, pcm, samples_per_channel, channels, sample_rate) + * + * pcm : final mixed int16 PCM, interleaved when channels == 2. + * samples_per_channel : samples per channel for this block (960 @ 20 ms / 48 kHz). + * channels : the engine's playback channel count (2 = stereo). + * sample_rate : always 48000. + * + * The callback fires on the core's mixer-timer thread (NOT a hardware audio thread). It fires + * steadily even with no remote streams (a silent block), so the renderer has a continuous + * clock. The callback MUST NOT block, lock, or allocate — copy into a lock-free ring and + * return. Independent of vc_set_pcm_sink (the per-stream tap), which still works. Pass cb=NULL + * to disable (default: disabled). */ +typedef void (*vc_mixed_output_cb)(void* user, const int16_t* pcm, + size_t samples_per_channel, uint32_t channels, + uint32_t sample_rate); +VC_API vc_result vc_set_mixed_output_sink(vc_client* c, vc_mixed_output_cb cb, void* user); + +/* Enable/disable external-playback mode (default: disabled = normal hardware playback). When + * enabled, the core does NOT open a hardware playback device; decode+mix runs on an internal + * ~20 ms timer and the result is delivered via vc_set_mixed_output_sink. To also bypass the + * hardware mic (feeding VPIO-processed mic PCM instead), start the MIC stream with + * vc_stream_desc.external_feed=1 and push frames via vc_stream_feed_pcm — the core then skips + * the hardware capture device too. Apply BEFORE the engine starts, or follow with + * vc_audio_restart() to apply to a running engine. `enable` is a bool (0/1). */ +VC_API vc_result vc_set_external_playback(vc_client* c, int enable); + /* ── Text ─────────────────────────────────────────────────────────────────── */ VC_API vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id, const char* utf8); diff --git a/core/src/audio/audio_engine.cpp b/core/src/audio/audio_engine.cpp index 6ecd741..4713ac2 100644 --- a/core/src/audio/audio_engine.cpp +++ b/core/src/audio/audio_engine.cpp @@ -216,6 +216,10 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) { } ma_context* ctx = context_inited_ ? &context_ : nullptr; + // External playback (iOS VPIO): skip the hardware playback device entirely — a timer + // thread drives the mixer and the final mix goes to the Swift VPIO renderer (launched + // after the capture block below). + if (!external_playback_) { // ── Playback device (opened first on iOS: commits the output route — e.g. A2DP — // before the capture device starts. Starting stereo capture can trigger an iOS audio // route reconfiguration; opening playback first ensures A2DP is already committed @@ -252,7 +256,11 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) { } } } + } // end if (!external_playback_) + // External capture (iOS VPIO / external feed): skip the hardware mic device — PCM is fed + // via inject_capture / vc_stream_feed_pcm. capture_cb_ (set above) still fires for fed frames. + if (!params_.external_capture) { // ── Capture device (opened after playback so the output route is already committed) ── ma_device_id cap_id{}; bool have_cap_id = !p.capture_device_id.empty() && @@ -277,6 +285,17 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) { ma_device_uninit(&capture_device_); } } + } // end if (!params_.external_capture) + + // External playback: launch the mixer-timer thread now that the engine is configured. It + // drives on_playback() (decode + mix all remote streams) every frame_ms and ships the final + // mix to mixed_sink_ for the Swift VPIO renderer. No hardware playback device exists. + if (external_playback_) { + mixer_scratch_.assign( + static_cast(frame_samples_) * params_.playback_channels, 0); + mixer_timer_stop_.store(false, std::memory_order_release); + mixer_timer_thread_ = std::thread([this] { run_mixer_timer(); }); + } #endif // VOICECAT_HAS_AUDIO return true; @@ -322,6 +341,9 @@ void AudioEngine::stop() { if (!running_.exchange(false)) return; #ifdef VOICECAT_HAS_AUDIO + // External playback: stop + join the mixer-timer thread before tearing down state it reads. + mixer_timer_stop_.store(true, std::memory_order_release); + if (mixer_timer_thread_.joinable()) mixer_timer_thread_.join(); if (capture_started_) { ma_device_stop(&capture_device_); ma_device_uninit(&capture_device_); @@ -338,6 +360,12 @@ void AudioEngine::stop() { bool AudioEngine::suspend() { #ifdef VOICECAT_HAS_AUDIO if (!running_.load(std::memory_order_acquire)) return true; + // External playback: pause the mixer timer (there is no playback device to stop). This + // matches the VPIO renderer going down during an AVAudioSession interruption. + if (external_playback_) { + mixer_timer_stop_.store(true, std::memory_order_release); + if (mixer_timer_thread_.joinable()) mixer_timer_thread_.join(); + } bool ok = true; if (capture_started_) ok &= (ma_device_stop(&capture_device_) == MA_SUCCESS); if (playback_started_) ok &= (ma_device_stop(&playback_device_) == MA_SUCCESS); @@ -350,6 +378,11 @@ bool AudioEngine::suspend() { bool AudioEngine::resume() { #ifdef VOICECAT_HAS_AUDIO if (!running_.load(std::memory_order_acquire)) return true; + // External playback: relaunch the mixer timer (mixer_scratch_ is still sized from start()). + if (external_playback_ && !mixer_timer_thread_.joinable()) { + mixer_timer_stop_.store(false, std::memory_order_release); + mixer_timer_thread_ = std::thread([this] { run_mixer_timer(); }); + } bool ok = true; if (capture_started_) ok &= (ma_device_start(&capture_device_) == MA_SUCCESS); if (playback_started_) ok &= (ma_device_start(&playback_device_) == MA_SUCCESS); @@ -515,6 +548,11 @@ void AudioEngine::set_pcm_sink(PcmSink cb, void* user) { pcm_sink_.store(cb, std::memory_order_release); } +void AudioEngine::set_mixed_output_sink(MixedSink cb, void* user) { + mixed_sink_user_.store(user, std::memory_order_relaxed); + mixed_sink_.store(cb, std::memory_order_release); +} + #ifdef VOICECAT_HAS_AUDIO void AudioEngine::capture_data_cb(ma_device* dev, void* /*out*/, @@ -707,6 +745,32 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) { #endif } +// External-playback timer (iOS VPIO): with no hardware playback device to "pull" frames, this +// dedicated thread drives the mixer on a steady cadence. It is NOT a real-time audio thread (a +// plain timed worker, same class as vc_client::run_talk_timer), but it must still not allocate +// in the loop because on_playback() takes streams_mu_ via try_lock and runs the RT-safe decode +// path — so mixer_scratch_ is pre-sized in start(). Uses a deadline-based sleep to bound drift. +void AudioEngine::run_mixer_timer() { + using clock = std::chrono::steady_clock; + const uint32_t ch = params_.playback_channels; + const uint32_t spc = static_cast(frame_samples_); + const auto period = std::chrono::milliseconds(params_.frame_ms); + auto next = clock::now(); + while (!mixer_timer_stop_.load(std::memory_order_acquire)) { + on_playback(mixer_scratch_.data(), spc); + if (auto sink = mixed_sink_.load(std::memory_order_relaxed)) { + sink(mixed_sink_user_.load(std::memory_order_relaxed), mixer_scratch_.data(), spc, ch, + params_.sample_rate); + } + next += period; + // If we fell badly behind (e.g. the thread was descheduled), reset the deadline rather + // than spin to catch up — the VPIO renderer rides its own clock + jitter ring. + auto now = clock::now(); + if (next < now) next = now + period; + std::this_thread::sleep_until(next); + } +} + #ifdef VOICECAT_HAS_LOOPBACK void AudioEngine::loopback_data_cb(ma_device* dev, void* /*out*/, const void* in, ma_uint32 frame_count) { diff --git a/core/src/audio/audio_engine.h b/core/src/audio/audio_engine.h index 81af736..e08ce56 100644 --- a/core/src/audio/audio_engine.h +++ b/core/src/audio/audio_engine.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -92,6 +93,10 @@ struct AudioParams { uint32_t frame_ms = 20; std::string capture_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices std::string playback_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices + // External capture: the mic is fed via inject_capture/vc_stream_feed_pcm (e.g. iOS VPIO), + // so start() skips opening the hardware capture device. Derived from the MIC LocalStream's + // external_feed flag in vc_client::ensure_audio_running(). + bool external_capture = false; }; // One enumerated device, returned by AudioEngine::enumerate_devices(). `id` is an internal, @@ -202,6 +207,18 @@ class AudioEngine { using PcmSink = void(*)(void*, uint32_t, uint32_t, const int16_t*, size_t, uint32_t, uint32_t); void set_pcm_sink(PcmSink cb, void* user); + // External mixed-output sink (iOS VPIO). Receives the FINAL mixed PCM (post output-volume, + // all remote streams summed) on the mixer-timer thread when external playback is enabled. + // Matching signature to vc_mixed_output_cb (cast at the C-ABI boundary). Pass nullptr to + // disable. Thread-safe (atomic store; the timer read is relaxed-load). + using MixedSink = void(*)(void*, const int16_t*, size_t, uint32_t, uint32_t); + void set_mixed_output_sink(MixedSink cb, void* user); + + // External-playback mode (iOS VPIO): when enabled, start() does NOT open a hardware + // playback device; a timer thread drives the mixer (on_playback) on a ~20 ms cadence and + // ships the final mix to the mixed-output sink. Set before start() (or apply via restart). + void set_external_playback(bool enable) { external_playback_ = enable; } + // External PCM feed overload: stereo-aware variant of inject_capture. samples_per_channel // is samples per channel; total samples written = samples_per_channel * channels. void inject_capture(int kind, const int16_t* pcm, size_t samples_per_channel, int channels); @@ -336,6 +353,14 @@ class AudioEngine { bool capture_started_ = false; bool playback_started_ = false; + // External-playback mode (iOS VPIO): no hardware playback device; this timer thread drives + // on_playback() on a ~20 ms cadence and delivers the final mix to mixed_sink_. mixer_scratch_ + // is pre-sized in start() (frame_samples_ * playback_channels) so the loop never allocates. + std::thread mixer_timer_thread_; + std::atomic mixer_timer_stop_{false}; + std::vector mixer_scratch_; + void run_mixer_timer(); + #ifdef VOICECAT_HAS_LOOPBACK // Desktop-audio loopback capture (SCREEN_AUDIO) — own lifecycle, decoupled from // capture_device_/playback_device_ start/stop (a screen-share can start/stop independently @@ -491,6 +516,13 @@ class AudioEngine { std::atomic pcm_sink_{nullptr}; std::atomic pcm_sink_user_{nullptr}; + // External mixed-output sink + mode flag (iOS VPIO). mixed_sink_ is written by + // set_mixed_output_sink (any thread); read by run_mixer_timer via relaxed load. + // external_playback_ is read in start() to gate hardware-playback-device creation. + std::atomic mixed_sink_{nullptr}; + std::atomic mixed_sink_user_{nullptr}; + bool external_playback_ = false; + #ifdef VOICECAT_HAS_OPUS ::OpusDREDDecoder* dred_dec_ = nullptr; // shared DRED decoder; null if unsupported #endif diff --git a/core/src/core/client.cpp b/core/src/core/client.cpp index a3c6744..11b5518 100644 --- a/core/src/core/client.cpp +++ b/core/src/core/client.cpp @@ -1086,8 +1086,14 @@ void vc_client::ensure_audio_running() { if (it != local_streams_.end()) { p.capture_device_id = it->second.capture_device_id; p.capture_channels = it->second.capture_channels; + // iOS VPIO: when the mic is fed externally, skip the hardware capture device — the + // Swift AVAudioEngine VPIO path feeds processed mic PCM via vc_stream_feed_pcm. + p.external_capture = it->second.external_feed; } } + // iOS VPIO: skip the hardware playback device and drive the mixer on a timer, delivering the + // final mix to the mixed-output sink for the Swift VPIO renderer (vc_set_external_playback). + audio_engine_.set_external_playback(external_playback_.load(std::memory_order_acquire)); audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples, int channels) { on_capture_frame(kind, pcm, samples, channels); }); @@ -1518,6 +1524,20 @@ vc_result vc_client::set_pcm_sink(vc_pcm_sink_cb cb, void* user) { return VC_OK; } +vc_result vc_client::set_mixed_output_sink(vc_mixed_output_cb cb, void* user) { + audio_engine_.set_mixed_output_sink( + reinterpret_cast(cb), user); + return VC_OK; +} + +vc_result vc_client::set_external_playback(bool enable) { + external_playback_.store(enable, std::memory_order_release); + // Stored on the engine too; takes effect on the next start()/vc_audio_restart() (matching + // the vc_set_capture_channels "apply on next restart" contract). + audio_engine_.set_external_playback(enable); + return VC_OK; +} + vc_result vc_client::test_inject_capture(uint32_t stream_id, const int16_t* pcm, size_t samples) { return stream_feed_pcm(stream_id, pcm, samples, 1); } @@ -1898,6 +1918,12 @@ vc_result vc_client::stream_feed_pcm(uint32_t, const int16_t*, size_t, uint32_t) vc_result vc_client::set_pcm_sink(vc_pcm_sink_cb, void*) { return VC_ERR_NOT_IMPLEMENTED; } +vc_result vc_client::set_mixed_output_sink(vc_mixed_output_cb, void*) { + return VC_ERR_NOT_IMPLEMENTED; +} +vc_result vc_client::set_external_playback(bool) { + return VC_ERR_NOT_IMPLEMENTED; +} vc_result vc_client::test_inject_capture(uint32_t, const int16_t*, size_t) { return VC_ERR_NOT_IMPLEMENTED; } diff --git a/core/src/core/client.h b/core/src/core/client.h index df22fcd..339acd8 100644 --- a/core/src/core/client.h +++ b/core/src/core/client.h @@ -89,6 +89,10 @@ struct vc_client { // External PCM sink (see voicecat.h: vc_set_pcm_sink). Delegates to AudioEngine. vc_result set_pcm_sink(vc_pcm_sink_cb cb, void* user); + // External mixed-output sink + external-playback mode (iOS VPIO; see voicecat.h). + vc_result set_mixed_output_sink(vc_mixed_output_cb cb, void* user); + vc_result set_external_playback(bool enable); + // TEST-ONLY (see voicecat.h) — deprecated alias for stream_feed_pcm(..., channels=1). vc_result test_inject_capture(uint32_t stream_id, const int16_t* pcm, size_t samples); @@ -287,6 +291,11 @@ struct vc_client { std::atomic current_input_mode_{VC_INPUT_VOICE_ACTIVATION}; std::atomic ptt_active_{false}; std::atomic vad_threshold_{0.025f}; // remembered across mode switches + + // External-playback mode (iOS VPIO): when true, ensure_audio_running() configures the + // AudioEngine to skip its hardware playback device and drive the mixer on a timer instead, + // delivering the final mix to the mixed-output sink. Set via vc_set_external_playback. + std::atomic external_playback_{false}; std::unique_ptr mic_vad_; // teardown_voice() is called both from run_io()'s own cleanup (on the io_thread_, when diff --git a/core/src/voicecat.cpp b/core/src/voicecat.cpp index bdcf861..4a41b26 100644 --- a/core/src/voicecat.cpp +++ b/core/src/voicecat.cpp @@ -158,6 +158,16 @@ vc_result vc_set_pcm_sink(vc_client* c, vc_pcm_sink_cb cb, void* user) { return c->set_pcm_sink(cb, user); } +vc_result vc_set_mixed_output_sink(vc_client* c, vc_mixed_output_cb cb, void* user) { + if (c == nullptr) return VC_ERR_INVALID_ARG; + return c->set_mixed_output_sink(cb, user); +} + +vc_result vc_set_external_playback(vc_client* c, int enable) { + if (c == nullptr) return VC_ERR_INVALID_ARG; + return c->set_external_playback(enable != 0); +} + vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels) { if (c == nullptr) return VC_ERR_INVALID_ARG; return c->set_capture_channels(stream_id, channels); diff --git a/docs/voice.md b/docs/voice.md index b9b7b3c..0ede2c1 100644 --- a/docs/voice.md +++ b/docs/voice.md @@ -215,6 +215,23 @@ Each receiver keeps an **adaptive jitter buffer per ssrc**. `AVAudioSession`, then restarts the devices (`vc_audio_restart`) so they reopen against the new route — mirroring TeamTalk5's `closeSoundDevices`/`initSoundInputDevice`/ `initSoundOutputDevice` pattern. +- **iOS voice processing (AEC/NS/AGC) — native VPIO path.** Real iOS echo cancellation, noise + suppression and AGC are provided ONLY by Apple's **Voice-Processing I/O audio unit (VPIO)**, + *not* by the `AVAudioSession` mode alone. The core uses miniaudio's plain `RemoteIO` audio + units, which never engage VPIO — so `.voiceChat` mode by itself yields no AEC. For VPIO to + cancel echo it must own BOTH the mic capture and the remote-audio playback (it subtracts the + played-back signal from the mic), so on the AEC presets (Voice Chat / Bluetooth Headset HFP / + Wired Headset) the Swift layer runs a native `AVAudioEngine` with + `inputNode.setVoiceProcessingEnabled(true)` and the core runs in **external mode**: + - **Mic:** the MIC stream is started with `vc_stream_desc.external_feed=1`; the VPIO input tap + feeds processed mic PCM via `vc_stream_feed_pcm`. The core skips its hardware capture device + (`AudioParams.external_capture`). + - **Playback:** `vc_set_external_playback(1)` makes the core skip its hardware playback device; + a mixer-timer thread drives decode+mix on a ~20 ms cadence and delivers the FINAL mixed PCM + via `vc_set_mixed_output_sink`. The Swift engine renders that through the VPIO output, so + VPIO has its echo-cancellation reference signal. + The Stereo Mic / Studio / A2DP presets keep the miniaudio path (they want raw / stereo / + no-AEC routing that VPIO can't provide — VPIO forces mono). - **DSP engine: see §11.** The original plan was `webrtc-audio-processing` (AEC + NS + AGC + VAD in one tuned module, BSD-licensed) — but it has no working Windows/MSVC build upstream (confirmed via its own issue tracker: GCC-only Meson build, MinGW support unfinished, hard diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c98d6ec..73e2dcb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,14 @@ if(VOICECAT_USE_VCPKG_DEPS) target_include_directories(test_plc_cap PRIVATE ${VC_TEST_INTERNAL_INCLUDES}) add_test(NAME plc_cap COMMAND test_plc_cap) + # External playback (iOS VPIO): the mixer-timer thread drives decode+mix with NO hardware + # device and delivers the final mix to the mixed-output sink. White-box AudioEngine test. + add_executable(test_external_playback test_external_playback.cpp) + target_link_libraries(test_external_playback PRIVATE voicecat::voicecat) + target_compile_features(test_external_playback PRIVATE cxx_std_20) + target_include_directories(test_external_playback PRIVATE ${VC_TEST_INTERNAL_INCLUDES}) + add_test(NAME external_playback COMMAND test_external_playback) + # M2 exit criterion: two headless clients relay encrypted Opus frames via the SFU. add_executable(test_m2_voice test_m2_voice.cpp) target_link_libraries(test_m2_voice PRIVATE voicecat::server) diff --git a/tests/test_external_playback.cpp b/tests/test_external_playback.cpp new file mode 100644 index 0000000..85528ee --- /dev/null +++ b/tests/test_external_playback.cpp @@ -0,0 +1,146 @@ +/* + * test_external_playback — verifies AudioEngine's external-playback mode (iOS VPIO path). + * + * When external playback is enabled, the engine opens NO hardware playback device; a mixer-timer + * thread drives decode+mix on a ~20ms cadence and delivers the FINAL mixed PCM to the + * mixed-output sink (the Swift AVAudioEngine VPIO renderer consumes this). This test asserts: + * 1. The mixed sink fires steadily on the timer thread (count grows over time) with the right + * format (48kHz, stereo), and carries real energy while a stream is being decoded. + * 2. The per-stream pcm_sink still fires concurrently (both taps coexist). + * 3. With no remote streams, the mixed sink KEEPS firing (silent-but-present blocks) so the + * renderer has a continuous clock. + * + * White-box: constructs AudioEngine directly (no server, no audio hardware needed) — the timer + * thread drives the mixer with no ma_device, which is the core new behavior under test. + */ +#include +#include +#include +#include +#include +#include + +#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS) + +#include "audio/audio_engine.h" +#include "codec/opus_codec.h" + +static int g_failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL [%s:%d]: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// Shared state written by the sink callbacks (timer thread) and read by main. +struct MixedSinkState { + std::atomic calls{0}; + std::atomic max_energy{0}; + std::atomic last_channels{0}; + std::atomic last_sample_rate{0}; +}; +static MixedSinkState g_mixed; +static std::atomic g_pcm_sink_calls{0}; + +static void mixed_cb(void* user, const int16_t* pcm, size_t spc, uint32_t ch, uint32_t sr) { + auto* s = static_cast(user); + s->calls.fetch_add(1, std::memory_order_relaxed); + s->last_channels.store(ch, std::memory_order_relaxed); + s->last_sample_rate.store(sr, std::memory_order_relaxed); + int64_t e = 0; + for (size_t i = 0; i < spc * ch; ++i) e += std::abs(static_cast(pcm[i])); + int64_t prev = s->max_energy.load(std::memory_order_relaxed); + while (e > prev && !s->max_energy.compare_exchange_weak(prev, e, std::memory_order_relaxed)) { + } +} + +static void pcm_cb(void*, uint32_t, uint32_t, const int16_t*, size_t, uint32_t, uint32_t) { + g_pcm_sink_calls.fetch_add(1, std::memory_order_relaxed); +} + +int main() { + voicecat::audio::AudioEngine engine; + engine.set_external_playback(true); + engine.set_mixed_output_sink(&mixed_cb, &g_mixed); + engine.set_pcm_sink(&pcm_cb, nullptr); + + voicecat::audio::AudioParams p; + p.sample_rate = 48000; + p.capture_channels = 1; + p.playback_channels = 2; + p.frame_ms = 20; + CHECK(engine.start(p)); // no hardware device opened — the timer thread drives the mixer + + voicecat::codec::OpusParams op; + op.sample_rate = 48000; + op.frame_ms = 20; + op.stereo = false; + int frame_samples = voicecat::codec::opus_frame_samples(op); // 960 + + voicecat::codec::OpusEncoder enc; + CHECK(enc.init(op)); + std::vector sine(static_cast(frame_samples)); + for (int i = 0; i < frame_samples; ++i) { + float t = static_cast(i) / 48000.0f; + sine[i] = static_cast(std::sin(2.0f * 3.14159265f * 440.0f * t) * 20000.0f); + } + uint8_t opus_buf[1500]; + int opus_len = enc.encode(sine.data(), frame_samples, opus_buf, sizeof(opus_buf)); + CHECK(opus_len > 0); + + const uint32_t ssrc = 1; + engine.init_recv_stream(ssrc, op, /*user_id=*/7, /*stream_id=*/3); + + // ── Phase 1: feed ~600ms of real frames; the timer must decode + mix them. ────────── + uint32_t ts = 0; + for (int i = 0; i < 30; ++i) { // 30 * 20ms = 600ms of audio + voicecat::audio::JitterBuffer::Frame f; + f.seq = static_cast(i); + f.timestamp = ts; + f.fec_present = false; + f.payload.assign(opus_buf, opus_buf + opus_len); + engine.push_recv_frame(ssrc, std::move(f)); + ts += static_cast(frame_samples); + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + int active_calls = g_mixed.calls.load(std::memory_order_relaxed); + std::printf("external_playback: phase1 mixed-sink calls=%d max_energy=%lld pcm_sink=%d\n", + active_calls, static_cast(g_mixed.max_energy.load()), + g_pcm_sink_calls.load()); + // ~25 blocks expected at a 20ms cadence over 500ms; allow generous slack for scheduler/debug. + CHECK(active_calls >= 10); + CHECK(g_mixed.max_energy.load(std::memory_order_relaxed) > 0); // real decoded audio in the mix + CHECK(g_mixed.last_channels.load(std::memory_order_relaxed) == 2); + CHECK(g_mixed.last_sample_rate.load(std::memory_order_relaxed) == 48000); + CHECK(g_pcm_sink_calls.load(std::memory_order_relaxed) > 0); // per-stream tap coexists + + // ── Phase 2: remove the stream; the mixed sink must KEEP firing (silent blocks). ───── + engine.remove_stream(ssrc); + int before = g_mixed.calls.load(std::memory_order_relaxed); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + int after = g_mixed.calls.load(std::memory_order_relaxed); + std::printf("external_playback: phase2 silent blocks delivered=%d\n", after - before); + CHECK(after - before >= 5); // continuous clock even with nothing to play + + engine.stop(); // joins the mixer-timer thread + enc.destroy(); + + if (g_failures == 0) { + std::printf("external_playback: all checks passed\n"); + return 0; + } + std::printf("external_playback: %d failure(s)\n", g_failures); + return 1; +} + +#else + +int main() { + std::printf("external_playback: SKIP (VOICECAT_HAS_AUDIO or VOICECAT_HAS_OPUS not defined)\n"); + return 0; +} + +#endif