feat(ios): real echo cancellation/NR via native Voice-Processing engine
iOS "voice chat" had echo and no noise suppression: real iOS AEC/NS/AGC come only from Apple's Voice-Processing I/O unit (VPIO), but the core plays/captures via miniaudio's plain RemoteIO units, so .voiceChat mode alone never engaged AEC. Core (ABI PATCH 1->2): - vc_set_mixed_output_sink + vc_set_external_playback. In external mode the AudioEngine opens no hardware playback device; a mixer-timer thread drives on_playback (decode+mix) on a ~20ms cadence and ships the final mix to the sink. start() also skips the hardware capture device when the MIC stream is external_feed (AudioParams.external_capture). - New white-box test test_external_playback (drives the timer with no hw). iOS/Swift: - StreamDescriptor.externalFeed; VoiceCatClient.setMixedOutputSink / setExternalPlayback wrappers. - IOSVoiceProcessingEngine: AVAudioEngine + setVoiceProcessingEnabled; mic tap -> feedPcm, mixed-sink lock-free ring -> AVAudioSourceNode (both share the VPIO unit so AEC has its reference signal). - IOSAudioRouter.currentConfigUsesVoiceProcessing scopes VPIO to the AEC presets; SessionState join/leave + reconcileVoicePath() switch paths; Voice Chat defaults to speaker; Settings surfaces AEC/NS state. Known: pending on-device verification; a few bugs to fix afterward.
This commit is contained in:
@@ -33,6 +33,7 @@
|
||||
BBBB00000000000000000046 /* AccountsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002D /* AccountsView.swift */; };
|
||||
BBBB00000000000000000047 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002E /* SettingsView.swift */; };
|
||||
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB0000000000000000002F /* IOSAudioRouter.swift */; };
|
||||
BBBB00000000000000000F02 /* IOSVoiceProcessingEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */; };
|
||||
BBBB00000000000000000048 /* VoiceCatCore in Frameworks */ = {isa = PBXBuildFile; productRef = BBBB0000000000000000004A /* VoiceCatCore */; };
|
||||
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000002 /* BroadcastAudioPump.swift */; };
|
||||
CCCC00000000000000000011 /* BroadcastAudioRing.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCCC00000000000000000001 /* BroadcastAudioRing.swift */; };
|
||||
@@ -89,6 +90,7 @@
|
||||
BBBB0000000000000000002D /* AccountsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountsView.swift; sourceTree = "<group>"; };
|
||||
BBBB0000000000000000002E /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = "<group>"; };
|
||||
BBBB0000000000000000002F /* IOSAudioRouter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSAudioRouter.swift; sourceTree = "<group>"; };
|
||||
BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOSVoiceProcessingEngine.swift; sourceTree = "<group>"; };
|
||||
CCCC00000000000000000001 /* BroadcastAudioRing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioRing.swift; sourceTree = "<group>"; };
|
||||
CCCC00000000000000000002 /* BroadcastAudioPump.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BroadcastAudioPump.swift; sourceTree = "<group>"; };
|
||||
CCCC00000000000000000003 /* SampleHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SampleHandler.swift; sourceTree = "<group>"; };
|
||||
@@ -143,6 +145,7 @@
|
||||
BBBB00000000000000000019 /* SessionState.swift */,
|
||||
BBBB0000000000000000001A /* AudioSessionManager.swift */,
|
||||
BBBB0000000000000000002F /* IOSAudioRouter.swift */,
|
||||
BBBB00000000000000000F01 /* IOSVoiceProcessingEngine.swift */,
|
||||
BBBB0000000000000000001B /* ServerListStore.swift */,
|
||||
BBBB0000000000000000001C /* SavedServer.swift */,
|
||||
CCCC00000000000000000002 /* BroadcastAudioPump.swift */,
|
||||
@@ -296,6 +299,7 @@
|
||||
BBBB00000000000000000032 /* SessionState.swift in Sources */,
|
||||
BBBB00000000000000000033 /* AudioSessionManager.swift in Sources */,
|
||||
BBBB0000000000000000004B /* IOSAudioRouter.swift in Sources */,
|
||||
BBBB00000000000000000F02 /* IOSVoiceProcessingEngine.swift in Sources */,
|
||||
BBBB00000000000000000034 /* ServerListStore.swift in Sources */,
|
||||
BBBB00000000000000000035 /* SavedServer.swift in Sources */,
|
||||
CCCC00000000000000000010 /* BroadcastAudioPump.swift in Sources */,
|
||||
|
||||
@@ -15,6 +15,12 @@ final class AudioSessionManager {
|
||||
/// channel count (e.g. when switching stereo → mono) without going through `SessionState`.
|
||||
var activeMicStreamId: UInt32?
|
||||
|
||||
/// Set by `SessionState`. Invoked by `IOSAudioRouter` after an audio-config change so the
|
||||
/// voice path (native VPIO vs the core's miniaudio path) can be restarted to match the new
|
||||
/// preset/route when the mic is active. No-op when not in voice. See
|
||||
/// `SessionState.reconcileVoicePath()` and `IOSVoiceProcessingEngine`.
|
||||
var reconcileVoicePath: (() -> Void)?
|
||||
|
||||
/// Tracks whether WE activated the session. The session must be active whenever the
|
||||
/// AudioEngine is running (for capture OR playback), so it is activated when any audio
|
||||
/// needs to play (a remote stream started OR the user joins voice) and only deactivated
|
||||
|
||||
@@ -307,6 +307,14 @@ final class IOSAudioRouter: ObservableObject {
|
||||
return .custom
|
||||
}
|
||||
|
||||
/// Whether the current configuration should use the native iOS Voice-Processing path (VPIO:
|
||||
/// real AEC/NS/AGC via `IOSVoiceProcessingEngine`). True exactly when `applyConfiguration`
|
||||
/// selects the `.voiceChat` AVAudioSession mode — mono + standard processing + not A2DP
|
||||
/// (A2DP / stereo / raw modes can't use VPIO, so they keep the core's miniaudio path).
|
||||
var currentConfigUsesVoiceProcessing: Bool {
|
||||
captureChannels == .mono && micMode == .standard && bluetoothMode != .builtInMicBtA2dp
|
||||
}
|
||||
|
||||
// MARK: - Apply configuration
|
||||
|
||||
/// Apply the full audio configuration to AVAudioSession. Call this before the core
|
||||
@@ -573,6 +581,8 @@ final class IOSAudioRouter: ObservableObject {
|
||||
savePreferences()
|
||||
applyConfiguration()
|
||||
refreshRoutes()
|
||||
// VPIO class or route may have changed — restart the voice path if mic is active.
|
||||
AudioSessionManager.shared.reconcileVoicePath?()
|
||||
}
|
||||
|
||||
func setForceSpeaker(_ on: Bool) {
|
||||
@@ -580,6 +590,8 @@ final class IOSAudioRouter: ObservableObject {
|
||||
savePreferences()
|
||||
applyConfiguration()
|
||||
refreshRoutes()
|
||||
// Route changed under a possibly-running VPIO engine — reconcile if mic is active.
|
||||
AudioSessionManager.shared.reconcileVoicePath?()
|
||||
}
|
||||
|
||||
func selectMicMode(_ mode: MicMode) {
|
||||
@@ -587,6 +599,8 @@ final class IOSAudioRouter: ObservableObject {
|
||||
savePreferences()
|
||||
applyConfiguration()
|
||||
updateWarnings()
|
||||
// Standard↔Raw flips the VPIO class — reconcile if mic is active.
|
||||
AudioSessionManager.shared.reconcileVoicePath?()
|
||||
}
|
||||
|
||||
func selectCaptureChannels(_ channels: CaptureChannels) {
|
||||
@@ -603,6 +617,8 @@ final class IOSAudioRouter: ObservableObject {
|
||||
// route), then capture — avoiding the race where stereo capture activation drops
|
||||
// A2DP before the playback device has a chance to claim the route.
|
||||
_ = AudioSessionManager.shared.client?.audioRestart()
|
||||
// Mono↔stereo flips the VPIO class (stereo can't use VPIO) — reconcile if mic is active.
|
||||
AudioSessionManager.shared.reconcileVoicePath?()
|
||||
}
|
||||
|
||||
// MARK: - Presets
|
||||
@@ -617,6 +633,10 @@ final class IOSAudioRouter: ObservableObject {
|
||||
micMode = preset.micMode
|
||||
captureChannels = preset.captureChannels
|
||||
|
||||
// Voice Chat is a phone-call experience — default to the loud speaker so output doesn't
|
||||
// land on the quiet earpiece (receiver). Still yields to connected BT/wired output.
|
||||
if preset == .voiceChat { forceSpeaker = true }
|
||||
|
||||
if preset.usesBuiltInMic {
|
||||
// Find the built-in mic port from available inputs and select it.
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
@@ -648,6 +668,9 @@ final class IOSAudioRouter: ObservableObject {
|
||||
// count is stored. Playback opens first (commits A2DP route), then capture.
|
||||
_ = AudioSessionManager.shared.client?.audioRestart()
|
||||
refreshRoutes()
|
||||
// The preset may have flipped the VPIO class (and/or the route) — restart the voice path
|
||||
// if the mic is active so AEC/NS engage (or disengage) to match the new preset.
|
||||
AudioSessionManager.shared.reconcileVoicePath?()
|
||||
logger.info("applyPreset — \(preset.rawValue)")
|
||||
}
|
||||
|
||||
|
||||
247
clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift
Normal file
247
clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift
Normal file
@@ -0,0 +1,247 @@
|
||||
import AVFoundation
|
||||
import Darwin
|
||||
import os
|
||||
import VoiceCatCore
|
||||
|
||||
private let logger = Logger(subsystem: "cat.voice.VoiceCatiOS", category: "IOSVoiceProcessingEngine")
|
||||
|
||||
/// In-process single-producer/single-consumer int16 PCM ring for the VPIO playback path.
|
||||
///
|
||||
/// producer = the core's mixer-timer thread (the `vc_set_mixed_output_sink` callback)
|
||||
/// consumer = the `AVAudioSourceNode` render thread
|
||||
///
|
||||
/// Heap-backed (not shared memory like `BroadcastAudioRing`), but the same discipline: aligned
|
||||
/// 64-bit monotonic indices with `OSMemoryBarrier` for acquire/release ordering. Both the C
|
||||
/// callback and the render block are real-time — they only do index math + a memcpy here, never
|
||||
/// lock or allocate.
|
||||
final class PCMRing {
|
||||
private let data: UnsafeMutablePointer<Int16>
|
||||
private let capacity: Int
|
||||
private var writeIdx: UInt64 = 0
|
||||
private var readIdx: UInt64 = 0
|
||||
|
||||
init(capacitySamples: Int) {
|
||||
capacity = capacitySamples
|
||||
data = UnsafeMutablePointer<Int16>.allocate(capacity: capacitySamples)
|
||||
data.initialize(repeating: 0, count: capacitySamples)
|
||||
}
|
||||
deinit { data.deallocate() }
|
||||
|
||||
/// Producer: append `count` interleaved int16 samples. Drops the chunk if it doesn't fit
|
||||
/// (better to skip than tear). Single producer only (the core mixer-timer thread).
|
||||
func write(_ src: UnsafePointer<Int16>, count: Int) {
|
||||
guard count > 0, count <= capacity else { return }
|
||||
let w = writeIdx
|
||||
OSMemoryBarrier()
|
||||
let r = readIdx
|
||||
if capacity - Int(w &- r) < count { return } // full: drop
|
||||
var idx = Int(w % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = count
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(data + idx).update(from: src + off, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
OSMemoryBarrier()
|
||||
writeIdx = w &+ UInt64(count)
|
||||
}
|
||||
|
||||
/// Consumer: read up to `count` interleaved int16 samples into `dst`; returns the number
|
||||
/// read (the rest is the caller's to silence-fill). Single consumer only (render thread).
|
||||
func read(into dst: UnsafeMutablePointer<Int16>, count: Int) -> Int {
|
||||
let r = readIdx
|
||||
OSMemoryBarrier()
|
||||
let w = writeIdx
|
||||
let available = Int(w &- r)
|
||||
if available <= 0 { return 0 }
|
||||
let n = min(available, count)
|
||||
var idx = Int(r % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = n
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(dst + off).update(from: data + idx, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
OSMemoryBarrier()
|
||||
readIdx = r &+ UInt64(n)
|
||||
return n
|
||||
}
|
||||
|
||||
/// Discard everything buffered — call before (re)starting so stale pre-roll isn't played.
|
||||
func reset() { OSMemoryBarrier(); readIdx = writeIdx }
|
||||
}
|
||||
|
||||
/// Native iOS voice-processing audio path (docs/voice.md §8 "iOS voice processing").
|
||||
///
|
||||
/// Real iOS echo cancellation / noise suppression / AGC come ONLY from Apple's Voice-Processing
|
||||
/// I/O unit (VPIO), which `AVAudioEngine.setVoiceProcessingEnabled(true)` enables. For VPIO to
|
||||
/// cancel echo it must own BOTH the mic capture and the remote-audio playback (it subtracts the
|
||||
/// played-back signal from the mic), so on the AEC presets this engine drives both directions and
|
||||
/// the core runs in external mode (no hardware devices):
|
||||
/// - **Mic → core:** a tap on the VPIO input node → 48 kHz int16 → `client.feedPcm(micStreamId)`.
|
||||
/// - **core → speaker:** the core's mixed-output sink fills `ring`; an `AVAudioSourceNode` pulls
|
||||
/// from it and renders through the VPIO output, giving AEC its reference signal.
|
||||
///
|
||||
/// Lifecycle is driven by `SessionState` join/leave. The Stereo Mic / Studio / A2DP presets keep
|
||||
/// the core's miniaudio path instead (they want raw / stereo / no-AEC routing VPIO can't provide).
|
||||
@MainActor
|
||||
final class IOSVoiceProcessingEngine {
|
||||
static let shared = IOSVoiceProcessingEngine()
|
||||
|
||||
private(set) var isRunning = false
|
||||
|
||||
private let engine = AVAudioEngine()
|
||||
private var sourceNode: AVAudioSourceNode?
|
||||
private weak var client: VoiceCatClient?
|
||||
private var micStreamId: UInt32 = 0
|
||||
|
||||
// 48 kHz stereo Float32 (deinterleaved) — the format the source node renders and the engine
|
||||
// processes in. The core delivers 48 kHz stereo int16 via the mixed-output sink.
|
||||
private let outFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false)!
|
||||
|
||||
// Playback ring (mixed remote audio): ~0.5 s of 48 kHz stereo int16. Filled by the core's
|
||||
// mixer-timer thread, drained by the source-node render thread.
|
||||
private let ring = PCMRing(capacitySamples: 48000 * 2 / 2)
|
||||
// Render-thread scratch for deinterleaving — pre-allocated so the render block never allocates.
|
||||
private let renderScratchFrames = 8192
|
||||
private let renderScratch: UnsafeMutablePointer<Int16>
|
||||
|
||||
// Mic-feed converter (input-node format → 48 kHz int16) and its target buffer. Owned here so
|
||||
// the (background) tap block reuses them instead of allocating per callback.
|
||||
private var micConverter: AVAudioConverter?
|
||||
private var micTargetFormat: AVAudioFormat?
|
||||
|
||||
private init() {
|
||||
renderScratch = UnsafeMutablePointer<Int16>.allocate(capacity: renderScratchFrames * 2)
|
||||
renderScratch.initialize(repeating: 0, count: renderScratchFrames * 2)
|
||||
}
|
||||
|
||||
/// Start the VPIO engine for an active mic stream. The caller must have already enabled
|
||||
/// external playback on the core (`client.setExternalPlayback(true)` + `audioRestart()`) and
|
||||
/// started the MIC stream with `externalFeed: true`.
|
||||
func start(client: VoiceCatClient, micStreamId: UInt32, captureChannels: UInt32) {
|
||||
guard !isRunning else { return }
|
||||
self.client = client
|
||||
self.micStreamId = micStreamId
|
||||
ring.reset()
|
||||
|
||||
// Enable the voice-processing I/O unit (AEC/NS/AGC) on the shared input+output unit.
|
||||
do {
|
||||
try engine.inputNode.setVoiceProcessingEnabled(true)
|
||||
} catch {
|
||||
logger.error("setVoiceProcessingEnabled failed: \(error.localizedDescription) — AEC unavailable")
|
||||
}
|
||||
|
||||
// ── Playback: source node pulls mixed PCM from the ring through the VPIO output. ──
|
||||
let ring = self.ring
|
||||
let scratch = self.renderScratch
|
||||
let scratchFrames = self.renderScratchFrames
|
||||
let src = AVAudioSourceNode(format: outFormat) { _, _, frameCount, ablPtr in
|
||||
let frames = Int(frameCount)
|
||||
let abl = UnsafeMutableAudioBufferListPointer(ablPtr)
|
||||
let n = min(frames, scratchFrames)
|
||||
let got = ring.read(into: scratch, count: n * 2) / 2 // interleaved stereo → frames
|
||||
// Deinterleave int16 → Float32 per channel; silence-fill any underrun tail.
|
||||
let scale: Float = 1.0 / 32768.0
|
||||
for ch in 0..<abl.count {
|
||||
guard let base = abl[ch].mData?.assumingMemoryBound(to: Float.self) else { continue }
|
||||
for i in 0..<frames {
|
||||
if i < got {
|
||||
let s = scratch[i * 2 + min(ch, 1)]
|
||||
base[i] = Float(s) * scale
|
||||
} else {
|
||||
base[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
return noErr
|
||||
}
|
||||
sourceNode = src
|
||||
engine.attach(src)
|
||||
engine.connect(src, to: engine.mainMixerNode, format: outFormat)
|
||||
|
||||
// ── Mic: tap the VPIO input node, convert to 48 kHz int16, feed the core. ──
|
||||
let inFormat = engine.inputNode.outputFormat(forBus: 0)
|
||||
let targetCh = max(1, min(2, captureChannels))
|
||||
let target = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000,
|
||||
channels: AVAudioChannelCount(targetCh), interleaved: true)
|
||||
micTargetFormat = target
|
||||
micConverter = (target != nil && inFormat.sampleRate > 0)
|
||||
? AVAudioConverter(from: inFormat, to: target!) : nil
|
||||
if micConverter == nil {
|
||||
logger.error("mic converter unavailable (in=\(inFormat)) — mic will not transmit")
|
||||
}
|
||||
|
||||
let c = client
|
||||
let sid = micStreamId
|
||||
let converter = micConverter
|
||||
let tgt = micTargetFormat
|
||||
engine.inputNode.installTap(onBus: 0, bufferSize: 960, format: inFormat) { buffer, _ in
|
||||
guard let converter, let tgt else { return }
|
||||
// Convert this tap buffer to 48 kHz int16. Output capacity scaled for any upsample.
|
||||
let ratio = tgt.sampleRate / buffer.format.sampleRate
|
||||
let outCap = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 16)
|
||||
guard let outBuf = AVAudioPCMBuffer(pcmFormat: tgt, frameCapacity: outCap) else { return }
|
||||
var fed = false
|
||||
let status = converter.convert(to: outBuf, error: nil) { _, outStatus in
|
||||
if fed { outStatus.pointee = .noDataNow; return nil }
|
||||
fed = true
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status != .error, outBuf.frameLength > 0,
|
||||
let chData = outBuf.int16ChannelData else { return }
|
||||
let spc = Int(outBuf.frameLength)
|
||||
// int16 interleaved → channelData[0] is the interleaved buffer for interleaved formats.
|
||||
c.feedPcm(streamId: sid, pcm: chData[0], samplesPerChannel: spc, channels: targetCh)
|
||||
}
|
||||
|
||||
// ── Wire the core's mixed-output sink into the ring (C function pointer, no captures). ──
|
||||
let ringPtr = Unmanaged.passUnretained(self.ring).toOpaque()
|
||||
client.setMixedOutputSink({ user, pcm, spc, ch, _ in
|
||||
guard let user, let pcm else { return }
|
||||
let ring = Unmanaged<PCMRing>.fromOpaque(user).takeUnretainedValue()
|
||||
ring.write(pcm, count: spc * Int(ch))
|
||||
}, user: ringPtr)
|
||||
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
isRunning = true
|
||||
logger.info("VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh)")
|
||||
} catch {
|
||||
logger.error("VPIO engine start failed: \(error.localizedDescription)")
|
||||
teardown()
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop the VPIO engine. The caller is responsible for restoring the core's hardware playback
|
||||
/// afterwards (`client.setExternalPlayback(false)` + `audioRestart()`).
|
||||
func stop() {
|
||||
guard isRunning else { return }
|
||||
teardown()
|
||||
logger.info("VPIO engine stopped")
|
||||
}
|
||||
|
||||
private func teardown() {
|
||||
client?.setMixedOutputSink(nil, user: nil)
|
||||
engine.inputNode.removeTap(onBus: 0)
|
||||
if engine.isRunning { engine.stop() }
|
||||
try? engine.inputNode.setVoiceProcessingEnabled(false)
|
||||
if let src = sourceNode {
|
||||
engine.detach(src)
|
||||
sourceNode = nil
|
||||
}
|
||||
micConverter = nil
|
||||
micTargetFormat = nil
|
||||
ring.reset()
|
||||
isRunning = false
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,9 @@ final class SessionState {
|
||||
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
|
||||
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
|
||||
broadcastPump.start()
|
||||
// When IOSAudioRouter changes the audio config, restart the voice path if needed so the
|
||||
// native VPIO engine (AEC/NS/AGC) engages or disengages to match the new preset/route.
|
||||
AudioSessionManager.shared.reconcileVoicePath = { [weak self] in self?.reconcileVoicePath() }
|
||||
}
|
||||
|
||||
deinit {
|
||||
@@ -222,7 +225,21 @@ final class SessionState {
|
||||
addActivity("AVAudioSession activate failed: \(error)")
|
||||
return
|
||||
}
|
||||
let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic")
|
||||
|
||||
// VPIO path: on the AEC presets, the native AVAudioEngine does AEC/NS/AGC and the core
|
||||
// runs in external mode (no hardware mic/playback). The mic stream is started with
|
||||
// externalFeed so the core skips the hardware capture device; setExternalPlayback makes
|
||||
// it skip the hardware playback device and deliver the mix to IOSVoiceProcessingEngine.
|
||||
let useVPIO = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing
|
||||
if useVPIO {
|
||||
client.setExternalPlayback(true)
|
||||
client.audioRestart() // flip any already-running (pre-join) engine into external mode
|
||||
} else {
|
||||
client.setExternalPlayback(false)
|
||||
}
|
||||
|
||||
let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic",
|
||||
externalFeed: useVPIO)
|
||||
let (result, streamId) = client.startStream(desc)
|
||||
if result == .ok {
|
||||
voiceState.micActive = true
|
||||
@@ -241,17 +258,37 @@ final class SessionState {
|
||||
if channels != 1 {
|
||||
client.setCaptureChannels(streamId: streamId, channels: channels)
|
||||
}
|
||||
if useVPIO {
|
||||
IOSVoiceProcessingEngine.shared.start(
|
||||
client: client, micStreamId: streamId, captureChannels: channels)
|
||||
}
|
||||
} else {
|
||||
addActivity("Failed to start mic: \(result.description)")
|
||||
if useVPIO { // revert external-playback mode so remote audio still plays
|
||||
client.setExternalPlayback(false)
|
||||
client.audioRestart()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stopMicStream() {
|
||||
// Tear down the VPIO engine first (removes the mic tap + unregisters the mixed sink),
|
||||
// then stop the mic stream, then restore the core's hardware playback for any remaining
|
||||
// remote audio. Order matters: the mic stream must be gone before audioRestart so the
|
||||
// core opens a normal playback device (and no capture device — there's no mic stream).
|
||||
let wasVPIO = IOSVoiceProcessingEngine.shared.isRunning
|
||||
if wasVPIO {
|
||||
IOSVoiceProcessingEngine.shared.stop()
|
||||
}
|
||||
if voiceState.localStreamId != 0 {
|
||||
client.stopStream(voiceState.localStreamId)
|
||||
voiceState.localStreamId = 0
|
||||
AudioSessionManager.shared.activeMicStreamId = nil
|
||||
}
|
||||
if wasVPIO {
|
||||
client.setExternalPlayback(false)
|
||||
client.audioRestart() // reopen hardware playback (no mic stream → no hw capture)
|
||||
}
|
||||
voiceState.micActive = false
|
||||
voiceState.level = 0
|
||||
// Do NOT deactivate the AVAudioSession here — the user may still want to hear
|
||||
@@ -259,6 +296,19 @@ final class SessionState {
|
||||
// disconnecting from the server (see AppState.disconnect / .disconnected event).
|
||||
}
|
||||
|
||||
/// Restart the voice path when the audio config changes mid-call (driven by IOSAudioRouter).
|
||||
/// If VPIO is involved on either the current or desired side, restart the mic so the native
|
||||
/// voice-processing engine engages/disengages and re-binds to the new route. Pure miniaudio
|
||||
/// config tweaks need no restart — the core's own audioRestart (already issued) handles them.
|
||||
private func reconcileVoicePath() {
|
||||
guard voiceState.micActive else { return }
|
||||
let want = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing
|
||||
let have = IOSVoiceProcessingEngine.shared.isRunning
|
||||
guard want || have else { return }
|
||||
stopMicStream()
|
||||
doStartMicStream()
|
||||
}
|
||||
|
||||
// MARK: - Screen audio share
|
||||
|
||||
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
|
||||
|
||||
@@ -30,6 +30,23 @@ struct SettingsView: View {
|
||||
.accessibilityLabel("Speaker output")
|
||||
.accessibilityHint("Routes audio to the speaker instead of the earpiece when no headphones are connected.")
|
||||
|
||||
// Surface the voice-processing state. On the AEC presets the native iOS
|
||||
// Voice-Processing unit (VPIO) does echo cancellation, noise suppression and
|
||||
// automatic gain control; the other presets (stereo/studio/A2DP) can't use it.
|
||||
if router.currentConfigUsesVoiceProcessing {
|
||||
Label("Echo cancellation & noise suppression on (iOS voice processing)",
|
||||
systemImage: "waveform.badge.mic")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.accessibilityLabel("Echo cancellation and noise suppression are on")
|
||||
} else {
|
||||
Label("No echo cancellation in this preset (stereo / studio / A2DP)",
|
||||
systemImage: "waveform.slash")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.accessibilityLabel("Echo cancellation is off in this preset")
|
||||
}
|
||||
|
||||
if !router.hasBluetoothDevice && !router.hasWiredHeadset {
|
||||
Text("Connect Bluetooth headphones or a wired headset for more presets.")
|
||||
.font(.caption)
|
||||
|
||||
Reference in New Issue
Block a user