feat(ios): real echo cancellation/NR via native Voice-Processing engine
iOS "voice chat" had echo and no noise suppression: real iOS AEC/NS/AGC come only from Apple's Voice-Processing I/O unit (VPIO), but the core plays/captures via miniaudio's plain RemoteIO units, so .voiceChat mode alone never engaged AEC. Core (ABI PATCH 1->2): - vc_set_mixed_output_sink + vc_set_external_playback. In external mode the AudioEngine opens no hardware playback device; a mixer-timer thread drives on_playback (decode+mix) on a ~20ms cadence and ships the final mix to the sink. start() also skips the hardware capture device when the MIC stream is external_feed (AudioParams.external_capture). - New white-box test test_external_playback (drives the timer with no hw). iOS/Swift: - StreamDescriptor.externalFeed; VoiceCatClient.setMixedOutputSink / setExternalPlayback wrappers. - IOSVoiceProcessingEngine: AVAudioEngine + setVoiceProcessingEnabled; mic tap -> feedPcm, mixed-sink lock-free ring -> AVAudioSourceNode (both share the VPIO unit so AEC has its reference signal). - IOSAudioRouter.currentConfigUsesVoiceProcessing scopes VPIO to the AEC presets; SessionState join/leave + reconcileVoicePath() switch paths; Voice Chat defaults to speaker; Settings surfaces AEC/NS state. Known: pending on-device verification; a few bugs to fix afterward.
This commit is contained in:
247
clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift
Normal file
247
clients/apple/iOS/VoiceCatiOS/IOSVoiceProcessingEngine.swift
Normal file
@@ -0,0 +1,247 @@
|
||||
import AVFoundation
|
||||
import Darwin
|
||||
import os
|
||||
import VoiceCatCore
|
||||
|
||||
private let logger = Logger(subsystem: "cat.voice.VoiceCatiOS", category: "IOSVoiceProcessingEngine")
|
||||
|
||||
/// In-process single-producer/single-consumer int16 PCM ring for the VPIO playback path.
|
||||
///
|
||||
/// producer = the core's mixer-timer thread (the `vc_set_mixed_output_sink` callback)
|
||||
/// consumer = the `AVAudioSourceNode` render thread
|
||||
///
|
||||
/// Heap-backed (not shared memory like `BroadcastAudioRing`), but the same discipline: aligned
|
||||
/// 64-bit monotonic indices with `OSMemoryBarrier` for acquire/release ordering. Both the C
|
||||
/// callback and the render block are real-time — they only do index math + a memcpy here, never
|
||||
/// lock or allocate.
|
||||
final class PCMRing {
|
||||
private let data: UnsafeMutablePointer<Int16>
|
||||
private let capacity: Int
|
||||
private var writeIdx: UInt64 = 0
|
||||
private var readIdx: UInt64 = 0
|
||||
|
||||
init(capacitySamples: Int) {
|
||||
capacity = capacitySamples
|
||||
data = UnsafeMutablePointer<Int16>.allocate(capacity: capacitySamples)
|
||||
data.initialize(repeating: 0, count: capacitySamples)
|
||||
}
|
||||
deinit { data.deallocate() }
|
||||
|
||||
/// Producer: append `count` interleaved int16 samples. Drops the chunk if it doesn't fit
|
||||
/// (better to skip than tear). Single producer only (the core mixer-timer thread).
|
||||
func write(_ src: UnsafePointer<Int16>, count: Int) {
|
||||
guard count > 0, count <= capacity else { return }
|
||||
let w = writeIdx
|
||||
OSMemoryBarrier()
|
||||
let r = readIdx
|
||||
if capacity - Int(w &- r) < count { return } // full: drop
|
||||
var idx = Int(w % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = count
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(data + idx).update(from: src + off, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
OSMemoryBarrier()
|
||||
writeIdx = w &+ UInt64(count)
|
||||
}
|
||||
|
||||
/// Consumer: read up to `count` interleaved int16 samples into `dst`; returns the number
|
||||
/// read (the rest is the caller's to silence-fill). Single consumer only (render thread).
|
||||
func read(into dst: UnsafeMutablePointer<Int16>, count: Int) -> Int {
|
||||
let r = readIdx
|
||||
OSMemoryBarrier()
|
||||
let w = writeIdx
|
||||
let available = Int(w &- r)
|
||||
if available <= 0 { return 0 }
|
||||
let n = min(available, count)
|
||||
var idx = Int(r % UInt64(capacity))
|
||||
var off = 0
|
||||
var rem = n
|
||||
while rem > 0 {
|
||||
let chunk = min(rem, capacity - idx)
|
||||
(dst + off).update(from: data + idx, count: chunk)
|
||||
idx = (idx + chunk) % capacity
|
||||
off += chunk
|
||||
rem -= chunk
|
||||
}
|
||||
OSMemoryBarrier()
|
||||
readIdx = r &+ UInt64(n)
|
||||
return n
|
||||
}
|
||||
|
||||
/// Discard everything buffered — call before (re)starting so stale pre-roll isn't played.
|
||||
func reset() { OSMemoryBarrier(); readIdx = writeIdx }
|
||||
}
|
||||
|
||||
/// Native iOS voice-processing audio path (docs/voice.md §8 "iOS voice processing").
|
||||
///
|
||||
/// Real iOS echo cancellation / noise suppression / AGC come ONLY from Apple's Voice-Processing
|
||||
/// I/O unit (VPIO), which `AVAudioEngine.setVoiceProcessingEnabled(true)` enables. For VPIO to
|
||||
/// cancel echo it must own BOTH the mic capture and the remote-audio playback (it subtracts the
|
||||
/// played-back signal from the mic), so on the AEC presets this engine drives both directions and
|
||||
/// the core runs in external mode (no hardware devices):
|
||||
/// - **Mic → core:** a tap on the VPIO input node → 48 kHz int16 → `client.feedPcm(micStreamId)`.
|
||||
/// - **core → speaker:** the core's mixed-output sink fills `ring`; an `AVAudioSourceNode` pulls
|
||||
/// from it and renders through the VPIO output, giving AEC its reference signal.
|
||||
///
|
||||
/// Lifecycle is driven by `SessionState` join/leave. The Stereo Mic / Studio / A2DP presets keep
|
||||
/// the core's miniaudio path instead (they want raw / stereo / no-AEC routing VPIO can't provide).
|
||||
@MainActor
|
||||
final class IOSVoiceProcessingEngine {
|
||||
static let shared = IOSVoiceProcessingEngine()
|
||||
|
||||
private(set) var isRunning = false
|
||||
|
||||
private let engine = AVAudioEngine()
|
||||
private var sourceNode: AVAudioSourceNode?
|
||||
private weak var client: VoiceCatClient?
|
||||
private var micStreamId: UInt32 = 0
|
||||
|
||||
// 48 kHz stereo Float32 (deinterleaved) — the format the source node renders and the engine
|
||||
// processes in. The core delivers 48 kHz stereo int16 via the mixed-output sink.
|
||||
private let outFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false)!
|
||||
|
||||
// Playback ring (mixed remote audio): ~0.5 s of 48 kHz stereo int16. Filled by the core's
|
||||
// mixer-timer thread, drained by the source-node render thread.
|
||||
private let ring = PCMRing(capacitySamples: 48000 * 2 / 2)
|
||||
// Render-thread scratch for deinterleaving — pre-allocated so the render block never allocates.
|
||||
private let renderScratchFrames = 8192
|
||||
private let renderScratch: UnsafeMutablePointer<Int16>
|
||||
|
||||
// Mic-feed converter (input-node format → 48 kHz int16) and its target buffer. Owned here so
|
||||
// the (background) tap block reuses them instead of allocating per callback.
|
||||
private var micConverter: AVAudioConverter?
|
||||
private var micTargetFormat: AVAudioFormat?
|
||||
|
||||
private init() {
|
||||
renderScratch = UnsafeMutablePointer<Int16>.allocate(capacity: renderScratchFrames * 2)
|
||||
renderScratch.initialize(repeating: 0, count: renderScratchFrames * 2)
|
||||
}
|
||||
|
||||
/// Start the VPIO engine for an active mic stream. The caller must have already enabled
|
||||
/// external playback on the core (`client.setExternalPlayback(true)` + `audioRestart()`) and
|
||||
/// started the MIC stream with `externalFeed: true`.
|
||||
func start(client: VoiceCatClient, micStreamId: UInt32, captureChannels: UInt32) {
|
||||
guard !isRunning else { return }
|
||||
self.client = client
|
||||
self.micStreamId = micStreamId
|
||||
ring.reset()
|
||||
|
||||
// Enable the voice-processing I/O unit (AEC/NS/AGC) on the shared input+output unit.
|
||||
do {
|
||||
try engine.inputNode.setVoiceProcessingEnabled(true)
|
||||
} catch {
|
||||
logger.error("setVoiceProcessingEnabled failed: \(error.localizedDescription) — AEC unavailable")
|
||||
}
|
||||
|
||||
// ── Playback: source node pulls mixed PCM from the ring through the VPIO output. ──
|
||||
let ring = self.ring
|
||||
let scratch = self.renderScratch
|
||||
let scratchFrames = self.renderScratchFrames
|
||||
let src = AVAudioSourceNode(format: outFormat) { _, _, frameCount, ablPtr in
|
||||
let frames = Int(frameCount)
|
||||
let abl = UnsafeMutableAudioBufferListPointer(ablPtr)
|
||||
let n = min(frames, scratchFrames)
|
||||
let got = ring.read(into: scratch, count: n * 2) / 2 // interleaved stereo → frames
|
||||
// Deinterleave int16 → Float32 per channel; silence-fill any underrun tail.
|
||||
let scale: Float = 1.0 / 32768.0
|
||||
for ch in 0..<abl.count {
|
||||
guard let base = abl[ch].mData?.assumingMemoryBound(to: Float.self) else { continue }
|
||||
for i in 0..<frames {
|
||||
if i < got {
|
||||
let s = scratch[i * 2 + min(ch, 1)]
|
||||
base[i] = Float(s) * scale
|
||||
} else {
|
||||
base[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
return noErr
|
||||
}
|
||||
sourceNode = src
|
||||
engine.attach(src)
|
||||
engine.connect(src, to: engine.mainMixerNode, format: outFormat)
|
||||
|
||||
// ── Mic: tap the VPIO input node, convert to 48 kHz int16, feed the core. ──
|
||||
let inFormat = engine.inputNode.outputFormat(forBus: 0)
|
||||
let targetCh = max(1, min(2, captureChannels))
|
||||
let target = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000,
|
||||
channels: AVAudioChannelCount(targetCh), interleaved: true)
|
||||
micTargetFormat = target
|
||||
micConverter = (target != nil && inFormat.sampleRate > 0)
|
||||
? AVAudioConverter(from: inFormat, to: target!) : nil
|
||||
if micConverter == nil {
|
||||
logger.error("mic converter unavailable (in=\(inFormat)) — mic will not transmit")
|
||||
}
|
||||
|
||||
let c = client
|
||||
let sid = micStreamId
|
||||
let converter = micConverter
|
||||
let tgt = micTargetFormat
|
||||
engine.inputNode.installTap(onBus: 0, bufferSize: 960, format: inFormat) { buffer, _ in
|
||||
guard let converter, let tgt else { return }
|
||||
// Convert this tap buffer to 48 kHz int16. Output capacity scaled for any upsample.
|
||||
let ratio = tgt.sampleRate / buffer.format.sampleRate
|
||||
let outCap = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 16)
|
||||
guard let outBuf = AVAudioPCMBuffer(pcmFormat: tgt, frameCapacity: outCap) else { return }
|
||||
var fed = false
|
||||
let status = converter.convert(to: outBuf, error: nil) { _, outStatus in
|
||||
if fed { outStatus.pointee = .noDataNow; return nil }
|
||||
fed = true
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status != .error, outBuf.frameLength > 0,
|
||||
let chData = outBuf.int16ChannelData else { return }
|
||||
let spc = Int(outBuf.frameLength)
|
||||
// int16 interleaved → channelData[0] is the interleaved buffer for interleaved formats.
|
||||
c.feedPcm(streamId: sid, pcm: chData[0], samplesPerChannel: spc, channels: targetCh)
|
||||
}
|
||||
|
||||
// ── Wire the core's mixed-output sink into the ring (C function pointer, no captures). ──
|
||||
let ringPtr = Unmanaged.passUnretained(self.ring).toOpaque()
|
||||
client.setMixedOutputSink({ user, pcm, spc, ch, _ in
|
||||
guard let user, let pcm else { return }
|
||||
let ring = Unmanaged<PCMRing>.fromOpaque(user).takeUnretainedValue()
|
||||
ring.write(pcm, count: spc * Int(ch))
|
||||
}, user: ringPtr)
|
||||
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
isRunning = true
|
||||
logger.info("VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh)")
|
||||
} catch {
|
||||
logger.error("VPIO engine start failed: \(error.localizedDescription)")
|
||||
teardown()
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop the VPIO engine. The caller is responsible for restoring the core's hardware playback
|
||||
/// afterwards (`client.setExternalPlayback(false)` + `audioRestart()`).
|
||||
func stop() {
|
||||
guard isRunning else { return }
|
||||
teardown()
|
||||
logger.info("VPIO engine stopped")
|
||||
}
|
||||
|
||||
private func teardown() {
|
||||
client?.setMixedOutputSink(nil, user: nil)
|
||||
engine.inputNode.removeTap(onBus: 0)
|
||||
if engine.isRunning { engine.stop() }
|
||||
try? engine.inputNode.setVoiceProcessingEnabled(false)
|
||||
if let src = sourceNode {
|
||||
engine.detach(src)
|
||||
sourceNode = nil
|
||||
}
|
||||
micConverter = nil
|
||||
micTargetFormat = nil
|
||||
ring.reset()
|
||||
isRunning = false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user