import AVFoundation import Darwin import os import VoiceCatCore private let logger = Logger(subsystem: "cat.voice.VoiceCatiOS", category: "IOSVoiceProcessingEngine") /// In-process single-producer/single-consumer int16 PCM ring for the VPIO playback path. /// /// producer = the core's mixer-timer thread (the `vc_set_mixed_output_sink` callback) /// consumer = the `AVAudioSourceNode` render thread /// /// Heap-backed (not shared memory like `BroadcastAudioRing`), but the same discipline: aligned /// 64-bit monotonic indices with `OSMemoryBarrier` for acquire/release ordering. Both the C /// callback and the render block are real-time — they only do index math + a memcpy here, never /// lock or allocate. final class PCMRing { private let data: UnsafeMutablePointer private let capacity: Int private var writeIdx: UInt64 = 0 private var readIdx: UInt64 = 0 init(capacitySamples: Int) { capacity = capacitySamples data = UnsafeMutablePointer.allocate(capacity: capacitySamples) data.initialize(repeating: 0, count: capacitySamples) } deinit { data.deallocate() } /// Producer: append `count` interleaved int16 samples. Drops the chunk if it doesn't fit /// (better to skip than tear). Single producer only (the core mixer-timer thread). func write(_ src: UnsafePointer, count: Int) { guard count > 0, count <= capacity else { return } let w = writeIdx OSMemoryBarrier() let r = readIdx if capacity - Int(w &- r) < count { return } // full: drop var idx = Int(w % UInt64(capacity)) var off = 0 var rem = count while rem > 0 { let chunk = min(rem, capacity - idx) (data + idx).update(from: src + off, count: chunk) idx = (idx + chunk) % capacity off += chunk rem -= chunk } OSMemoryBarrier() writeIdx = w &+ UInt64(count) } /// Consumer: read up to `count` interleaved int16 samples into `dst`; returns the number /// read (the rest is the caller's to silence-fill). Single consumer only (render thread). func read(into dst: UnsafeMutablePointer, count: Int) -> Int { let r = readIdx OSMemoryBarrier() let w = writeIdx let available = Int(w &- r) if available <= 0 { return 0 } let n = min(available, count) var idx = Int(r % UInt64(capacity)) var off = 0 var rem = n while rem > 0 { let chunk = min(rem, capacity - idx) (dst + off).update(from: data + idx, count: chunk) idx = (idx + chunk) % capacity off += chunk rem -= chunk } OSMemoryBarrier() readIdx = r &+ UInt64(n) return n } /// Discard everything buffered — call before (re)starting so stale pre-roll isn't played. func reset() { OSMemoryBarrier(); readIdx = writeIdx } /// Diagnostics: monotonic total samples written / read since the ring was created. The /// indices are already cumulative, so these are free. Only read them when both threads are /// quiesced (e.g. at teardown after the engine + mixer sink are stopped) — they are not /// synchronized for live cross-thread reads. Lets us tell "core never delivered PCM" (Bug 1 /// core path) apart from "PCM arrived but produced no sound" (AVAudioEngine output graph). var debugTotalWritten: UInt64 { writeIdx } var debugTotalRead: UInt64 { readIdx } } /// Native iOS voice-processing audio path (docs/voice.md §8 "iOS voice processing"). /// /// Real iOS echo cancellation / noise suppression / AGC come ONLY from Apple's Voice-Processing /// I/O unit (VPIO), which `AVAudioEngine.setVoiceProcessingEnabled(true)` enables. For VPIO to /// cancel echo it must own BOTH the mic capture and the remote-audio playback (it subtracts the /// played-back signal from the mic), so on the AEC presets this engine drives both directions and /// the core runs in external mode (no hardware devices): /// - **Mic → core:** a tap on the VPIO input node → 48 kHz int16 → `client.feedPcm(micStreamId)`. /// - **core → speaker:** the core's mixed-output sink fills `ring`; an `AVAudioSourceNode` pulls /// from it and renders through the VPIO output, giving AEC its reference signal. /// /// Lifecycle is driven by `SessionState` join/leave. The Stereo Mic / Studio / A2DP presets keep /// the core's miniaudio path instead (they want raw / stereo / no-AEC routing VPIO can't provide). @MainActor final class IOSVoiceProcessingEngine { static let shared = IOSVoiceProcessingEngine() private(set) var isRunning = false private let engine = AVAudioEngine() private var sourceNode: AVAudioSourceNode? private weak var client: VoiceCatClient? private var micStreamId: UInt32 = 0 // 48 kHz stereo Float32 (deinterleaved) — the format the source node renders and the engine // processes in. The core delivers 48 kHz stereo int16 via the mixed-output sink. private let outFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false)! // Playback ring (mixed remote audio): ~0.5 s of 48 kHz stereo int16. Filled by the core's // mixer-timer thread, drained by the source-node render thread. private let ring = PCMRing(capacitySamples: 48000 * 2 / 2) // Render-thread scratch for deinterleaving — pre-allocated so the render block never allocates. private let renderScratchFrames = 8192 private let renderScratch: UnsafeMutablePointer // Mic-feed converter (input-node format → 48 kHz int16) and its target buffer. Owned here so // the (background) tap block reuses them instead of allocating per callback. private var micConverter: AVAudioConverter? private var micTargetFormat: AVAudioFormat? private init() { renderScratch = UnsafeMutablePointer.allocate(capacity: renderScratchFrames * 2) renderScratch.initialize(repeating: 0, count: renderScratchFrames * 2) } /// Start the VPIO engine for an active mic stream. The caller must have already enabled /// external playback on the core (`client.setExternalPlayback(true)` + `audioRestart()`) and /// started the MIC stream with `externalFeed: true`. func start(client: VoiceCatClient, micStreamId: UInt32, captureChannels: UInt32) { guard !isRunning else { return } self.client = client self.micStreamId = micStreamId ring.reset() // Enable the voice-processing I/O unit (AEC/NS/AGC) on the shared input+output unit. do { try engine.inputNode.setVoiceProcessingEnabled(true) } catch { logger.error("setVoiceProcessingEnabled failed: \(error.localizedDescription) — AEC unavailable") } // ── Playback: source node pulls mixed PCM from the ring through the VPIO output. ── let ring = self.ring let scratch = self.renderScratch let scratchFrames = self.renderScratchFrames let src = AVAudioSourceNode(format: outFormat) { _, _, frameCount, ablPtr in let frames = Int(frameCount) let abl = UnsafeMutableAudioBufferListPointer(ablPtr) let n = min(frames, scratchFrames) let got = ring.read(into: scratch, count: n * 2) / 2 // interleaved stereo → frames // Deinterleave int16 → Float32 per channel; silence-fill any underrun tail. let scale: Float = 1.0 / 32768.0 for ch in 0.. 0) ? AVAudioConverter(from: inFormat, to: target!) : nil if micConverter == nil { logger.error("mic converter unavailable (in=\(inFormat)) — mic will not transmit") } let c = client let sid = micStreamId let converter = micConverter let tgt = micTargetFormat engine.inputNode.installTap(onBus: 0, bufferSize: 960, format: inFormat) { buffer, _ in guard let converter, let tgt else { return } // Convert this tap buffer to 48 kHz int16. Output capacity scaled for any upsample. let ratio = tgt.sampleRate / buffer.format.sampleRate let outCap = AVAudioFrameCount(Double(buffer.frameLength) * ratio + 16) guard let outBuf = AVAudioPCMBuffer(pcmFormat: tgt, frameCapacity: outCap) else { return } var fed = false let status = converter.convert(to: outBuf, error: nil) { _, outStatus in if fed { outStatus.pointee = .noDataNow; return nil } fed = true outStatus.pointee = .haveData return buffer } guard status != .error, outBuf.frameLength > 0, let chData = outBuf.int16ChannelData else { return } let spc = Int(outBuf.frameLength) // int16 interleaved → channelData[0] is the interleaved buffer for interleaved formats. c.feedPcm(streamId: sid, pcm: chData[0], samplesPerChannel: spc, channels: targetCh) } // ── Wire the core's mixed-output sink into the ring (C function pointer, no captures). ── let ringPtr = Unmanaged.passUnretained(self.ring).toOpaque() client.setMixedOutputSink({ user, pcm, spc, ch, _ in guard let user, let pcm else { return } let ring = Unmanaged.fromOpaque(user).takeUnretainedValue() ring.write(pcm, count: spc * Int(ch)) }, user: ringPtr) engine.prepare() do { try engine.start() isRunning = true // Diagnostics: capture the negotiated graph formats and the live output route so a // silent-playback report can be triaged (format/rate mismatch vs. routing vs. the // core not delivering PCM — see the ring stats logged in teardown()). let outFmt = engine.outputNode.outputFormat(forBus: 0) let mixFmt = engine.mainMixerNode.outputFormat(forBus: 0) let route = AVAudioSession.sharedInstance().currentRoute.outputs .map { "\($0.portName)[\($0.portType.rawValue)]" }.joined(separator: ", ") logger.info(""" VPIO engine started — inFormat=\(inFormat), captureCh=\(targetCh), \ outputNode=\(outFmt), mainMixer=\(mixFmt), outputRoute=[\(route)] """) } catch { logger.error("VPIO engine start failed: \(error.localizedDescription)") teardown() } } /// Stop the VPIO engine. The caller is responsible for restoring the core's hardware playback /// afterwards (`client.setExternalPlayback(false)` + `audioRestart()`). func stop() { guard isRunning else { return } teardown() logger.info("VPIO engine stopped") } private func teardown() { client?.setMixedOutputSink(nil, user: nil) engine.inputNode.removeTap(onBus: 0) if engine.isRunning { engine.stop() } // Diagnostics (threads now quiesced): how much mixed PCM the core delivered into the ring // vs. how much the render thread consumed. written==0 ⇒ the core never delivered (Bug 1 // core/lifecycle path); written>0 with no audible output ⇒ the AVAudioEngine output graph. logger.info("VPIO ring stats — written=\(self.ring.debugTotalWritten) read=\(self.ring.debugTotalRead) samples") try? engine.inputNode.setVoiceProcessingEnabled(false) if let src = sourceNode { engine.detach(src) sourceNode = nil } micConverter = nil micTargetFormat = nil ring.reset() isRunning = false } }