feat(ios): real echo cancellation/NR via native Voice-Processing engine
iOS "voice chat" had echo and no noise suppression: real iOS AEC/NS/AGC come only from Apple's Voice-Processing I/O unit (VPIO), but the core plays/captures via miniaudio's plain RemoteIO units, so .voiceChat mode alone never engaged AEC. Core (ABI PATCH 1->2): - vc_set_mixed_output_sink + vc_set_external_playback. In external mode the AudioEngine opens no hardware playback device; a mixer-timer thread drives on_playback (decode+mix) on a ~20ms cadence and ships the final mix to the sink. start() also skips the hardware capture device when the MIC stream is external_feed (AudioParams.external_capture). - New white-box test test_external_playback (drives the timer with no hw). iOS/Swift: - StreamDescriptor.externalFeed; VoiceCatClient.setMixedOutputSink / setExternalPlayback wrappers. - IOSVoiceProcessingEngine: AVAudioEngine + setVoiceProcessingEnabled; mic tap -> feedPcm, mixed-sink lock-free ring -> AVAudioSourceNode (both share the VPIO unit so AEC has its reference signal). - IOSAudioRouter.currentConfigUsesVoiceProcessing scopes VPIO to the AEC presets; SessionState join/leave + reconcileVoicePath() switch paths; Voice Chat defaults to speaker; Settings surfaces AEC/NS state. Known: pending on-device verification; a few bugs to fix afterward.
This commit is contained in:
@@ -73,6 +73,9 @@ final class SessionState {
|
||||
broadcastPump.onBroadcastStarted = { [weak self] in self?.startScreenShare() }
|
||||
broadcastPump.onBroadcastFinished = { [weak self] in self?.stopScreenShare() }
|
||||
broadcastPump.start()
|
||||
// When IOSAudioRouter changes the audio config, restart the voice path if needed so the
|
||||
// native VPIO engine (AEC/NS/AGC) engages or disengages to match the new preset/route.
|
||||
AudioSessionManager.shared.reconcileVoicePath = { [weak self] in self?.reconcileVoicePath() }
|
||||
}
|
||||
|
||||
deinit {
|
||||
@@ -222,7 +225,21 @@ final class SessionState {
|
||||
addActivity("AVAudioSession activate failed: \(error)")
|
||||
return
|
||||
}
|
||||
let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic")
|
||||
|
||||
// VPIO path: on the AEC presets, the native AVAudioEngine does AEC/NS/AGC and the core
|
||||
// runs in external mode (no hardware mic/playback). The mic stream is started with
|
||||
// externalFeed so the core skips the hardware capture device; setExternalPlayback makes
|
||||
// it skip the hardware playback device and deliver the mix to IOSVoiceProcessingEngine.
|
||||
let useVPIO = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing
|
||||
if useVPIO {
|
||||
client.setExternalPlayback(true)
|
||||
client.audioRestart() // flip any already-running (pre-join) engine into external mode
|
||||
} else {
|
||||
client.setExternalPlayback(false)
|
||||
}
|
||||
|
||||
let desc = StreamDescriptor(kind: .mic, deviceId: voiceState.currentDeviceId, label: "Mic",
|
||||
externalFeed: useVPIO)
|
||||
let (result, streamId) = client.startStream(desc)
|
||||
if result == .ok {
|
||||
voiceState.micActive = true
|
||||
@@ -241,17 +258,37 @@ final class SessionState {
|
||||
if channels != 1 {
|
||||
client.setCaptureChannels(streamId: streamId, channels: channels)
|
||||
}
|
||||
if useVPIO {
|
||||
IOSVoiceProcessingEngine.shared.start(
|
||||
client: client, micStreamId: streamId, captureChannels: channels)
|
||||
}
|
||||
} else {
|
||||
addActivity("Failed to start mic: \(result.description)")
|
||||
if useVPIO { // revert external-playback mode so remote audio still plays
|
||||
client.setExternalPlayback(false)
|
||||
client.audioRestart()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stopMicStream() {
|
||||
// Tear down the VPIO engine first (removes the mic tap + unregisters the mixed sink),
|
||||
// then stop the mic stream, then restore the core's hardware playback for any remaining
|
||||
// remote audio. Order matters: the mic stream must be gone before audioRestart so the
|
||||
// core opens a normal playback device (and no capture device — there's no mic stream).
|
||||
let wasVPIO = IOSVoiceProcessingEngine.shared.isRunning
|
||||
if wasVPIO {
|
||||
IOSVoiceProcessingEngine.shared.stop()
|
||||
}
|
||||
if voiceState.localStreamId != 0 {
|
||||
client.stopStream(voiceState.localStreamId)
|
||||
voiceState.localStreamId = 0
|
||||
AudioSessionManager.shared.activeMicStreamId = nil
|
||||
}
|
||||
if wasVPIO {
|
||||
client.setExternalPlayback(false)
|
||||
client.audioRestart() // reopen hardware playback (no mic stream → no hw capture)
|
||||
}
|
||||
voiceState.micActive = false
|
||||
voiceState.level = 0
|
||||
// Do NOT deactivate the AVAudioSession here — the user may still want to hear
|
||||
@@ -259,6 +296,19 @@ final class SessionState {
|
||||
// disconnecting from the server (see AppState.disconnect / .disconnected event).
|
||||
}
|
||||
|
||||
/// Restart the voice path when the audio config changes mid-call (driven by IOSAudioRouter).
|
||||
/// If VPIO is involved on either the current or desired side, restart the mic so the native
|
||||
/// voice-processing engine engages/disengages and re-binds to the new route. Pure miniaudio
|
||||
/// config tweaks need no restart — the core's own audioRestart (already issued) handles them.
|
||||
private func reconcileVoicePath() {
|
||||
guard voiceState.micActive else { return }
|
||||
let want = IOSAudioRouter.shared.currentConfigUsesVoiceProcessing
|
||||
let have = IOSVoiceProcessingEngine.shared.isRunning
|
||||
guard want || have else { return }
|
||||
stopMicStream()
|
||||
doStartMicStream()
|
||||
}
|
||||
|
||||
// MARK: - Screen audio share
|
||||
|
||||
/// Called when the broadcast extension becomes active. Announces the SCREEN_AUDIO stream;
|
||||
|
||||
Reference in New Issue
Block a user