feat(ios): real echo cancellation/NR via native Voice-Processing engine
iOS "voice chat" had echo and no noise suppression: real iOS AEC/NS/AGC come only from Apple's Voice-Processing I/O unit (VPIO), but the core plays/captures via miniaudio's plain RemoteIO units, so .voiceChat mode alone never engaged AEC. Core (ABI PATCH 1->2): - vc_set_mixed_output_sink + vc_set_external_playback. In external mode the AudioEngine opens no hardware playback device; a mixer-timer thread drives on_playback (decode+mix) on a ~20ms cadence and ships the final mix to the sink. start() also skips the hardware capture device when the MIC stream is external_feed (AudioParams.external_capture). - New white-box test test_external_playback (drives the timer with no hw). iOS/Swift: - StreamDescriptor.externalFeed; VoiceCatClient.setMixedOutputSink / setExternalPlayback wrappers. - IOSVoiceProcessingEngine: AVAudioEngine + setVoiceProcessingEnabled; mic tap -> feedPcm, mixed-sink lock-free ring -> AVAudioSourceNode (both share the VPIO unit so AEC has its reference signal). - IOSAudioRouter.currentConfigUsesVoiceProcessing scopes VPIO to the AEC presets; SessionState join/leave + reconcileVoicePath() switch paths; Voice Chat defaults to speaker; Settings surfaces AEC/NS state. Known: pending on-device verification; a few bugs to fix afterward.
This commit is contained in:
@@ -69,6 +69,14 @@ if(VOICECAT_USE_VCPKG_DEPS)
|
||||
target_include_directories(test_plc_cap PRIVATE ${VC_TEST_INTERNAL_INCLUDES})
|
||||
add_test(NAME plc_cap COMMAND test_plc_cap)
|
||||
|
||||
# External playback (iOS VPIO): the mixer-timer thread drives decode+mix with NO hardware
|
||||
# device and delivers the final mix to the mixed-output sink. White-box AudioEngine test.
|
||||
add_executable(test_external_playback test_external_playback.cpp)
|
||||
target_link_libraries(test_external_playback PRIVATE voicecat::voicecat)
|
||||
target_compile_features(test_external_playback PRIVATE cxx_std_20)
|
||||
target_include_directories(test_external_playback PRIVATE ${VC_TEST_INTERNAL_INCLUDES})
|
||||
add_test(NAME external_playback COMMAND test_external_playback)
|
||||
|
||||
# M2 exit criterion: two headless clients relay encrypted Opus frames via the SFU.
|
||||
add_executable(test_m2_voice test_m2_voice.cpp)
|
||||
target_link_libraries(test_m2_voice PRIVATE voicecat::server)
|
||||
|
||||
146
tests/test_external_playback.cpp
Normal file
146
tests/test_external_playback.cpp
Normal file
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* test_external_playback — verifies AudioEngine's external-playback mode (iOS VPIO path).
|
||||
*
|
||||
* When external playback is enabled, the engine opens NO hardware playback device; a mixer-timer
|
||||
* thread drives decode+mix on a ~20ms cadence and delivers the FINAL mixed PCM to the
|
||||
* mixed-output sink (the Swift AVAudioEngine VPIO renderer consumes this). This test asserts:
|
||||
* 1. The mixed sink fires steadily on the timer thread (count grows over time) with the right
|
||||
* format (48kHz, stereo), and carries real energy while a stream is being decoded.
|
||||
* 2. The per-stream pcm_sink still fires concurrently (both taps coexist).
|
||||
* 3. With no remote streams, the mixed sink KEEPS firing (silent-but-present blocks) so the
|
||||
* renderer has a continuous clock.
|
||||
*
|
||||
* White-box: constructs AudioEngine directly (no server, no audio hardware needed) — the timer
|
||||
* thread drives the mixer with no ma_device, which is the core new behavior under test.
|
||||
*/
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS)
|
||||
|
||||
#include "audio/audio_engine.h"
|
||||
#include "codec/opus_codec.h"
|
||||
|
||||
static int g_failures = 0;
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::printf("FAIL [%s:%d]: %s\n", __FILE__, __LINE__, #cond); \
|
||||
++g_failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
// Shared state written by the sink callbacks (timer thread) and read by main.
|
||||
struct MixedSinkState {
|
||||
std::atomic<int> calls{0};
|
||||
std::atomic<int64_t> max_energy{0};
|
||||
std::atomic<uint32_t> last_channels{0};
|
||||
std::atomic<uint32_t> last_sample_rate{0};
|
||||
};
|
||||
static MixedSinkState g_mixed;
|
||||
static std::atomic<int> g_pcm_sink_calls{0};
|
||||
|
||||
static void mixed_cb(void* user, const int16_t* pcm, size_t spc, uint32_t ch, uint32_t sr) {
|
||||
auto* s = static_cast<MixedSinkState*>(user);
|
||||
s->calls.fetch_add(1, std::memory_order_relaxed);
|
||||
s->last_channels.store(ch, std::memory_order_relaxed);
|
||||
s->last_sample_rate.store(sr, std::memory_order_relaxed);
|
||||
int64_t e = 0;
|
||||
for (size_t i = 0; i < spc * ch; ++i) e += std::abs(static_cast<int>(pcm[i]));
|
||||
int64_t prev = s->max_energy.load(std::memory_order_relaxed);
|
||||
while (e > prev && !s->max_energy.compare_exchange_weak(prev, e, std::memory_order_relaxed)) {
|
||||
}
|
||||
}
|
||||
|
||||
static void pcm_cb(void*, uint32_t, uint32_t, const int16_t*, size_t, uint32_t, uint32_t) {
|
||||
g_pcm_sink_calls.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
int main() {
|
||||
voicecat::audio::AudioEngine engine;
|
||||
engine.set_external_playback(true);
|
||||
engine.set_mixed_output_sink(&mixed_cb, &g_mixed);
|
||||
engine.set_pcm_sink(&pcm_cb, nullptr);
|
||||
|
||||
voicecat::audio::AudioParams p;
|
||||
p.sample_rate = 48000;
|
||||
p.capture_channels = 1;
|
||||
p.playback_channels = 2;
|
||||
p.frame_ms = 20;
|
||||
CHECK(engine.start(p)); // no hardware device opened — the timer thread drives the mixer
|
||||
|
||||
voicecat::codec::OpusParams op;
|
||||
op.sample_rate = 48000;
|
||||
op.frame_ms = 20;
|
||||
op.stereo = false;
|
||||
int frame_samples = voicecat::codec::opus_frame_samples(op); // 960
|
||||
|
||||
voicecat::codec::OpusEncoder enc;
|
||||
CHECK(enc.init(op));
|
||||
std::vector<int16_t> sine(static_cast<size_t>(frame_samples));
|
||||
for (int i = 0; i < frame_samples; ++i) {
|
||||
float t = static_cast<float>(i) / 48000.0f;
|
||||
sine[i] = static_cast<int16_t>(std::sin(2.0f * 3.14159265f * 440.0f * t) * 20000.0f);
|
||||
}
|
||||
uint8_t opus_buf[1500];
|
||||
int opus_len = enc.encode(sine.data(), frame_samples, opus_buf, sizeof(opus_buf));
|
||||
CHECK(opus_len > 0);
|
||||
|
||||
const uint32_t ssrc = 1;
|
||||
engine.init_recv_stream(ssrc, op, /*user_id=*/7, /*stream_id=*/3);
|
||||
|
||||
// ── Phase 1: feed ~600ms of real frames; the timer must decode + mix them. ──────────
|
||||
uint32_t ts = 0;
|
||||
for (int i = 0; i < 30; ++i) { // 30 * 20ms = 600ms of audio
|
||||
voicecat::audio::JitterBuffer::Frame f;
|
||||
f.seq = static_cast<uint64_t>(i);
|
||||
f.timestamp = ts;
|
||||
f.fec_present = false;
|
||||
f.payload.assign(opus_buf, opus_buf + opus_len);
|
||||
engine.push_recv_frame(ssrc, std::move(f));
|
||||
ts += static_cast<uint32_t>(frame_samples);
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
||||
|
||||
int active_calls = g_mixed.calls.load(std::memory_order_relaxed);
|
||||
std::printf("external_playback: phase1 mixed-sink calls=%d max_energy=%lld pcm_sink=%d\n",
|
||||
active_calls, static_cast<long long>(g_mixed.max_energy.load()),
|
||||
g_pcm_sink_calls.load());
|
||||
// ~25 blocks expected at a 20ms cadence over 500ms; allow generous slack for scheduler/debug.
|
||||
CHECK(active_calls >= 10);
|
||||
CHECK(g_mixed.max_energy.load(std::memory_order_relaxed) > 0); // real decoded audio in the mix
|
||||
CHECK(g_mixed.last_channels.load(std::memory_order_relaxed) == 2);
|
||||
CHECK(g_mixed.last_sample_rate.load(std::memory_order_relaxed) == 48000);
|
||||
CHECK(g_pcm_sink_calls.load(std::memory_order_relaxed) > 0); // per-stream tap coexists
|
||||
|
||||
// ── Phase 2: remove the stream; the mixed sink must KEEP firing (silent blocks). ─────
|
||||
engine.remove_stream(ssrc);
|
||||
int before = g_mixed.calls.load(std::memory_order_relaxed);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
int after = g_mixed.calls.load(std::memory_order_relaxed);
|
||||
std::printf("external_playback: phase2 silent blocks delivered=%d\n", after - before);
|
||||
CHECK(after - before >= 5); // continuous clock even with nothing to play
|
||||
|
||||
engine.stop(); // joins the mixer-timer thread
|
||||
enc.destroy();
|
||||
|
||||
if (g_failures == 0) {
|
||||
std::printf("external_playback: all checks passed\n");
|
||||
return 0;
|
||||
}
|
||||
std::printf("external_playback: %d failure(s)\n", g_failures);
|
||||
return 1;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int main() {
|
||||
std::printf("external_playback: SKIP (VOICECAT_HAS_AUDIO or VOICECAT_HAS_OPUS not defined)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user