feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305 AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX, an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain wired through ConnSession/SessionRegistry into a new server-side SFU (MediaRelay) that decrypts and re-encrypts frames per channel member. Exit criterion verified: test_m2_voice — two headless clients relay 50 encrypted Opus frames through the server; ctest --preset m1-dev is 9/9 green. Also corrects protocol.md's UdpBinding diagram, which described the UDP-side binding packet as AEAD-sealed when it is in fact a plaintext bootstrap frame (separate from the TCP/TLS UdpBinding ack). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,8 +1,261 @@
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
#define MINIAUDIO_IMPLEMENTATION
|
||||
#include <miniaudio.h>
|
||||
#endif
|
||||
|
||||
#include "audio/audio_engine.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
|
||||
namespace voicecat::audio {
|
||||
|
||||
// M0 stub. Capture/playback (miniaudio), APM DSP, jitter buffer, and mixer land in M2/M3.
|
||||
// See docs/voice.md §8–11.
|
||||
// ── JitterBuffer ─────────────────────────────────────────────────────────────
|
||||
|
||||
void JitterBuffer::push(Frame f) {
|
||||
std::lock_guard lk(mu_);
|
||||
|
||||
uint32_t ts = f.timestamp;
|
||||
|
||||
// Jitter estimation (EWMA of inter-arrival gap vs expected gap).
|
||||
if (!first_push_) {
|
||||
uint32_t arrived_gap = ts - last_push_ts_;
|
||||
uint32_t expected_gap = 960; // 20 ms @48k; TODO: derive from params
|
||||
uint32_t diff = (arrived_gap > expected_gap) ? (arrived_gap - expected_gap)
|
||||
: (expected_gap - arrived_gap);
|
||||
jitter_est_ = (jitter_est_ * 7 + diff) / 8;
|
||||
uint32_t depth = std::clamp(jitter_est_ * 2 + 960u, 960u, 48000u * 200u / 1000u);
|
||||
target_depth_ms_.store(depth * 1000u / 48000u, std::memory_order_relaxed);
|
||||
}
|
||||
last_push_ts_ = ts;
|
||||
first_push_ = false;
|
||||
|
||||
buf_.emplace(ts, std::move(f));
|
||||
}
|
||||
|
||||
std::optional<JitterBuffer::Frame> JitterBuffer::pop(uint32_t playout_ts) {
|
||||
std::unique_lock lk(mu_, std::try_to_lock);
|
||||
if (!lk) return std::nullopt; // contended — caller does PLC
|
||||
|
||||
if (buf_.empty()) return std::nullopt;
|
||||
|
||||
auto it = buf_.begin();
|
||||
uint32_t ts = it->first;
|
||||
|
||||
// Drop frames that are too old (> 500 ms late).
|
||||
if (static_cast<int32_t>(playout_ts - ts) > static_cast<int32_t>(kLateDropSamples)) {
|
||||
lost_.fetch_add(1, std::memory_order_relaxed);
|
||||
buf_.erase(it);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Return frame only when it's due.
|
||||
if (static_cast<int32_t>(ts - playout_ts) > 0) return std::nullopt;
|
||||
|
||||
Frame f = std::move(it->second);
|
||||
buf_.erase(it);
|
||||
return f;
|
||||
}
|
||||
|
||||
void JitterBuffer::reset() {
|
||||
std::lock_guard lk(mu_);
|
||||
buf_.clear();
|
||||
lost_.store(0);
|
||||
first_push_ = true;
|
||||
}
|
||||
|
||||
// ── AudioEngine ──────────────────────────────────────────────────────────────
|
||||
|
||||
AudioEngine::AudioEngine() {
|
||||
inject_ring_.resize(kInjectCapSamples, 0);
|
||||
}
|
||||
|
||||
AudioEngine::~AudioEngine() { stop(); }
|
||||
|
||||
bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
|
||||
if (running_.load()) return false;
|
||||
params_ = p;
|
||||
capture_cb_ = std::move(capture_cb);
|
||||
frame_samples_ = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
|
||||
running_.store(true, std::memory_order_release);
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// ── Capture device ──────────────────────────────────────────────────────
|
||||
ma_device_config cap_cfg = ma_device_config_init(ma_device_type_capture);
|
||||
cap_cfg.capture.format = ma_format_s16;
|
||||
cap_cfg.capture.channels = p.channels;
|
||||
cap_cfg.sampleRate = p.sample_rate;
|
||||
cap_cfg.dataCallback = capture_data_cb;
|
||||
cap_cfg.pUserData = this;
|
||||
cap_cfg.capture.pDeviceID = nullptr; // always default for now
|
||||
|
||||
if (ma_device_init(nullptr, &cap_cfg, &capture_device_) == MA_SUCCESS) {
|
||||
if (ma_device_start(&capture_device_) == MA_SUCCESS) {
|
||||
capture_started_ = true;
|
||||
} else {
|
||||
ma_device_uninit(&capture_device_);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Playback device ─────────────────────────────────────────────────────
|
||||
ma_device_config pb_cfg = ma_device_config_init(ma_device_type_playback);
|
||||
pb_cfg.playback.format = ma_format_s16;
|
||||
pb_cfg.playback.channels = p.channels;
|
||||
pb_cfg.sampleRate = p.sample_rate;
|
||||
pb_cfg.dataCallback = playback_data_cb;
|
||||
pb_cfg.pUserData = this;
|
||||
pb_cfg.playback.pDeviceID = nullptr;
|
||||
|
||||
if (ma_device_init(nullptr, &pb_cfg, &playback_device_) == MA_SUCCESS) {
|
||||
if (ma_device_start(&playback_device_) == MA_SUCCESS) {
|
||||
playback_started_ = true;
|
||||
} else {
|
||||
ma_device_uninit(&playback_device_);
|
||||
}
|
||||
}
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void AudioEngine::stop() {
|
||||
if (!running_.exchange(false)) return;
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
if (capture_started_) {
|
||||
ma_device_stop(&capture_device_);
|
||||
ma_device_uninit(&capture_device_);
|
||||
capture_started_ = false;
|
||||
}
|
||||
if (playback_started_) {
|
||||
ma_device_stop(&playback_device_);
|
||||
ma_device_uninit(&playback_device_);
|
||||
playback_started_ = false;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void AudioEngine::inject_capture(const int16_t* pcm, size_t n) {
|
||||
std::lock_guard lk(inject_mu_);
|
||||
size_t w = inject_write_.load(std::memory_order_relaxed);
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
inject_ring_[(w + i) % kInjectCapSamples] = pcm[i];
|
||||
inject_write_.store(w + n, std::memory_order_release);
|
||||
|
||||
// Fire capture_cb_ for each complete frame now available.
|
||||
while (true) {
|
||||
size_t r = inject_read_.load(std::memory_order_relaxed);
|
||||
size_t avail = inject_write_.load(std::memory_order_acquire) - r;
|
||||
if (avail < static_cast<size_t>(frame_samples_)) break;
|
||||
|
||||
std::vector<int16_t> frame(frame_samples_);
|
||||
for (int i = 0; i < frame_samples_; ++i)
|
||||
frame[i] = inject_ring_[(r + i) % kInjectCapSamples];
|
||||
inject_read_.store(r + frame_samples_, std::memory_order_release);
|
||||
|
||||
if (capture_cb_) capture_cb_(frame.data(), frame_samples_);
|
||||
}
|
||||
}
|
||||
|
||||
void AudioEngine::push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f) {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
streams_[ssrc].jitter.push(std::move(f));
|
||||
}
|
||||
|
||||
void AudioEngine::set_stream_gain(uint32_t ssrc, float gain) {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
streams_[ssrc].gain = gain;
|
||||
}
|
||||
|
||||
void AudioEngine::set_stream_mute(uint32_t ssrc, bool mute) {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
streams_[ssrc].mute = mute;
|
||||
}
|
||||
|
||||
void AudioEngine::remove_stream(uint32_t ssrc) {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
streams_.erase(ssrc);
|
||||
}
|
||||
|
||||
uint32_t AudioEngine::stream_packets_lost(uint32_t ssrc) const {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
auto it = streams_.find(ssrc);
|
||||
return (it != streams_.end()) ? it->second.jitter.packets_lost() : 0;
|
||||
}
|
||||
|
||||
uint32_t AudioEngine::stream_target_depth_ms(uint32_t ssrc) const {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
auto it = streams_.find(ssrc);
|
||||
return (it != streams_.end()) ? it->second.jitter.target_depth_ms() : 40;
|
||||
}
|
||||
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
|
||||
std::lock_guard lk(streams_mu_);
|
||||
streams_[ssrc].decoder.init(p);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
|
||||
void AudioEngine::capture_data_cb(ma_device* dev, void* /*out*/,
|
||||
const void* in, ma_uint32 frame_count) {
|
||||
auto* self = static_cast<AudioEngine*>(dev->pUserData);
|
||||
self->on_capture(static_cast<const int16_t*>(in), frame_count);
|
||||
}
|
||||
|
||||
void AudioEngine::on_capture(const int16_t* pcm, ma_uint32 frames) {
|
||||
if (capture_cb_) capture_cb_(pcm, static_cast<int>(frames));
|
||||
}
|
||||
|
||||
void AudioEngine::playback_data_cb(ma_device* dev, void* out,
|
||||
const void* /*in*/, ma_uint32 frame_count) {
|
||||
auto* self = static_cast<AudioEngine*>(dev->pUserData);
|
||||
self->on_playback(static_cast<int16_t*>(out), frame_count);
|
||||
}
|
||||
|
||||
void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
|
||||
std::memset(out, 0, frames * params_.channels * sizeof(int16_t));
|
||||
|
||||
std::unique_lock lk(streams_mu_, std::try_to_lock);
|
||||
if (!lk) return; // contended: emit silence this period
|
||||
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
std::vector<int32_t> mix(frames * params_.channels, 0);
|
||||
|
||||
for (auto& [ssrc, stream] : streams_) {
|
||||
if (stream.mute || !stream.decoder.valid()) continue;
|
||||
|
||||
auto maybe_frame = stream.jitter.pop(stream.playout_ts);
|
||||
std::vector<int16_t> pcm(frames * params_.channels);
|
||||
int n;
|
||||
|
||||
if (maybe_frame) {
|
||||
n = stream.decoder.decode(
|
||||
maybe_frame->payload.data(),
|
||||
static_cast<int>(maybe_frame->payload.size()),
|
||||
pcm.data(), static_cast<int>(pcm.size()));
|
||||
} else {
|
||||
n = stream.decoder.decode(nullptr, 0, pcm.data(),
|
||||
static_cast<int>(pcm.size()));
|
||||
}
|
||||
|
||||
if (n > 0) {
|
||||
float g = stream.gain;
|
||||
for (int i = 0; i < n * static_cast<int>(params_.channels); ++i)
|
||||
mix[i] += static_cast<int32_t>(static_cast<float>(pcm[i]) * g);
|
||||
}
|
||||
stream.playout_ts += frames;
|
||||
}
|
||||
|
||||
for (ma_uint32 i = 0; i < frames * params_.channels; ++i)
|
||||
out[i] = static_cast<int16_t>(std::clamp(mix[i], -32768, 32767));
|
||||
#else
|
||||
(void)out;
|
||||
(void)frames;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
|
||||
} // namespace voicecat::audio
|
||||
|
||||
Reference in New Issue
Block a user