feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer

Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.

Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 01:31:14 +02:00
parent 63f457fc54
commit 694494a5be
29 changed files with 2548 additions and 86 deletions

View File

@@ -0,0 +1,21 @@
#include "audio/apm_processor.h"
namespace voicecat::audio {
// ── ApmPassthrough ────────────────────────────────────────────────────────────
// No-op: returns true (VAD always open), does not modify PCM.
// Replaced by WebrtcApmProcessor when VOICECAT_HAS_APM is defined.
class ApmPassthrough final : public ApmProcessor {
public:
void process_render(const int16_t*, int, int) override {}
bool process_capture(int16_t*, int, int) override { return true; }
};
std::unique_ptr<ApmProcessor> ApmProcessor::create() {
#ifdef VOICECAT_HAS_APM
// TODO(M3): return std::make_unique<WebrtcApmProcessor>();
#endif
return std::make_unique<ApmPassthrough>();
}
} // namespace voicecat::audio

View File

@@ -0,0 +1,33 @@
/*
* audio/apm_processor.h — Send-side audio processing module (AEC/NS/AGC/VAD).
*
* Design: docs/voice.md §11. Uses webrtc-audio-processing when VOICECAT_HAS_APM is defined;
* falls back to a no-op passthrough (VAD always open, PCM unmodified) otherwise.
*/
#ifndef VOICECAT_AUDIO_APM_PROCESSOR_H
#define VOICECAT_AUDIO_APM_PROCESSOR_H
#include <cstdint>
#include <memory>
namespace voicecat::audio {
class ApmProcessor {
public:
virtual ~ApmProcessor() = default;
// Feed the most recent playback reference (for AEC). Call before process_capture().
virtual void process_render(const int16_t* pcm, int samples, int sample_rate) = 0;
// Process one capture frame in-place (AEC, NS, AGC).
// Returns true if VAD detects speech (or always true in passthrough mode).
// Returns false → caller should skip encode/send (silence gate).
virtual bool process_capture(int16_t* pcm, int samples, int sample_rate) = 0;
// Factory: returns a real APM if VOICECAT_HAS_APM is defined, else a passthrough.
static std::unique_ptr<ApmProcessor> create();
};
} // namespace voicecat::audio
#endif // VOICECAT_AUDIO_APM_PROCESSOR_H

View File

@@ -1,8 +1,261 @@
#ifdef VOICECAT_HAS_AUDIO
#define MINIAUDIO_IMPLEMENTATION
#include <miniaudio.h>
#endif
#include "audio/audio_engine.h"
#include <algorithm>
#include <cstring>
namespace voicecat::audio {
// M0 stub. Capture/playback (miniaudio), APM DSP, jitter buffer, and mixer land in M2/M3.
// See docs/voice.md §811.
// ── JitterBuffer ─────────────────────────────────────────────────────────────
void JitterBuffer::push(Frame f) {
std::lock_guard lk(mu_);
uint32_t ts = f.timestamp;
// Jitter estimation (EWMA of inter-arrival gap vs expected gap).
if (!first_push_) {
uint32_t arrived_gap = ts - last_push_ts_;
uint32_t expected_gap = 960; // 20 ms @48k; TODO: derive from params
uint32_t diff = (arrived_gap > expected_gap) ? (arrived_gap - expected_gap)
: (expected_gap - arrived_gap);
jitter_est_ = (jitter_est_ * 7 + diff) / 8;
uint32_t depth = std::clamp(jitter_est_ * 2 + 960u, 960u, 48000u * 200u / 1000u);
target_depth_ms_.store(depth * 1000u / 48000u, std::memory_order_relaxed);
}
last_push_ts_ = ts;
first_push_ = false;
buf_.emplace(ts, std::move(f));
}
std::optional<JitterBuffer::Frame> JitterBuffer::pop(uint32_t playout_ts) {
std::unique_lock lk(mu_, std::try_to_lock);
if (!lk) return std::nullopt; // contended — caller does PLC
if (buf_.empty()) return std::nullopt;
auto it = buf_.begin();
uint32_t ts = it->first;
// Drop frames that are too old (> 500 ms late).
if (static_cast<int32_t>(playout_ts - ts) > static_cast<int32_t>(kLateDropSamples)) {
lost_.fetch_add(1, std::memory_order_relaxed);
buf_.erase(it);
return std::nullopt;
}
// Return frame only when it's due.
if (static_cast<int32_t>(ts - playout_ts) > 0) return std::nullopt;
Frame f = std::move(it->second);
buf_.erase(it);
return f;
}
void JitterBuffer::reset() {
std::lock_guard lk(mu_);
buf_.clear();
lost_.store(0);
first_push_ = true;
}
// ── AudioEngine ──────────────────────────────────────────────────────────────
AudioEngine::AudioEngine() {
inject_ring_.resize(kInjectCapSamples, 0);
}
AudioEngine::~AudioEngine() { stop(); }
bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
if (running_.load()) return false;
params_ = p;
capture_cb_ = std::move(capture_cb);
frame_samples_ = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
running_.store(true, std::memory_order_release);
#ifdef VOICECAT_HAS_AUDIO
// ── Capture device ──────────────────────────────────────────────────────
ma_device_config cap_cfg = ma_device_config_init(ma_device_type_capture);
cap_cfg.capture.format = ma_format_s16;
cap_cfg.capture.channels = p.channels;
cap_cfg.sampleRate = p.sample_rate;
cap_cfg.dataCallback = capture_data_cb;
cap_cfg.pUserData = this;
cap_cfg.capture.pDeviceID = nullptr; // always default for now
if (ma_device_init(nullptr, &cap_cfg, &capture_device_) == MA_SUCCESS) {
if (ma_device_start(&capture_device_) == MA_SUCCESS) {
capture_started_ = true;
} else {
ma_device_uninit(&capture_device_);
}
}
// ── Playback device ─────────────────────────────────────────────────────
ma_device_config pb_cfg = ma_device_config_init(ma_device_type_playback);
pb_cfg.playback.format = ma_format_s16;
pb_cfg.playback.channels = p.channels;
pb_cfg.sampleRate = p.sample_rate;
pb_cfg.dataCallback = playback_data_cb;
pb_cfg.pUserData = this;
pb_cfg.playback.pDeviceID = nullptr;
if (ma_device_init(nullptr, &pb_cfg, &playback_device_) == MA_SUCCESS) {
if (ma_device_start(&playback_device_) == MA_SUCCESS) {
playback_started_ = true;
} else {
ma_device_uninit(&playback_device_);
}
}
#endif // VOICECAT_HAS_AUDIO
return true;
}
void AudioEngine::stop() {
if (!running_.exchange(false)) return;
#ifdef VOICECAT_HAS_AUDIO
if (capture_started_) {
ma_device_stop(&capture_device_);
ma_device_uninit(&capture_device_);
capture_started_ = false;
}
if (playback_started_) {
ma_device_stop(&playback_device_);
ma_device_uninit(&playback_device_);
playback_started_ = false;
}
#endif
}
void AudioEngine::inject_capture(const int16_t* pcm, size_t n) {
std::lock_guard lk(inject_mu_);
size_t w = inject_write_.load(std::memory_order_relaxed);
for (size_t i = 0; i < n; ++i)
inject_ring_[(w + i) % kInjectCapSamples] = pcm[i];
inject_write_.store(w + n, std::memory_order_release);
// Fire capture_cb_ for each complete frame now available.
while (true) {
size_t r = inject_read_.load(std::memory_order_relaxed);
size_t avail = inject_write_.load(std::memory_order_acquire) - r;
if (avail < static_cast<size_t>(frame_samples_)) break;
std::vector<int16_t> frame(frame_samples_);
for (int i = 0; i < frame_samples_; ++i)
frame[i] = inject_ring_[(r + i) % kInjectCapSamples];
inject_read_.store(r + frame_samples_, std::memory_order_release);
if (capture_cb_) capture_cb_(frame.data(), frame_samples_);
}
}
void AudioEngine::push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f) {
std::lock_guard lk(streams_mu_);
streams_[ssrc].jitter.push(std::move(f));
}
void AudioEngine::set_stream_gain(uint32_t ssrc, float gain) {
std::lock_guard lk(streams_mu_);
streams_[ssrc].gain = gain;
}
void AudioEngine::set_stream_mute(uint32_t ssrc, bool mute) {
std::lock_guard lk(streams_mu_);
streams_[ssrc].mute = mute;
}
void AudioEngine::remove_stream(uint32_t ssrc) {
std::lock_guard lk(streams_mu_);
streams_.erase(ssrc);
}
uint32_t AudioEngine::stream_packets_lost(uint32_t ssrc) const {
std::lock_guard lk(streams_mu_);
auto it = streams_.find(ssrc);
return (it != streams_.end()) ? it->second.jitter.packets_lost() : 0;
}
uint32_t AudioEngine::stream_target_depth_ms(uint32_t ssrc) const {
std::lock_guard lk(streams_mu_);
auto it = streams_.find(ssrc);
return (it != streams_.end()) ? it->second.jitter.target_depth_ms() : 40;
}
#ifdef VOICECAT_HAS_OPUS
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
std::lock_guard lk(streams_mu_);
streams_[ssrc].decoder.init(p);
}
#endif
#ifdef VOICECAT_HAS_AUDIO
void AudioEngine::capture_data_cb(ma_device* dev, void* /*out*/,
const void* in, ma_uint32 frame_count) {
auto* self = static_cast<AudioEngine*>(dev->pUserData);
self->on_capture(static_cast<const int16_t*>(in), frame_count);
}
void AudioEngine::on_capture(const int16_t* pcm, ma_uint32 frames) {
if (capture_cb_) capture_cb_(pcm, static_cast<int>(frames));
}
void AudioEngine::playback_data_cb(ma_device* dev, void* out,
const void* /*in*/, ma_uint32 frame_count) {
auto* self = static_cast<AudioEngine*>(dev->pUserData);
self->on_playback(static_cast<int16_t*>(out), frame_count);
}
void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
std::memset(out, 0, frames * params_.channels * sizeof(int16_t));
std::unique_lock lk(streams_mu_, std::try_to_lock);
if (!lk) return; // contended: emit silence this period
#ifdef VOICECAT_HAS_OPUS
std::vector<int32_t> mix(frames * params_.channels, 0);
for (auto& [ssrc, stream] : streams_) {
if (stream.mute || !stream.decoder.valid()) continue;
auto maybe_frame = stream.jitter.pop(stream.playout_ts);
std::vector<int16_t> pcm(frames * params_.channels);
int n;
if (maybe_frame) {
n = stream.decoder.decode(
maybe_frame->payload.data(),
static_cast<int>(maybe_frame->payload.size()),
pcm.data(), static_cast<int>(pcm.size()));
} else {
n = stream.decoder.decode(nullptr, 0, pcm.data(),
static_cast<int>(pcm.size()));
}
if (n > 0) {
float g = stream.gain;
for (int i = 0; i < n * static_cast<int>(params_.channels); ++i)
mix[i] += static_cast<int32_t>(static_cast<float>(pcm[i]) * g);
}
stream.playout_ts += frames;
}
for (ma_uint32 i = 0; i < frames * params_.channels; ++i)
out[i] = static_cast<int16_t>(std::clamp(mix[i], -32768, 32767));
#else
(void)out;
(void)frames;
#endif
}
#endif // VOICECAT_HAS_AUDIO
} // namespace voicecat::audio

View File

@@ -6,33 +6,160 @@
* ... → Opus decode → per-user recv NS (listener-chosen) → gain/mute → mix → playback
*
* REAL-TIME RULE: audio-callback threads never allocate, lock, or block (architecture.md §3).
*
* STATUS: M0 stub.
* The JitterBuffer and per-stream maps are accessed only under a try_lock; a failed lock
* causes PLC for that period (acceptable for M2; lock-free ring buffer is the M3 upgrade).
*/
#ifndef VOICECAT_AUDIO_AUDIO_ENGINE_H
#define VOICECAT_AUDIO_AUDIO_ENGINE_H
#include <atomic>
#include <cstdint>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <unordered_map>
#include <vector>
#ifdef VOICECAT_HAS_AUDIO
// miniaudio single-header — MINIAUDIO_IMPLEMENTATION defined in audio_engine.cpp
#include <miniaudio.h>
#endif
#ifdef VOICECAT_HAS_OPUS
#include "codec/opus_codec.h"
#endif
namespace voicecat::audio {
// Adaptive per-ssrc jitter buffer (voice.md §5). TODO(M2).
// ── JitterBuffer ─────────────────────────────────────────────────────────────
// Per-ssrc adaptive jitter buffer. Thread-safe via internal mutex.
class JitterBuffer {
public:
uint32_t target_depth_ms() const { return target_depth_ms_; }
struct Frame {
uint16_t seq;
uint32_t timestamp;
bool fec_present;
std::vector<uint8_t> payload;
};
// Insert an incoming frame. Thread-safe.
void push(Frame f);
// Return the next frame whose timestamp <= playout_ts, or nullopt (caller should PLC).
// Drops frames that are too old (more than kLateDropSamples late).
std::optional<Frame> pop(uint32_t playout_ts);
uint32_t target_depth_ms()const { return target_depth_ms_.load(); }
uint32_t packets_lost() const { return lost_.load(); }
void reset();
private:
uint32_t target_depth_ms_ = 40;
static constexpr uint32_t kLateDropSamples = 48000 / 2; // 500 ms @48 kHz
mutable std::mutex mu_;
std::map<uint32_t, Frame> buf_; // keyed by timestamp (u32 wraps are handled below)
std::atomic<uint32_t> target_depth_ms_{40};
std::atomic<uint32_t> lost_{0};
// Jitter estimation (EWMA).
uint32_t last_push_ts_ = 0; // local clock estimate on last push
uint32_t jitter_est_ = 0; // EWMA jitter in samples
bool first_push_ = true;
};
// Owns miniaudio capture/playback, the APM instances, codecs, jitter buffers, and the mixer.
// ── AudioParams ──────────────────────────────────────────────────────────────
struct AudioParams {
uint32_t sample_rate = 48000;
uint32_t channels = 1;
uint32_t frame_ms = 20;
std::string capture_device_id; // "" = default
std::string playback_device_id; // "" = default
};
// ── AudioEngine ──────────────────────────────────────────────────────────────
// Owns miniaudio capture/playback, per-ssrc jitter buffers + Opus decoders, and the mixer.
class AudioEngine {
public:
// TODO(M2): start/stop capture+playback; push/pull frames via lock-free ring buffers.
bool running() const { return running_; }
// Callback type for encoded capture frames ready to be sent.
using CaptureCallback = std::function<void(const int16_t* pcm, int samples)>;
AudioEngine();
~AudioEngine();
AudioEngine(const AudioEngine&) = delete;
AudioEngine& operator=(const AudioEngine&) = delete;
// Start capture+playback devices. capture_cb is called on the encode thread
// for each capture frame (not on the audio callback thread).
bool start(const AudioParams& p, CaptureCallback capture_cb = nullptr);
void stop();
bool running() const { return running_.load(std::memory_order_acquire); }
// Inject synthetic PCM directly into the capture pipeline (bypasses real device).
// Thread-safe; can be called from any thread including tests.
void inject_capture(const int16_t* pcm, size_t n);
// Called by the net thread when a decoded voice frame arrives for a remote stream.
void push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f);
// Per-stream receive-side controls (safe from any thread).
void set_stream_gain(uint32_t ssrc, float gain); // 0.02.0, default 1.0
void set_stream_mute(uint32_t ssrc, bool mute);
void remove_stream(uint32_t ssrc);
// Get stats for a remote stream's jitter buffer.
uint32_t stream_packets_lost(uint32_t ssrc) const;
uint32_t stream_target_depth_ms(uint32_t ssrc) const;
#ifdef VOICECAT_HAS_OPUS
// Configure the Opus decoder for an incoming ssrc (must be called before
// push_recv_frame for that ssrc). Thread-safe.
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p);
#endif
private:
bool running_ = false;
#ifdef VOICECAT_HAS_AUDIO
static void capture_data_cb(ma_device*, void*, const void*, ma_uint32);
static void playback_data_cb(ma_device*, void*, const void*, ma_uint32);
void on_capture(const int16_t* pcm, ma_uint32 frames);
void on_playback(int16_t* out, ma_uint32 frames);
ma_device capture_device_{};
ma_device playback_device_{};
bool capture_started_ = false;
bool playback_started_ = false;
#endif
AudioParams params_{};
CaptureCallback capture_cb_;
std::atomic<bool> running_{false};
// Inject ring: stores raw int16 PCM written by inject_capture().
// The encode thread reads from this (no real capture device needed in tests).
std::mutex inject_mu_;
std::vector<int16_t> inject_ring_; // circular, size = frame_samples_
std::atomic<size_t> inject_write_{0};
std::atomic<size_t> inject_read_{0};
static constexpr size_t kInjectCapSamples = 48000 * 2; // 2 s @48 kHz mono
// Per remote stream (protected by streams_mu_).
struct RemoteStream {
JitterBuffer jitter;
#ifdef VOICECAT_HAS_OPUS
codec::OpusDecoder decoder;
#endif
float gain = 1.0f;
bool mute = false;
uint32_t playout_ts = 0;
};
mutable std::mutex streams_mu_;
std::unordered_map<uint32_t, RemoteStream> streams_;
int frame_samples_ = 960; // 20 ms @48 kHz
};
} // namespace voicecat::audio