feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305 AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX, an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain wired through ConnSession/SessionRegistry into a new server-side SFU (MediaRelay) that decrypts and re-encrypts frames per channel member. Exit criterion verified: test_m2_voice — two headless clients relay 50 encrypted Opus frames through the server; ctest --preset m1-dev is 9/9 green. Also corrects protocol.md's UdpBinding diagram, which described the UDP-side binding packet as AEAD-sealed when it is in fact a plaintext bootstrap frame (separate from the TCP/TLS UdpBinding ack). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -6,33 +6,160 @@
|
||||
* ... → Opus decode → per-user recv NS (listener-chosen) → gain/mute → mix → playback
|
||||
*
|
||||
* REAL-TIME RULE: audio-callback threads never allocate, lock, or block (architecture.md §3).
|
||||
*
|
||||
* STATUS: M0 stub.
|
||||
* The JitterBuffer and per-stream maps are accessed only under a try_lock; a failed lock
|
||||
* causes PLC for that period (acceptable for M2; lock-free ring buffer is the M3 upgrade).
|
||||
*/
|
||||
#ifndef VOICECAT_AUDIO_AUDIO_ENGINE_H
|
||||
#define VOICECAT_AUDIO_AUDIO_ENGINE_H
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// miniaudio single-header — MINIAUDIO_IMPLEMENTATION defined in audio_engine.cpp
|
||||
#include <miniaudio.h>
|
||||
#endif
|
||||
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
#include "codec/opus_codec.h"
|
||||
#endif
|
||||
|
||||
namespace voicecat::audio {
|
||||
|
||||
// Adaptive per-ssrc jitter buffer (voice.md §5). TODO(M2).
|
||||
// ── JitterBuffer ─────────────────────────────────────────────────────────────
|
||||
// Per-ssrc adaptive jitter buffer. Thread-safe via internal mutex.
|
||||
class JitterBuffer {
|
||||
public:
|
||||
uint32_t target_depth_ms() const { return target_depth_ms_; }
|
||||
struct Frame {
|
||||
uint16_t seq;
|
||||
uint32_t timestamp;
|
||||
bool fec_present;
|
||||
std::vector<uint8_t> payload;
|
||||
};
|
||||
|
||||
// Insert an incoming frame. Thread-safe.
|
||||
void push(Frame f);
|
||||
|
||||
// Return the next frame whose timestamp <= playout_ts, or nullopt (caller should PLC).
|
||||
// Drops frames that are too old (more than kLateDropSamples late).
|
||||
std::optional<Frame> pop(uint32_t playout_ts);
|
||||
|
||||
uint32_t target_depth_ms()const { return target_depth_ms_.load(); }
|
||||
uint32_t packets_lost() const { return lost_.load(); }
|
||||
void reset();
|
||||
|
||||
private:
|
||||
uint32_t target_depth_ms_ = 40;
|
||||
static constexpr uint32_t kLateDropSamples = 48000 / 2; // 500 ms @48 kHz
|
||||
|
||||
mutable std::mutex mu_;
|
||||
std::map<uint32_t, Frame> buf_; // keyed by timestamp (u32 wraps are handled below)
|
||||
|
||||
std::atomic<uint32_t> target_depth_ms_{40};
|
||||
std::atomic<uint32_t> lost_{0};
|
||||
|
||||
// Jitter estimation (EWMA).
|
||||
uint32_t last_push_ts_ = 0; // local clock estimate on last push
|
||||
uint32_t jitter_est_ = 0; // EWMA jitter in samples
|
||||
bool first_push_ = true;
|
||||
};
|
||||
|
||||
// Owns miniaudio capture/playback, the APM instances, codecs, jitter buffers, and the mixer.
|
||||
// ── AudioParams ──────────────────────────────────────────────────────────────
|
||||
struct AudioParams {
|
||||
uint32_t sample_rate = 48000;
|
||||
uint32_t channels = 1;
|
||||
uint32_t frame_ms = 20;
|
||||
std::string capture_device_id; // "" = default
|
||||
std::string playback_device_id; // "" = default
|
||||
};
|
||||
|
||||
// ── AudioEngine ──────────────────────────────────────────────────────────────
|
||||
// Owns miniaudio capture/playback, per-ssrc jitter buffers + Opus decoders, and the mixer.
|
||||
class AudioEngine {
|
||||
public:
|
||||
// TODO(M2): start/stop capture+playback; push/pull frames via lock-free ring buffers.
|
||||
bool running() const { return running_; }
|
||||
// Callback type for encoded capture frames ready to be sent.
|
||||
using CaptureCallback = std::function<void(const int16_t* pcm, int samples)>;
|
||||
|
||||
AudioEngine();
|
||||
~AudioEngine();
|
||||
|
||||
AudioEngine(const AudioEngine&) = delete;
|
||||
AudioEngine& operator=(const AudioEngine&) = delete;
|
||||
|
||||
// Start capture+playback devices. capture_cb is called on the encode thread
|
||||
// for each capture frame (not on the audio callback thread).
|
||||
bool start(const AudioParams& p, CaptureCallback capture_cb = nullptr);
|
||||
void stop();
|
||||
|
||||
bool running() const { return running_.load(std::memory_order_acquire); }
|
||||
|
||||
// Inject synthetic PCM directly into the capture pipeline (bypasses real device).
|
||||
// Thread-safe; can be called from any thread including tests.
|
||||
void inject_capture(const int16_t* pcm, size_t n);
|
||||
|
||||
// Called by the net thread when a decoded voice frame arrives for a remote stream.
|
||||
void push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f);
|
||||
|
||||
// Per-stream receive-side controls (safe from any thread).
|
||||
void set_stream_gain(uint32_t ssrc, float gain); // 0.0–2.0, default 1.0
|
||||
void set_stream_mute(uint32_t ssrc, bool mute);
|
||||
void remove_stream(uint32_t ssrc);
|
||||
|
||||
// Get stats for a remote stream's jitter buffer.
|
||||
uint32_t stream_packets_lost(uint32_t ssrc) const;
|
||||
uint32_t stream_target_depth_ms(uint32_t ssrc) const;
|
||||
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
// Configure the Opus decoder for an incoming ssrc (must be called before
|
||||
// push_recv_frame for that ssrc). Thread-safe.
|
||||
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p);
|
||||
#endif
|
||||
|
||||
private:
|
||||
bool running_ = false;
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
static void capture_data_cb(ma_device*, void*, const void*, ma_uint32);
|
||||
static void playback_data_cb(ma_device*, void*, const void*, ma_uint32);
|
||||
void on_capture(const int16_t* pcm, ma_uint32 frames);
|
||||
void on_playback(int16_t* out, ma_uint32 frames);
|
||||
|
||||
ma_device capture_device_{};
|
||||
ma_device playback_device_{};
|
||||
bool capture_started_ = false;
|
||||
bool playback_started_ = false;
|
||||
#endif
|
||||
|
||||
AudioParams params_{};
|
||||
CaptureCallback capture_cb_;
|
||||
std::atomic<bool> running_{false};
|
||||
|
||||
// Inject ring: stores raw int16 PCM written by inject_capture().
|
||||
// The encode thread reads from this (no real capture device needed in tests).
|
||||
std::mutex inject_mu_;
|
||||
std::vector<int16_t> inject_ring_; // circular, size = frame_samples_
|
||||
std::atomic<size_t> inject_write_{0};
|
||||
std::atomic<size_t> inject_read_{0};
|
||||
static constexpr size_t kInjectCapSamples = 48000 * 2; // 2 s @48 kHz mono
|
||||
|
||||
// Per remote stream (protected by streams_mu_).
|
||||
struct RemoteStream {
|
||||
JitterBuffer jitter;
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
codec::OpusDecoder decoder;
|
||||
#endif
|
||||
float gain = 1.0f;
|
||||
bool mute = false;
|
||||
uint32_t playout_ts = 0;
|
||||
};
|
||||
mutable std::mutex streams_mu_;
|
||||
std::unordered_map<uint32_t, RemoteStream> streams_;
|
||||
|
||||
int frame_samples_ = 960; // 20 ms @48 kHz
|
||||
};
|
||||
|
||||
} // namespace voicecat::audio
|
||||
|
||||
Reference in New Issue
Block a user