Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer

Browse Source 
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.

Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-16 01:31:14 +02:00
parent 63f457fc54
commit 694494a5be
29 changed files with 2548 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
core/src/codec/opus_codec.cpp
View File

@@ -2,6 +2,81 @@
namespace voicecat::codec {
// M0 stub. Brought up in M2. See docs/voice.md §3–4.
#ifdef VOICECAT_HAS_OPUS
// ── OpusEncoder ──────────────────────────────────────────────────────────────
bool OpusEncoder::init(const OpusParams& p) {
destroy();
channels_ = p.stereo ? 2 : 1;
frame_samples_ = opus_frame_samples(p);
int err = 0;
enc_ = opus_encoder_create(static_cast<opus_int32>(p.sample_rate), channels_,
OPUS_APPLICATION_VOIP, &err);
if (err != OPUS_OK || !enc_) {
err_ = opus_strerror(err);
return false;
}
opus_encoder_ctl(enc_, OPUS_SET_BITRATE(static_cast<opus_int32>(p.bitrate_bps)));
opus_encoder_ctl(enc_, OPUS_SET_COMPLEXITY(static_cast<opus_int32>(p.complexity)));
opus_encoder_ctl(enc_, OPUS_SET_INBAND_FEC(p.fec ? 1 : 0));
opus_encoder_ctl(enc_, OPUS_SET_DTX(p.dtx ? 1 : 0));
opus_encoder_ctl(enc_, OPUS_SET_PACKET_LOSS_PERC(
static_cast<opus_int32>(p.expected_packet_loss)));
return true;
}
int OpusEncoder::encode(const int16_t* pcm, int frame_samples, uint8_t* out_buf, int out_cap) {
if (!enc_) return -1;
int n = opus_encode(enc_, pcm, frame_samples, out_buf, out_cap);
if (n < 0) { err_ = opus_strerror(n); return -1; }
return n;
}
void OpusEncoder::destroy() {
if (enc_) { opus_encoder_destroy(enc_); enc_ = nullptr; }
}
// ── OpusDecoder ──────────────────────────────────────────────────────────────
bool OpusDecoder::init(const OpusParams& p) {
destroy();
channels_ = p.stereo ? 2 : 1;
frame_samples_ = opus_frame_samples(p);
int err = 0;
dec_ = opus_decoder_create(static_cast<opus_int32>(p.sample_rate), channels_, &err);
if (err != OPUS_OK || !dec_) {
err_ = opus_strerror(err);
return false;
}
return true;
}
int OpusDecoder::decode(const uint8_t* opus_data, int len, int16_t* out_pcm, int max_samples,
bool fec) {
if (!dec_) return -1;
int n = opus_decode(dec_, opus_data, len, out_pcm, max_samples, fec ? 1 : 0);
if (n < 0) { err_ = opus_strerror(n); return -1; }
return n;
}
void OpusDecoder::destroy() {
if (dec_) { opus_decoder_destroy(dec_); dec_ = nullptr; }
}
#else // !VOICECAT_HAS_OPUS — stubs
bool OpusEncoder::init(const OpusParams&) { err_ = "OPUS not compiled in"; return false; }
int OpusEncoder::encode(const int16_t*, int, uint8_t*, int) { return -1; }
void OpusEncoder::destroy() {}
bool OpusDecoder::init(const OpusParams&) { err_ = "OPUS not compiled in"; return false; }
int OpusDecoder::decode(const uint8_t*, int, int16_t*, int, bool) { return -1; }
void OpusDecoder::destroy() {}
#endif // VOICECAT_HAS_OPUS
} // namespace voicecat::codec

94
core/src/codec/opus_codec.h
View File

@@ -3,35 +3,105 @@
*
* Design: docs/voice.md §3–4. Per-channel AudioConfig (mono/stereo, bitrate, frame size,
* FEC, DTX, complexity). The server relays Opus payloads unmodified (no transcode).
*
* STATUS: M0 stub.
*/
#ifndef VOICECAT_CODEC_OPUS_CODEC_H
#define VOICECAT_CODEC_OPUS_CODEC_H
#include <cstdint>
#include <vector>
#ifdef VOICECAT_HAS_OPUS
#include <opus/opus.h>
#endif
namespace voicecat::codec {
struct OpusParams {
uint32_t sample_rate = 48000;
uint32_t bitrate_bps = 24000;
uint32_t frame_ms = 20;
bool stereo = false;
bool fec = true;
bool dtx = true;
uint32_t complexity = 10;
uint32_t expected_packet_loss = 0;
uint32_t sample_rate = 48000;
uint32_t bitrate_bps = 24000;
uint32_t frame_ms = 20;
bool stereo = false;
bool fec = true;
bool dtx = false;
uint32_t complexity = 10;
uint32_t expected_packet_loss = 0; // % 0..100
};
// Returns frame_samples for a given sample_rate + frame_ms.
inline int opus_frame_samples(const OpusParams& p) {
return static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
}
class OpusEncoder {
public:
// TODO(M2): init(params); encode(pcm, frame) -> opus bytes.
OpusEncoder() = default;
~OpusEncoder() { destroy(); }
OpusEncoder(const OpusEncoder&) = delete;
OpusEncoder& operator=(const OpusEncoder&) = delete;
// Initialise with the given params. Must be called before encode().
// Returns true on success; check error_string() on failure.
bool init(const OpusParams& p);
// Encode one frame of PCM (frame_ms ms @ sample_rate Hz, mono or stereo).
// pcm: interleaved int16 samples (frame_samples * channels samples).
// out_buf: caller-allocated output buffer (recommend >= 4000 bytes).
// Returns number of bytes written to out_buf, or -1 on error.
int encode(const int16_t* pcm, int frame_samples, uint8_t* out_buf, int out_cap);
void destroy();
bool valid() const { return enc_ != nullptr; }
int frame_samples()const { return frame_samples_; }
int channels() const { return channels_; }
const char* error_string() const { return err_; }
private:
#ifdef VOICECAT_HAS_OPUS
::OpusEncoder* enc_ = nullptr;
#else
void* enc_ = nullptr;
#endif
int frame_samples_ = 0;
int channels_ = 1;
const char* err_ = nullptr;
};
class OpusDecoder {
public:
// TODO(M2): init(params); decode(opus, out_pcm); PLC on loss; FEC from next packet.
OpusDecoder() = default;
~OpusDecoder() { destroy(); }
OpusDecoder(const OpusDecoder&) = delete;
OpusDecoder& operator=(const OpusDecoder&) = delete;
// Initialise. Must be called before decode().
bool init(const OpusParams& p);
// Decode one Opus packet into out_pcm (frame_samples * channels int16 samples).
// opus_data=nullptr, len=0 → PLC (free, always enabled by libopus).
// fec=true, next valid packet in opus_data → FEC recovery from previous loss.
// Returns number of samples decoded (= frame_samples), or -1 on error.
int decode(const uint8_t* opus_data, int len, int16_t* out_pcm, int max_samples,
bool fec = false);
void destroy();
bool valid() const { return dec_ != nullptr; }
int frame_samples()const { return frame_samples_; }
int channels() const { return channels_; }
const char* error_string() const { return err_; }
private:
#ifdef VOICECAT_HAS_OPUS
::OpusDecoder* dec_ = nullptr;
#else
void* dec_ = nullptr;
#endif
int frame_samples_ = 0;
int channels_ = 1;
const char* err_ = nullptr;
};
} // namespace voicecat::codec