feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer

Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.

Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 01:31:14 +02:00
parent 63f457fc54
commit 694494a5be
29 changed files with 2548 additions and 86 deletions

View File

@@ -175,6 +175,8 @@ void TcpServerConn::start() {
return;
}
self->connected_.store(true, std::memory_order_release);
// Export media keying material before the read loop starts.
if (self->cbs_.on_tls_ready) self->cbs_.on_tls_ready(*self->tls_);
// 50 ms timeout so tls_read_loop can drain the send queue between reads.
self->tls_->set_read_timeout(50);
self->tls_thread_ = std::thread([self] { self->tls_read_loop(); });
@@ -351,6 +353,67 @@ void TcpAcceptor::do_accept() {
});
}
// ── UdpMediaChannel ──────────────────────────────────────────────────────────
bool UdpMediaChannel::bind(asio::io_context& io, uint16_t port) {
if (bound_.load()) return false;
try {
socket_ = std::make_unique<asio::ip::udp::socket>(io);
socket_->open(asio::ip::udp::v4());
socket_->set_option(asio::socket_base::reuse_address(true));
socket_->bind(asio::ip::udp::endpoint(asio::ip::udp::v4(), port));
bound_.store(true, std::memory_order_release);
return true;
} catch (...) {
socket_.reset();
return false;
}
}
void UdpMediaChannel::start_recv(FrameCallback cb) {
frame_cb_ = std::move(cb);
do_recv();
}
void UdpMediaChannel::do_recv() {
if (!socket_ || closed_.load()) return;
socket_->async_receive_from(
asio::buffer(recv_buf_), sender_ep_,
[this](std::error_code ec, std::size_t n) {
if (ec || closed_.load()) return;
if (frame_cb_ && n > 0)
frame_cb_(recv_buf_.data(), n, sender_ep_);
do_recv();
});
}
void UdpMediaChannel::send_to(const uint8_t* data, size_t len,
asio::ip::udp::endpoint dst) {
if (!socket_ || closed_.load() || len == 0) return;
auto buf = std::make_shared<std::vector<uint8_t>>(data, data + len);
asio::post(socket_->get_executor(), [this, buf, dst]() mutable {
if (closed_.load()) return;
socket_->async_send_to(
asio::buffer(*buf), dst,
[buf](std::error_code, std::size_t) {});
});
}
void UdpMediaChannel::close() {
if (closed_.exchange(true)) return;
if (socket_) {
std::error_code ec;
socket_->cancel(ec);
socket_->close(ec);
}
}
asio::ip::udp::endpoint UdpMediaChannel::local_endpoint() const {
if (!socket_) return {};
std::error_code ec;
return socket_->local_endpoint(ec);
}
} // namespace voicecat::net
#endif // VOICECAT_HAS_NET

View File

@@ -40,6 +40,9 @@ struct TcpChannelCallbacks {
std::function<void(std::vector<uint8_t>)> on_frame; // one decoded frame payload
std::function<void(std::error_code)> on_error;
std::function<void()> on_disconnected;
// Called (on the handshake thread) right after TLS succeeds, before reads begin.
// Use to export keying material while the handshake context is still fresh.
std::function<void(voicecat::crypto::TlsContext&)> on_tls_ready;
};
// ── Client-side: owns an io_context + dedicated net thread ──────────────────
@@ -166,12 +169,44 @@ class TcpAcceptor {
};
// ── UDP media channel (M2) ───────────────────────────────────────────────────
// Thin async UDP socket. send_to() is thread-safe. Recv callbacks fire on the
// io_context's thread (same thread that runs the io_context::run() loop).
class UdpMediaChannel {
public:
bool bound() const { return bound_; }
using FrameCallback =
std::function<void(const uint8_t*, size_t, asio::ip::udp::endpoint)>;
UdpMediaChannel() = default;
~UdpMediaChannel() { close(); }
UdpMediaChannel(const UdpMediaChannel&) = delete;
UdpMediaChannel& operator=(const UdpMediaChannel&) = delete;
// Bind to 0.0.0.0:port (0 = OS-assigned). Must be called before start_recv/send_to.
bool bind(asio::io_context& io, uint16_t port = 0);
// Begin the async recv loop. cb is called on the io_context thread.
void start_recv(FrameCallback cb);
// Thread-safe fire-and-forget send. Copies data into a heap buffer.
void send_to(const uint8_t* data, size_t len, asio::ip::udp::endpoint dst);
// Cancel all async ops and close the socket. Safe to call from any thread.
void close();
asio::ip::udp::endpoint local_endpoint() const;
bool bound() const { return bound_.load(std::memory_order_acquire); }
private:
bool bound_ = false;
void do_recv();
// Socket + recv state live here; only accessed from the io_context thread after bind().
std::unique_ptr<asio::ip::udp::socket> socket_;
asio::ip::udp::endpoint sender_ep_;
std::array<uint8_t, 1500> recv_buf_{};
FrameCallback frame_cb_;
std::atomic<bool> bound_{false};
std::atomic<bool> closed_{false};
};
} // namespace voicecat::net

103
core/src/net/voice_frame.h Normal file
View File

@@ -0,0 +1,103 @@
/*
* net/voice_frame.h — UDP media frame wire format (header-only).
*
* Design: docs/voice.md §2. The 14-byte fixed header is also used as AEAD associated data.
* Multi-byte fields are big-endian. The payload (Opus packet) is AEAD-encrypted.
*/
#ifndef VOICECAT_NET_VOICE_FRAME_H
#define VOICECAT_NET_VOICE_FRAME_H
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <vector>
namespace voicecat::net {
// Frame types.
inline constexpr uint8_t kFrameVoice = 1;
inline constexpr uint8_t kFrameKeepalive = 2;
inline constexpr uint8_t kFrameUdpBinding = 3;
// Flag bits in VoiceFrame::flags.
inline constexpr uint8_t kFlagMarker = 0x01; // start of talkspurt
inline constexpr uint8_t kFlagFecPresent = 0x02; // this frame carries previous-frame FEC
inline constexpr uint8_t kFlagDtx = 0x04; // comfort-noise / DTX silence frame
inline constexpr uint8_t kFlagLast = 0x08; // last frame before stream stop
// Codec IDs.
inline constexpr uint16_t kCodecOpus = 0;
// Size of the serialized header (bytes before the payload).
inline constexpr size_t kVoiceHeaderSize = 14;
/*
* Wire layout (big-endian):
* [0] type u8
* [1] flags u8
* [2..3] codec u16
* [4..7] ssrc u32
* [8..9] seq u16 (low 16 bits of monotonic send counter)
* [10..13] timestamp u32 (sample clock @48 kHz)
* [14+] payload (AEAD-encrypted Opus packet)
*
* The 14-byte header is the AEAD AAD (authenticated, not encrypted).
* The payload region is the AEAD ciphertext + 16-byte Poly1305 MAC.
*/
struct VoiceFrame {
uint8_t type = kFrameVoice;
uint8_t flags = 0;
uint16_t codec = kCodecOpus;
uint32_t ssrc = 0;
uint16_t seq = 0;
uint32_t timestamp = 0;
std::vector<uint8_t> payload; // Opus bytes (pre-AEAD on send; post-AEAD on recv)
};
// Serialize the 14-byte header into buf[0..13]. buf must be at least kVoiceHeaderSize bytes.
inline void serialize_header(const VoiceFrame& f, uint8_t* buf) {
buf[0] = f.type;
buf[1] = f.flags;
buf[2] = static_cast<uint8_t>(f.codec >> 8);
buf[3] = static_cast<uint8_t>(f.codec & 0xFF);
buf[4] = static_cast<uint8_t>(f.ssrc >> 24);
buf[5] = static_cast<uint8_t>(f.ssrc >> 16);
buf[6] = static_cast<uint8_t>(f.ssrc >> 8);
buf[7] = static_cast<uint8_t>(f.ssrc & 0xFF);
buf[8] = static_cast<uint8_t>(f.seq >> 8);
buf[9] = static_cast<uint8_t>(f.seq & 0xFF);
buf[10] = static_cast<uint8_t>(f.timestamp >> 24);
buf[11] = static_cast<uint8_t>(f.timestamp >> 16);
buf[12] = static_cast<uint8_t>(f.timestamp >> 8);
buf[13] = static_cast<uint8_t>(f.timestamp & 0xFF);
}
// Parse the 14-byte header from buf. Returns false if len < kVoiceHeaderSize.
inline bool parse_header(const uint8_t* buf, size_t len, VoiceFrame& out) {
if (len < kVoiceHeaderSize) return false;
out.type = buf[0];
out.flags = buf[1];
out.codec = static_cast<uint16_t>((buf[2] << 8) | buf[3]);
out.ssrc = (static_cast<uint32_t>(buf[4]) << 24) |
(static_cast<uint32_t>(buf[5]) << 16) |
(static_cast<uint32_t>(buf[6]) << 8) |
static_cast<uint32_t>(buf[7]);
out.seq = static_cast<uint16_t>((buf[8] << 8) | buf[9]);
out.timestamp = (static_cast<uint32_t>(buf[10]) << 24) |
(static_cast<uint32_t>(buf[11]) << 16) |
(static_cast<uint32_t>(buf[12]) << 8) |
static_cast<uint32_t>(buf[13]);
return true;
}
// Serialize a full UDP_BINDING packet (type=3, token in payload, no AEAD).
inline std::vector<uint8_t> make_udp_binding_packet(const uint8_t* token, size_t token_len) {
std::vector<uint8_t> pkt(kVoiceHeaderSize + token_len, 0);
pkt[0] = kFrameUdpBinding;
std::memcpy(pkt.data() + kVoiceHeaderSize, token, token_len);
return pkt;
}
} // namespace voicecat::net
#endif // VOICECAT_NET_VOICE_FRAME_H