feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305 AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX, an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain wired through ConnSession/SessionRegistry into a new server-side SFU (MediaRelay) that decrypts and re-encrypts frames per channel member. Exit criterion verified: test_m2_voice — two headless clients relay 50 encrypted Opus frames through the server; ctest --preset m1-dev is 9/9 green. Also corrects protocol.md's UdpBinding diagram, which described the UDP-side binding packet as AEAD-sealed when it is in fact a plaintext bootstrap frame (separate from the TCP/TLS UdpBinding ack). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -175,6 +175,8 @@ void TcpServerConn::start() {
|
||||
return;
|
||||
}
|
||||
self->connected_.store(true, std::memory_order_release);
|
||||
// Export media keying material before the read loop starts.
|
||||
if (self->cbs_.on_tls_ready) self->cbs_.on_tls_ready(*self->tls_);
|
||||
// 50 ms timeout so tls_read_loop can drain the send queue between reads.
|
||||
self->tls_->set_read_timeout(50);
|
||||
self->tls_thread_ = std::thread([self] { self->tls_read_loop(); });
|
||||
@@ -351,6 +353,67 @@ void TcpAcceptor::do_accept() {
|
||||
});
|
||||
}
|
||||
|
||||
// ── UdpMediaChannel ──────────────────────────────────────────────────────────
|
||||
|
||||
bool UdpMediaChannel::bind(asio::io_context& io, uint16_t port) {
|
||||
if (bound_.load()) return false;
|
||||
try {
|
||||
socket_ = std::make_unique<asio::ip::udp::socket>(io);
|
||||
socket_->open(asio::ip::udp::v4());
|
||||
socket_->set_option(asio::socket_base::reuse_address(true));
|
||||
socket_->bind(asio::ip::udp::endpoint(asio::ip::udp::v4(), port));
|
||||
bound_.store(true, std::memory_order_release);
|
||||
return true;
|
||||
} catch (...) {
|
||||
socket_.reset();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void UdpMediaChannel::start_recv(FrameCallback cb) {
|
||||
frame_cb_ = std::move(cb);
|
||||
do_recv();
|
||||
}
|
||||
|
||||
void UdpMediaChannel::do_recv() {
|
||||
if (!socket_ || closed_.load()) return;
|
||||
socket_->async_receive_from(
|
||||
asio::buffer(recv_buf_), sender_ep_,
|
||||
[this](std::error_code ec, std::size_t n) {
|
||||
if (ec || closed_.load()) return;
|
||||
if (frame_cb_ && n > 0)
|
||||
frame_cb_(recv_buf_.data(), n, sender_ep_);
|
||||
do_recv();
|
||||
});
|
||||
}
|
||||
|
||||
void UdpMediaChannel::send_to(const uint8_t* data, size_t len,
|
||||
asio::ip::udp::endpoint dst) {
|
||||
if (!socket_ || closed_.load() || len == 0) return;
|
||||
auto buf = std::make_shared<std::vector<uint8_t>>(data, data + len);
|
||||
asio::post(socket_->get_executor(), [this, buf, dst]() mutable {
|
||||
if (closed_.load()) return;
|
||||
socket_->async_send_to(
|
||||
asio::buffer(*buf), dst,
|
||||
[buf](std::error_code, std::size_t) {});
|
||||
});
|
||||
}
|
||||
|
||||
void UdpMediaChannel::close() {
|
||||
if (closed_.exchange(true)) return;
|
||||
if (socket_) {
|
||||
std::error_code ec;
|
||||
socket_->cancel(ec);
|
||||
socket_->close(ec);
|
||||
}
|
||||
}
|
||||
|
||||
asio::ip::udp::endpoint UdpMediaChannel::local_endpoint() const {
|
||||
if (!socket_) return {};
|
||||
std::error_code ec;
|
||||
return socket_->local_endpoint(ec);
|
||||
}
|
||||
|
||||
} // namespace voicecat::net
|
||||
|
||||
#endif // VOICECAT_HAS_NET
|
||||
|
||||
@@ -40,6 +40,9 @@ struct TcpChannelCallbacks {
|
||||
std::function<void(std::vector<uint8_t>)> on_frame; // one decoded frame payload
|
||||
std::function<void(std::error_code)> on_error;
|
||||
std::function<void()> on_disconnected;
|
||||
// Called (on the handshake thread) right after TLS succeeds, before reads begin.
|
||||
// Use to export keying material while the handshake context is still fresh.
|
||||
std::function<void(voicecat::crypto::TlsContext&)> on_tls_ready;
|
||||
};
|
||||
|
||||
// ── Client-side: owns an io_context + dedicated net thread ──────────────────
|
||||
@@ -166,12 +169,44 @@ class TcpAcceptor {
|
||||
};
|
||||
|
||||
// ── UDP media channel (M2) ───────────────────────────────────────────────────
|
||||
// Thin async UDP socket. send_to() is thread-safe. Recv callbacks fire on the
|
||||
// io_context's thread (same thread that runs the io_context::run() loop).
|
||||
class UdpMediaChannel {
|
||||
public:
|
||||
bool bound() const { return bound_; }
|
||||
using FrameCallback =
|
||||
std::function<void(const uint8_t*, size_t, asio::ip::udp::endpoint)>;
|
||||
|
||||
UdpMediaChannel() = default;
|
||||
~UdpMediaChannel() { close(); }
|
||||
|
||||
UdpMediaChannel(const UdpMediaChannel&) = delete;
|
||||
UdpMediaChannel& operator=(const UdpMediaChannel&) = delete;
|
||||
|
||||
// Bind to 0.0.0.0:port (0 = OS-assigned). Must be called before start_recv/send_to.
|
||||
bool bind(asio::io_context& io, uint16_t port = 0);
|
||||
|
||||
// Begin the async recv loop. cb is called on the io_context thread.
|
||||
void start_recv(FrameCallback cb);
|
||||
|
||||
// Thread-safe fire-and-forget send. Copies data into a heap buffer.
|
||||
void send_to(const uint8_t* data, size_t len, asio::ip::udp::endpoint dst);
|
||||
|
||||
// Cancel all async ops and close the socket. Safe to call from any thread.
|
||||
void close();
|
||||
|
||||
asio::ip::udp::endpoint local_endpoint() const;
|
||||
bool bound() const { return bound_.load(std::memory_order_acquire); }
|
||||
|
||||
private:
|
||||
bool bound_ = false;
|
||||
void do_recv();
|
||||
|
||||
// Socket + recv state live here; only accessed from the io_context thread after bind().
|
||||
std::unique_ptr<asio::ip::udp::socket> socket_;
|
||||
asio::ip::udp::endpoint sender_ep_;
|
||||
std::array<uint8_t, 1500> recv_buf_{};
|
||||
FrameCallback frame_cb_;
|
||||
std::atomic<bool> bound_{false};
|
||||
std::atomic<bool> closed_{false};
|
||||
};
|
||||
|
||||
} // namespace voicecat::net
|
||||
|
||||
103
core/src/net/voice_frame.h
Normal file
103
core/src/net/voice_frame.h
Normal file
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* net/voice_frame.h — UDP media frame wire format (header-only).
|
||||
*
|
||||
* Design: docs/voice.md §2. The 14-byte fixed header is also used as AEAD associated data.
|
||||
* Multi-byte fields are big-endian. The payload (Opus packet) is AEAD-encrypted.
|
||||
*/
|
||||
#ifndef VOICECAT_NET_VOICE_FRAME_H
|
||||
#define VOICECAT_NET_VOICE_FRAME_H
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
namespace voicecat::net {
|
||||
|
||||
// Frame types.
|
||||
inline constexpr uint8_t kFrameVoice = 1;
|
||||
inline constexpr uint8_t kFrameKeepalive = 2;
|
||||
inline constexpr uint8_t kFrameUdpBinding = 3;
|
||||
|
||||
// Flag bits in VoiceFrame::flags.
|
||||
inline constexpr uint8_t kFlagMarker = 0x01; // start of talkspurt
|
||||
inline constexpr uint8_t kFlagFecPresent = 0x02; // this frame carries previous-frame FEC
|
||||
inline constexpr uint8_t kFlagDtx = 0x04; // comfort-noise / DTX silence frame
|
||||
inline constexpr uint8_t kFlagLast = 0x08; // last frame before stream stop
|
||||
|
||||
// Codec IDs.
|
||||
inline constexpr uint16_t kCodecOpus = 0;
|
||||
|
||||
// Size of the serialized header (bytes before the payload).
|
||||
inline constexpr size_t kVoiceHeaderSize = 14;
|
||||
|
||||
/*
|
||||
* Wire layout (big-endian):
|
||||
* [0] type u8
|
||||
* [1] flags u8
|
||||
* [2..3] codec u16
|
||||
* [4..7] ssrc u32
|
||||
* [8..9] seq u16 (low 16 bits of monotonic send counter)
|
||||
* [10..13] timestamp u32 (sample clock @48 kHz)
|
||||
* [14+] payload (AEAD-encrypted Opus packet)
|
||||
*
|
||||
* The 14-byte header is the AEAD AAD (authenticated, not encrypted).
|
||||
* The payload region is the AEAD ciphertext + 16-byte Poly1305 MAC.
|
||||
*/
|
||||
struct VoiceFrame {
|
||||
uint8_t type = kFrameVoice;
|
||||
uint8_t flags = 0;
|
||||
uint16_t codec = kCodecOpus;
|
||||
uint32_t ssrc = 0;
|
||||
uint16_t seq = 0;
|
||||
uint32_t timestamp = 0;
|
||||
std::vector<uint8_t> payload; // Opus bytes (pre-AEAD on send; post-AEAD on recv)
|
||||
};
|
||||
|
||||
// Serialize the 14-byte header into buf[0..13]. buf must be at least kVoiceHeaderSize bytes.
|
||||
inline void serialize_header(const VoiceFrame& f, uint8_t* buf) {
|
||||
buf[0] = f.type;
|
||||
buf[1] = f.flags;
|
||||
buf[2] = static_cast<uint8_t>(f.codec >> 8);
|
||||
buf[3] = static_cast<uint8_t>(f.codec & 0xFF);
|
||||
buf[4] = static_cast<uint8_t>(f.ssrc >> 24);
|
||||
buf[5] = static_cast<uint8_t>(f.ssrc >> 16);
|
||||
buf[6] = static_cast<uint8_t>(f.ssrc >> 8);
|
||||
buf[7] = static_cast<uint8_t>(f.ssrc & 0xFF);
|
||||
buf[8] = static_cast<uint8_t>(f.seq >> 8);
|
||||
buf[9] = static_cast<uint8_t>(f.seq & 0xFF);
|
||||
buf[10] = static_cast<uint8_t>(f.timestamp >> 24);
|
||||
buf[11] = static_cast<uint8_t>(f.timestamp >> 16);
|
||||
buf[12] = static_cast<uint8_t>(f.timestamp >> 8);
|
||||
buf[13] = static_cast<uint8_t>(f.timestamp & 0xFF);
|
||||
}
|
||||
|
||||
// Parse the 14-byte header from buf. Returns false if len < kVoiceHeaderSize.
|
||||
inline bool parse_header(const uint8_t* buf, size_t len, VoiceFrame& out) {
|
||||
if (len < kVoiceHeaderSize) return false;
|
||||
out.type = buf[0];
|
||||
out.flags = buf[1];
|
||||
out.codec = static_cast<uint16_t>((buf[2] << 8) | buf[3]);
|
||||
out.ssrc = (static_cast<uint32_t>(buf[4]) << 24) |
|
||||
(static_cast<uint32_t>(buf[5]) << 16) |
|
||||
(static_cast<uint32_t>(buf[6]) << 8) |
|
||||
static_cast<uint32_t>(buf[7]);
|
||||
out.seq = static_cast<uint16_t>((buf[8] << 8) | buf[9]);
|
||||
out.timestamp = (static_cast<uint32_t>(buf[10]) << 24) |
|
||||
(static_cast<uint32_t>(buf[11]) << 16) |
|
||||
(static_cast<uint32_t>(buf[12]) << 8) |
|
||||
static_cast<uint32_t>(buf[13]);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Serialize a full UDP_BINDING packet (type=3, token in payload, no AEAD).
|
||||
inline std::vector<uint8_t> make_udp_binding_packet(const uint8_t* token, size_t token_len) {
|
||||
std::vector<uint8_t> pkt(kVoiceHeaderSize + token_len, 0);
|
||||
pkt[0] = kFrameUdpBinding;
|
||||
std::memcpy(pkt.data() + kVoiceHeaderSize, token, token_len);
|
||||
return pkt;
|
||||
}
|
||||
|
||||
} // namespace voicecat::net
|
||||
|
||||
#endif // VOICECAT_NET_VOICE_FRAME_H
|
||||
Reference in New Issue
Block a user