feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.
Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 01:31:14 +02:00
|
|
|
#include "media_relay.h"
|
|
|
|
|
|
|
|
|
|
#ifdef VOICECAT_HAS_NET
|
|
|
|
|
|
|
|
|
|
#include <array>
|
|
|
|
|
#include <cstdio>
|
|
|
|
|
|
|
|
|
|
#include "conn_session.h"
|
|
|
|
|
#include "crypto/crypto.h"
|
|
|
|
|
#include "net/voice_frame.h"
|
|
|
|
|
#include "session_registry.h"
|
|
|
|
|
|
|
|
|
|
namespace voicecat::server {
|
|
|
|
|
|
|
|
|
|
MediaRelay::MediaRelay(asio::io_context& io, std::shared_ptr<SessionRegistry> registry)
|
|
|
|
|
: io_(io), registry_(std::move(registry)) {}
|
|
|
|
|
|
|
|
|
|
MediaRelay::~MediaRelay() { stop(); }
|
|
|
|
|
|
|
|
|
|
bool MediaRelay::bind(uint16_t port) {
|
|
|
|
|
return udp_.bind(io_, port);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void MediaRelay::start() {
|
|
|
|
|
udp_.start_recv([this](const uint8_t* data, size_t len, asio::ip::udp::endpoint sender) {
|
|
|
|
|
on_udp_frame(data, len, sender);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void MediaRelay::stop() {
|
|
|
|
|
udp_.close();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
uint16_t MediaRelay::media_port() const {
|
|
|
|
|
return static_cast<uint16_t>(udp_.local_endpoint().port());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void MediaRelay::on_udp_frame(const uint8_t* data, size_t len,
|
|
|
|
|
asio::ip::udp::endpoint sender) {
|
|
|
|
|
if (len < 1) return;
|
|
|
|
|
|
|
|
|
|
const uint8_t frame_type = data[0];
|
|
|
|
|
|
|
|
|
|
if (frame_type == voicecat::net::kFrameUdpBinding) {
|
|
|
|
|
// Payload = 16-byte token after the 14-byte header.
|
|
|
|
|
if (len < voicecat::net::kVoiceHeaderSize + 16) return;
|
|
|
|
|
std::array<uint8_t, 16> token{};
|
|
|
|
|
std::memcpy(token.data(), data + voicecat::net::kVoiceHeaderSize, 16);
|
|
|
|
|
|
|
|
|
|
auto session = registry_->find_by_udp_token(token);
|
|
|
|
|
if (!session) return;
|
|
|
|
|
|
|
|
|
|
session->set_udp_endpoint(sender);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (frame_type == voicecat::net::kFrameVoice) {
|
|
|
|
|
if (len < voicecat::net::kVoiceHeaderSize + crypto_aead_chacha20poly1305_ietf_ABYTES) return;
|
|
|
|
|
|
|
|
|
|
// Resolve sender session.
|
|
|
|
|
auto sender_session = registry_->find_by_udp_endpoint(sender);
|
|
|
|
|
if (!sender_session) return;
|
|
|
|
|
|
|
|
|
|
auto* recv_crypto = sender_session->recv_crypto();
|
|
|
|
|
if (!recv_crypto) return;
|
|
|
|
|
|
|
|
|
|
// AAD = 14-byte header (authenticated, not encrypted).
|
|
|
|
|
const uint8_t* aad = data;
|
|
|
|
|
const uint8_t* sealed = data + voicecat::net::kVoiceHeaderSize;
|
|
|
|
|
size_t sealed_len = len - voicecat::net::kVoiceHeaderSize;
|
|
|
|
|
|
|
|
|
|
if (plain_buf_.size() < sealed_len) plain_buf_.resize(sealed_len);
|
|
|
|
|
|
|
|
|
|
long plain_len = recv_crypto->open(sealed, sealed_len, aad, voicecat::net::kVoiceHeaderSize,
|
|
|
|
|
plain_buf_.data(), plain_buf_.size());
|
|
|
|
|
if (plain_len < 0) return; // auth failure or replay
|
|
|
|
|
|
|
|
|
|
// Parse the voice frame header to find the source ssrc/channel.
|
|
|
|
|
voicecat::net::VoiceFrame hdr{};
|
|
|
|
|
if (!voicecat::net::parse_header(data, len, hdr)) return;
|
|
|
|
|
|
|
|
|
|
// Find the channel and get all other members.
|
|
|
|
|
uint32_t uid = sender_session->user_id();
|
|
|
|
|
uint32_t channel = registry_->user_channel(uid);
|
|
|
|
|
if (channel == 0) return;
|
|
|
|
|
|
|
|
|
|
auto members = registry_->find_channel_sessions(channel, sender_session->session_id());
|
|
|
|
|
|
|
|
|
|
// Re-encrypt and relay to each member.
|
|
|
|
|
for (auto& member : members) {
|
|
|
|
|
if (!member->has_udp_endpoint()) continue;
|
|
|
|
|
|
|
|
|
|
auto* send_crypto = member->send_crypto();
|
|
|
|
|
if (!send_crypto) continue;
|
|
|
|
|
|
|
|
|
|
// Build an outgoing frame with the same 14-byte header.
|
|
|
|
|
if (seal_buf_.size() < voicecat::net::kVoiceHeaderSize +
|
|
|
|
|
static_cast<size_t>(plain_len) +
|
|
|
|
|
crypto_aead_chacha20poly1305_ietf_ABYTES) {
|
|
|
|
|
seal_buf_.resize(voicecat::net::kVoiceHeaderSize +
|
|
|
|
|
static_cast<size_t>(plain_len) +
|
|
|
|
|
crypto_aead_chacha20poly1305_ietf_ABYTES);
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-17 21:38:27 +02:00
|
|
|
// Copy header (ssrc, ts, flags pass through for demux/playout), then rewrite the
|
|
|
|
|
// seq field to THIS recipient's next send counter. The media AEAD nonce is an
|
|
|
|
|
// implicit per-direction monotonic counter; open() reconstructs it from the seq in
|
|
|
|
|
// the header (the AAD). Since we re-seal with the recipient's send_crypto (its own
|
|
|
|
|
// counter), the verbatim sender seq would no longer match the nonce seal() uses and
|
|
|
|
|
// every relayed frame would fail auth. Set seq = peek_send_counter() BEFORE sealing
|
|
|
|
|
// so the header (which is the authenticated AAD) carries the matching counter.
|
feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.
Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 01:31:14 +02:00
|
|
|
std::memcpy(seal_buf_.data(), data, voicecat::net::kVoiceHeaderSize);
|
2026-06-17 21:38:27 +02:00
|
|
|
const uint64_t send_ctr = send_crypto->peek_send_counter();
|
|
|
|
|
seal_buf_[8] = static_cast<uint8_t>((send_ctr >> 8) & 0xFF);
|
|
|
|
|
seal_buf_[9] = static_cast<uint8_t>(send_ctr & 0xFF);
|
feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.
Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 01:31:14 +02:00
|
|
|
|
|
|
|
|
uint8_t* out_payload = seal_buf_.data() + voicecat::net::kVoiceHeaderSize;
|
|
|
|
|
long sealed_out = send_crypto->seal(
|
|
|
|
|
plain_buf_.data(), static_cast<size_t>(plain_len),
|
|
|
|
|
seal_buf_.data(), voicecat::net::kVoiceHeaderSize,
|
|
|
|
|
out_payload,
|
|
|
|
|
static_cast<size_t>(plain_len) + crypto_aead_chacha20poly1305_ietf_ABYTES);
|
|
|
|
|
|
|
|
|
|
if (sealed_out < 0) continue;
|
|
|
|
|
|
|
|
|
|
udp_.send_to(seal_buf_.data(),
|
|
|
|
|
voicecat::net::kVoiceHeaderSize + static_cast<size_t>(sealed_out),
|
|
|
|
|
member->udp_endpoint());
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
fix(net): broadcast LEFT on disconnect, add keepalive/reaper, cap PLC hiss
Three reported bugs traced to one root cause plus two missing designed features:
1. Stale users + eternal PLC hiss (root cause): ConnSession::close() silently
erased dropped users without broadcasting UserEvent::LEFT, so peers never
learned the user left and their audio engines never called remove_stream —
Opus PLC synthesized comfort noise forever. Fix: broadcast_left() helper
+ close() broadcasts LEFT before erasing.
2. PLC cap (defense-in-depth): on_playback now caps pure PLC at ~2s, then
emits digital silence so a stale stream can never hiss forever even if
remove_stream is skipped. Resets automatically on fresh packets.
3. No timeout / no ping: client never sent Ping, server had no last_seen /
reaper, so half-open connections (NAT timeout, wifi loss, sleep) left
ghost users forever. Fix: client Ping every 15s with RTT measurement,
ConnSession::last_seen bumped on every inbound TCP/UDP frame, steady_timer
reaper sweeps every 15s and drops sessions older than 45s (configurable
via server::Config).
4. UDP KEEPALIVE: client sends plaintext kFrameKeepalive every 5s; server
bumps last_seen + echoes back. Keeps NAT bindings alive and lets media
activity defer the reaper independently of TCP.
5. Graceful client disconnect: vc_disconnect() sends Disconnect{code=0} via
a flag-based io-thread exit (no double-close race); server handles
client-sent Disconnect with immediate close() + LEFT broadcast.
3 new tests: disconnect_left, plc_cap, reaper_timeout. 21/21 ctest green.
Docs: protocol.md §6/§7, voice.md §6, architecture.md §5, PROGRESS.md.
2026-06-18 01:18:33 +02:00
|
|
|
if (frame_type == voicecat::net::kFrameKeepalive) {
|
|
|
|
|
// Plaintext KEEPALIVE (docs/voice.md §6): identify the sender by its verified UDP
|
|
|
|
|
// endpoint, bump last_seen so the reaper doesn't drop a client whose TCP control
|
|
|
|
|
// channel is idle but whose media path is alive, and echo the keepalive back so the
|
|
|
|
|
// client can measure media-path RTT/loss independently of the TCP ping.
|
|
|
|
|
auto session = registry_->find_by_udp_endpoint(sender);
|
|
|
|
|
if (!session) return;
|
|
|
|
|
session->touch_last_seen();
|
|
|
|
|
// Echo back to the sender (same plaintext header).
|
|
|
|
|
udp_.send_to(data, len, sender);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Unknown frame type: silently discard.
|
feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.
Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 01:31:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} // namespace voicecat::server
|
|
|
|
|
|
|
|
|
|
#endif // VOICECAT_HAS_NET
|