Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer

Browse Source 
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.

Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-16 01:31:14 +02:00
parent 63f457fc54
commit 694494a5be
29 changed files with 2548 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
server/src/conn_session.cpp
View File

@@ -4,6 +4,9 @@
#include <chrono>
#include <cstdio>
#include <cstring>
#include <sodium.h>
#include "db.h"
#include "session_registry.h"
@@ -22,12 +25,16 @@ ConnSession::ConnSession(std::shared_ptr<Database> db,
std::shared_ptr<SessionRegistry> registry,
std::shared_ptr<voicecat::WorkerPool> workers,
const std::array<uint8_t, 32>& server_fp,
bool allow_guests)
bool allow_guests,
uint16_t udp_media_port)
: db_(std::move(db)),
registry_(std::move(registry)),
workers_(std::move(workers)),
server_fp_(server_fp),
allow_guests_(allow_guests) {}
allow_guests_(allow_guests),
udp_media_port_(udp_media_port) {
randombytes_buf(udp_token_.data(), udp_token_.size());
}
void ConnSession::set_io(SendFn send_fn, CloseFn close_fn) {
send_fn_ = std::move(send_fn);
@@ -67,6 +74,14 @@ void ConnSession::on_frame(std::vector<uint8_t> frame) {
if (st == State::Authenticated)
registry_->set_user_channel(user_id_.load(), 1);
break;
case voicecat::v1::Envelope::kUdpBinding:
if (st == State::Authenticated)
handle_udp_binding(env.request_id(), env.udp_binding());
break;
case voicecat::v1::Envelope::kStreamAnnounce:
if (st == State::Authenticated)
handle_stream_announce(env.request_id(), env.stream_announce());
break;
default:
break;
}
@@ -93,6 +108,44 @@ void ConnSession::close() {
if (close_fn_) close_fn_();
}
// ── M2: media crypto ─────────────────────────────────────────────────────────
void ConnSession::set_media_crypto(
std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> send,
std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> recv) {
std::lock_guard lk(crypto_mu_);
send_crypto_ = std::move(send);
recv_crypto_ = std::move(recv);
}
voicecat::crypto::SodiumMediaCrypto* ConnSession::send_crypto() {
std::lock_guard lk(crypto_mu_);
return send_crypto_.get();
}
voicecat::crypto::SodiumMediaCrypto* ConnSession::recv_crypto() {
std::lock_guard lk(crypto_mu_);
return recv_crypto_.get();
}
// ── M2: UDP endpoint ─────────────────────────────────────────────────────────
void ConnSession::set_udp_endpoint(asio::ip::udp::endpoint ep) {
{
std::lock_guard lk(udp_ep_mu_);
udp_ep_ = ep;
}
has_udp_ep_.store(true, std::memory_order_release);
registry_->register_udp_endpoint(ep, session_id_);
}
asio::ip::udp::endpoint ConnSession::udp_endpoint() const {
std::lock_guard lk(udp_ep_mu_);
return udp_ep_;
}
// ── Handlers ─────────────────────────────────────────────────────────────────
void ConnSession::handle_client_hello(uint64_t req_id, const voicecat::v1::ClientHello& msg) {
if (msg.proto_version() != 1) {
send_disconnect_and_close(1, "unsupported protocol version");
@@ -106,6 +159,7 @@ void ConnSession::handle_client_hello(uint64_t req_id, const voicecat::v1::Clien
if (allow_guests_) hello->add_auth_methods("guest");
hello->add_auth_methods("password");
hello->set_server_identity_fingerprint(server_fp_.data(), server_fp_.size());
if (udp_media_port_) hello->set_udp_port(udp_media_port_);
send_envelope(env);
state_.store(State::WaitingAuth, std::memory_order_release);
}
@@ -141,12 +195,15 @@ void ConnSession::finish_guest_auth(const voicecat::v1::GuestAuth& guest, uint64
user_id_.store(uid, std::memory_order_relaxed);
state_.store(State::Authenticated, std::memory_order_release);
registry_->register_udp_token(udp_token_, session_id_);
{
auto env = make_env(req_id);
auto* res = env.mutable_auth_result();
res->set_ok(true);
res->set_session_id(session_id_);
*res->mutable_self() = user;
res->set_udp_token(udp_token_.data(), udp_token_.size());
send_envelope(env);
}
broadcast_user_joined(user);
@@ -176,6 +233,8 @@ void ConnSession::finish_password_auth(const std::string& username,
self->user_id_.store(uid, std::memory_order_relaxed);
self->state_.store(State::Authenticated, std::memory_order_release);
self->registry_->register_udp_token(self->udp_token_, self->session_id_);
{
auto env = make_env(req_id);
auto* res = env.mutable_auth_result();
@@ -184,6 +243,7 @@ void ConnSession::finish_password_auth(const std::string& username,
*res->mutable_self() = user;
auto* perms = res->mutable_permissions();
perms->set_is_admin(acc->is_admin);
res->set_udp_token(self->udp_token_.data(), self->udp_token_.size());
self->send_envelope(env);
}
self->broadcast_user_joined(user);
@@ -247,6 +307,48 @@ void ConnSession::handle_ping(const voicecat::v1::Ping& msg) {
send_envelope(env);
}
void ConnSession::handle_udp_binding(uint64_t req_id, const voicecat::v1::UdpBinding& msg) {
if (msg.ack()) return; // server→client direction; ignore if echoed back
const std::string& tok = msg.udp_token();
if (tok.size() != 16 || std::memcmp(tok.data(), udp_token_.data(), 16) != 0) {
// Bad token — silently ignore (don't leak timing information)
return;
}
// Ack over TCP; MediaRelay will set the UDP endpoint when the UDP binding packet arrives.
auto env = make_env(req_id);
env.mutable_udp_binding()->set_ack(true);
send_envelope(env);
}
void ConnSession::handle_stream_announce(uint64_t req_id,
const voicecat::v1::StreamAnnounce& msg) {
uint32_t ssrc = registry_->assign_ssrc(session_id_);
auto env = make_env(req_id);
auto* res = env.mutable_stream_announce_result();
res->set_ok(true);
res->set_stream_id(1);
res->set_ssrc(ssrc);
auto* eff = res->mutable_effective_audio();
if (msg.has_requested_audio()) {
*eff = msg.requested_audio();
} else {
eff->set_codec(0); // OPUS
eff->set_sample_rate(48000);
eff->set_bitrate_bps(24000);
eff->set_frame_ms(20);
eff->set_fec(true);
}
if (eff->sample_rate() == 0) eff->set_sample_rate(48000);
if (eff->bitrate_bps() == 0) eff->set_bitrate_bps(24000);
if (eff->frame_ms() == 0) eff->set_frame_ms(20);
send_envelope(env);
}
void ConnSession::send_disconnect_and_close(uint32_t code, const std::string& reason) {
auto env = make_env();
auto* d = env.mutable_disconnect();