Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(M2): UDP voice/media plane -- SFU relay, Opus, AEAD, jitter buffer

Browse Source 
Adds the full voice pipeline: 14-byte binary frame header, ChaCha20-Poly1305
AEAD keyed from the TLS exporter, libopus encode/decode with FEC/PLC/DTX,
an adaptive per-ssrc jitter buffer, a miniaudio capture/playback engine, an
APM passthrough stub, and the UdpBinding/StreamAnnounce signaling chain
wired through ConnSession/SessionRegistry into a new server-side SFU
(MediaRelay) that decrypts and re-encrypts frames per channel member.

Exit criterion verified: test_m2_voice — two headless clients relay 50
encrypted Opus frames through the server; ctest --preset m1-dev is 9/9
green. Also corrects protocol.md's UdpBinding diagram, which described the
UDP-side binding packet as AEAD-sealed when it is in fact a plaintext
bootstrap frame (separate from the TCP/TLS UdpBinding ack).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-16 01:31:14 +02:00
parent 63f457fc54
commit 694494a5be
29 changed files with 2548 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
server/src/conn_session.h
View File

@@ -21,9 +21,13 @@
#include <string>
#include <vector>
#define ASIO_STANDALONE 1
#include <asio.hpp>
#include "crypto/crypto.h"
#include "proto/voicecat.pb.h"
namespace voicecat { class WorkerPool; } // defined in core/worker_pool.h
namespace voicecat { class WorkerPool; }
namespace voicecat::server {
@@ -34,36 +38,42 @@ class ConnSession : public std::enable_shared_from_this<ConnSession> {
public:
enum class State { WaitingHello, WaitingAuth, Authenticated, Disconnecting };
using SendFn = std::function<void(std::vector<uint8_t>)>;
using SendFn = std::function<void(std::vector<uint8_t>)>;
using CloseFn = std::function<void()>;
ConnSession(std::shared_ptr<Database> db,
std::shared_ptr<SessionRegistry> registry,
std::shared_ptr<voicecat::WorkerPool> workers,
const std::array<uint8_t, 32>& server_fp,
bool allow_guests);
ConnSession(std::shared_ptr<Database> db,
std::shared_ptr<SessionRegistry> registry,
std::shared_ptr<voicecat::WorkerPool> workers,
const std::array<uint8_t, 32>& server_fp,
bool allow_guests,
uint16_t udp_media_port = 0);
// Called after construction: gives the session its send + close handles.
void set_io(SendFn send_fn, CloseFn close_fn);
// Called by server after it has registered the session id.
void set_session_id(uint64_t id) { session_id_ = id; }
// Entry point: send ServerHello and begin reading.
void begin();
// Deliver a received frame (called from TcpServerConn's strand).
void on_frame(std::vector<uint8_t> frame);
// Called when the TCP connection drops.
void on_disconnect();
// Thread-safe send.
void send_envelope(const voicecat::v1::Envelope& env);
// Graceful close (can be called from any thread).
void close();
// ── M2: media key injection (called from on_tls_ready) ───────────────────
void set_media_crypto(std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> send,
std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> recv);
// ── M2: UDP endpoint (set by MediaRelay on UdpBinding) ────────────────────
void set_udp_endpoint(asio::ip::udp::endpoint ep);
asio::ip::udp::endpoint udp_endpoint() const;
bool has_udp_endpoint() const { return has_udp_ep_.load(); }
// ── M2: media crypto access (for SFU relay) ──────────────────────────────
voicecat::crypto::SodiumMediaCrypto* send_crypto();
voicecat::crypto::SodiumMediaCrypto* recv_crypto();
// ── M2: UDP token (for binding) ───────────────────────────────────────────
const std::array<uint8_t, 16>& udp_token() const { return udp_token_; }
// ── Accessors ──────────────────────────────────────────────────────────────
State state() const { return state_.load(); }
uint64_t session_id() const { return session_id_; }
uint32_t user_id() const { return user_id_; }
@@ -74,25 +84,41 @@ class ConnSession : public std::enable_shared_from_this<ConnSession> {
void handle_join_channel(uint64_t req_id, const voicecat::v1::JoinChannelRequest& msg);
void handle_text_message(const voicecat::v1::TextMessage& msg);
void handle_ping(const voicecat::v1::Ping& msg);
void handle_udp_binding(uint64_t req_id, const voicecat::v1::UdpBinding& msg);
void handle_stream_announce(uint64_t req_id, const voicecat::v1::StreamAnnounce& msg);
void finish_guest_auth(const voicecat::v1::GuestAuth& guest, uint64_t req_id);
void finish_password_auth(const std::string& username, const std::string& password,
uint64_t req_id);
void send_auth_result_ok(uint64_t req_id, const voicecat::v1::User& user,
const voicecat::v1::Permissions* perms = nullptr);
void send_state_snapshot();
void broadcast_user_joined(const voicecat::v1::User& user);
void send_disconnect_and_close(uint32_t code, const std::string& reason);
std::shared_ptr<Database> db_;
std::shared_ptr<SessionRegistry> registry_;
std::shared_ptr<Database> db_;
std::shared_ptr<SessionRegistry> registry_;
std::shared_ptr<voicecat::WorkerPool> workers_;
std::array<uint8_t, 32> server_fp_;
bool allow_guests_;
std::array<uint8_t, 32> server_fp_;
bool allow_guests_;
uint16_t udp_media_port_;
SendFn send_fn_;
CloseFn close_fn_;
std::atomic<State> state_{State::WaitingHello};
uint64_t session_id_{0}; // set once before begin(), then read-only
std::atomic<uint32_t> user_id_{0};
std::atomic<bool> closed_{false};
SendFn send_fn_;
CloseFn close_fn_;
std::atomic<State> state_{State::WaitingHello};
uint64_t session_id_{0};
std::atomic<uint32_t> user_id_{0};
std::atomic<bool> closed_{false};
// M2 UDP / media
std::array<uint8_t, 16> udp_token_{};
mutable std::mutex udp_ep_mu_;
asio::ip::udp::endpoint udp_ep_;
std::atomic<bool> has_udp_ep_{false};
mutable std::mutex crypto_mu_;
std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> send_crypto_;
std::unique_ptr<voicecat::crypto::SodiumMediaCrypto> recv_crypto_;
};
} // namespace voicecat::server