Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: DRED (Deep REDundancy) per-channel toggle

Browse Source 
Adds Opus 1.6 DRED support end-to-end: encoder embeds 20 ms of ML
redundancy in every packet when enabled; decoder recovers lost frames
from the next buffered packet's DRED extension rather than falling back
to PLC comfort noise.

Protocol: bool dred = 11 added to AudioConfig (backward-compatible,
defaults false). C ABI: int dred added to vc_audio_config. Encoder:
OPUS_SET_DRED_DURATION(2) when dred=true. Decoder: OpusDREDDecoder +
per-stream OpusDRED scratch pre-allocated off the RT thread;
JitterBuffer::try_copy_front_payload peeks at the next packet without
popping on every PLC step; opus_decoder_dred_decode reconstructs the
lost frame if DRED data is present, otherwise falls back to PLC.

New test: test_dred_toggle (22/22 ctest green).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-20 13:40:47 +02:00
parent de2c253199
commit fdcd8d1427
13 changed files with 412 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/include/voicecat.h
View File

@@ -221,6 +221,7 @@ typedef struct vc_audio_config {
uint32_t expected_packet_loss; /* % 0..100 */
int dtx; /* bool */
uint32_t complexity; /* 0..10 */
int dred; /* bool — Deep REDundancy (Opus 1.6), off by default */
} vc_audio_config;
/* M5: permission bitset (mirrors protocol Permissions). */

1
core/proto/voicecat.proto
View File

@@ -87,6 +87,7 @@ message AudioConfig {
uint32 expected_packet_loss = 8; // %
bool dtx = 9;
uint32 complexity = 10; // 0..10
bool dred = 11; // Deep REDundancy (Opus 1.6); off by default
}
message StreamInfo {

67
core/src/audio/audio_engine.cpp
View File

@@ -127,6 +127,17 @@ std::optional<uint32_t> JitterBuffer::peek_front_ts() const {
return buf_.begin()->first;
}
size_t JitterBuffer::try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz) {
std::unique_lock lk(mu_, std::try_to_lock);
if (!lk || buf_.empty()) return 0;
auto it = buf_.begin();
if (it->first != expected_ts) return 0;
const auto& payload = it->second.payload;
size_t n = std::min(payload.size(), max_sz);
std::memcpy(out, payload.data(), n);
return n;
}
void JitterBuffer::reset() {
std::lock_guard lk(mu_);
buf_.clear();
@@ -147,6 +158,9 @@ AudioEngine::~AudioEngine() {
context_inited_ = false;
}
#endif
#ifdef VOICECAT_HAS_OPUS
if (dred_dec_) { opus_dred_decoder_destroy(dred_dec_); dred_dec_ = nullptr; }
#endif
}
#ifdef VOICECAT_HAS_AUDIO
@@ -167,6 +181,12 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
capture_cb_ = std::move(capture_cb);
frame_samples_ = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
running_.store(true, std::memory_order_release);
#ifdef VOICECAT_HAS_OPUS
if (!dred_dec_) {
int err = 0;
dred_dec_ = opus_dred_decoder_create(&err); // null on failure — DRED silently disabled
}
#endif
#ifdef VOICECAT_HAS_AUDIO
// Pre-allocate capture accumulators before the devices start so on_capture / on_loopback
@@ -412,7 +432,16 @@ bool AudioEngine::get_stream_state(uint32_t ssrc, float& gain, bool& mute, bool&
void AudioEngine::remove_stream(uint32_t ssrc) {
std::lock_guard lk(streams_mu_);
streams_.erase(ssrc);
auto it = streams_.find(ssrc);
if (it != streams_.end()) {
#ifdef VOICECAT_HAS_OPUS
if (it->second.dred_state_) {
opus_dred_free(it->second.dred_state_);
it->second.dred_state_ = nullptr;
}
#endif
streams_.erase(it);
}
}
std::vector<std::pair<uint32_t, bool>> AudioEngine::poll_talk_transitions() {
@@ -453,6 +482,12 @@ void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
int frame_samples = stream.decoder.frame_samples();
if (frame_samples <= 0) frame_samples = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
stream.init_ring(channels, frame_samples);
// DRED: pre-allocate per-stream scratch (no RT-thread allocation). 4000 bytes > max Opus pkt.
stream.dred_payload_scratch_.assign(4000, 0);
if (!stream.dred_state_) {
int err = 0;
stream.dred_state_ = opus_dred_alloc(&err); // null on failure — falls back to PLC
}
}
#endif
@@ -560,7 +595,35 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
sizeof(int16_t));
n = frame_samples;
} else {
n = stream.decoder.decode(nullptr, 0, stream.decode_scratch.data(), frame_samples);
// Try DRED recovery: if the next packet is already in the jitter buffer,
// parse its DRED extension and reconstruct the lost frame with it — producing
// better quality than PLC comfort noise. Falls back to PLC on any failure.
n = -1;
#ifdef VOICECAT_HAS_OPUS
if (dred_dec_ && stream.dred_state_) {
uint32_t next_ts = stream.playout_ts + static_cast<uint32_t>(frame_samples);
size_t psz = stream.jitter.try_copy_front_payload(
next_ts, stream.dred_payload_scratch_.data(),
stream.dred_payload_scratch_.size());
if (psz > 0) {
int dred_end = 0;
int ret = opus_dred_parse(
dred_dec_, stream.dred_state_,
stream.dred_payload_scratch_.data(),
static_cast<opus_int32>(psz),
frame_samples, static_cast<opus_int32>(params_.sample_rate),
&dred_end, 0);
if (ret > 0)
n = stream.decoder.decode_dred(stream.dred_state_, 0,
stream.decode_scratch.data(),
frame_samples);
}
}
#endif
if (n <= 0) {
n = stream.decoder.decode(nullptr, 0, stream.decode_scratch.data(),
frame_samples);
}
if (n > 0) stream.plc_samples_since_real += n; // track PLC streak
}
if (n <= 0) break; // decoder error/exhausted PLC; rest of this period stays silent

16
core/src/audio/audio_engine.h
View File

@@ -60,6 +60,11 @@ class JitterBuffer {
// AudioEngine::on_playback). Uses try_lock — never blocks the real-time callback.
std::optional<uint32_t> peek_front_ts() const;
// If the front frame's timestamp == expected_ts, copies its raw Opus payload into out
// (caller-allocated, max_sz bytes). Returns bytes copied, or 0 (lock miss / wrong ts /
// empty). Caller pre-allocates out to avoid RT-thread allocation. Uses try_lock.
size_t try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz);
uint32_t target_depth_ms()const { return target_depth_ms_.load(); }
uint32_t packets_lost() const { return lost_.load(); }
void reset();
@@ -389,6 +394,13 @@ class AudioEngine {
bool noise_reduction_enabled = false;
std::unique_ptr<ApmProcessor> recv_ns;
// DRED: pre-allocated scratch for loss recovery. dred_state_ is per-stream; see
// AudioEngine::dred_dec_ (shared). Allocated in init_recv_stream(); freed in remove_stream().
#ifdef VOICECAT_HAS_OPUS
::OpusDRED* dred_state_ = nullptr;
#endif
std::vector<uint8_t> dred_payload_scratch_; // pre-sized to 4000 bytes
// M3: talk-indicator edge detection (docs/voice.md §7) — updated by push_recv_frame
// (already off the real-time audio thread), polled by poll_talk_transitions().
std::atomic<int64_t> last_voice_ms{0};
@@ -452,6 +464,10 @@ class AudioEngine {
int frame_samples_ = 960; // 20 ms @48 kHz
#ifdef VOICECAT_HAS_OPUS
::OpusDREDDecoder* dred_dec_ = nullptr; // shared DRED decoder; null if unsupported
#endif
static constexpr int64_t kTalkHangoverMs = 300;
};

10
core/src/codec/opus_codec.cpp
View File

@@ -32,6 +32,8 @@ bool OpusEncoder::init(const OpusParams& p) {
opus_encoder_ctl(enc_, OPUS_SET_DTX(p.dtx ? 1 : 0));
opus_encoder_ctl(enc_, OPUS_SET_PACKET_LOSS_PERC(
static_cast<opus_int32>(p.expected_packet_loss)));
// DRED: 2 × 10ms frames of redundancy covers one 20ms frame loss with ML reconstruction.
opus_encoder_ctl(enc_, OPUS_SET_DRED_DURATION(p.dred ? 2 : 0));
return true;
}
@@ -70,6 +72,14 @@ int OpusDecoder::decode(const uint8_t* opus_data, int len, int16_t* out_pcm, int
return n;
}
int OpusDecoder::decode_dred(::OpusDRED* dred, int32_t dred_offset,
int16_t* out_pcm, int max_samples) {
if (!dec_ || !dred) return -1;
int n = opus_decoder_dred_decode(dec_, dred, dred_offset, out_pcm, max_samples);
if (n < 0) { err_ = opus_strerror(n); return -1; }
return n;
}
void OpusDecoder::destroy() {
if (dec_) { opus_decoder_destroy(dec_); dec_ = nullptr; }
}

9
core/src/codec/opus_codec.h
View File

@@ -34,6 +34,7 @@ struct OpusParams {
uint32_t complexity = 10;
uint32_t expected_packet_loss = 0; // % 0..100
OpusApplication application = OpusApplication::Voip;
bool dred = false;
};
// Returns frame_samples for a given sample_rate + frame_ms.
@@ -95,6 +96,14 @@ class OpusDecoder {
int decode(const uint8_t* opus_data, int len, int16_t* out_pcm, int max_samples,
bool fec = false);
// Decode a lost frame using pre-parsed DRED state from the next received packet.
// dred_offset=0 means the frame immediately before the next packet.
// Returns frame_samples on success, -1 if DRED unavailable or decode failed.
#ifdef VOICECAT_HAS_OPUS
int decode_dred(::OpusDRED* dred, int32_t dred_offset, int16_t* out_pcm, int max_samples);
::OpusDecoder* raw() const { return dec_; }
#endif
void destroy();
bool valid() const { return dec_ != nullptr; }

4
core/src/core/client.cpp
View File

@@ -953,6 +953,7 @@ voicecat::codec::OpusParams opus_params_from_audio_config(const voicecat::v1::Au
p.complexity = a.complexity() ? a.complexity() : 10;
p.expected_packet_loss = a.expected_packet_loss();
p.application = static_cast<voicecat::codec::OpusApplication>(a.application());
p.dred = a.dred();
return p;
}
@@ -968,6 +969,7 @@ voicecat::v1::AudioConfig audio_config_from_vc(const vc_audio_config& c) {
a.set_expected_packet_loss(c.expected_packet_loss);
a.set_dtx(c.dtx != 0);
a.set_complexity(c.complexity);
a.set_dred(c.dred != 0);
return a;
}
@@ -1458,6 +1460,7 @@ vc_result vc_client::get_stream_audio_config(uint32_t user_id, uint32_t stream_i
out->expected_packet_loss = p.expected_packet_loss;
out->dtx = p.dtx ? 1 : 0;
out->complexity = p.complexity;
out->dred = p.dred ? 1 : 0;
return VC_OK;
}
@@ -1476,6 +1479,7 @@ vc_result vc_client::get_stream_audio_config(uint32_t user_id, uint32_t stream_i
out->expected_packet_loss = s.expected_packet_loss;
out->dtx = s.dtx ? 1 : 0;
out->complexity = s.complexity;
out->dred = s.dred ? 1 : 0;
return VC_OK;
}
return VC_ERR_INVALID_ARG;

1
core/src/session/session.cpp
View File

@@ -45,6 +45,7 @@ std::vector<Stream> copy_streams(
s.expected_packet_loss = pb.audio().expected_packet_loss();
s.dtx = pb.audio().dtx();
s.complexity = pb.audio().complexity();
s.dred = pb.audio().dred();
out.push_back(std::move(s));
}
return out;

1
core/src/session/session.h
View File

@@ -45,6 +45,7 @@ struct Stream {
uint32_t expected_packet_loss{0};
bool dtx{false};
uint32_t complexity{0};
bool dred{false};
};
struct User {