Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: DRED (Deep REDundancy) per-channel toggle

Browse Source 
Adds Opus 1.6 DRED support end-to-end: encoder embeds 20 ms of ML
redundancy in every packet when enabled; decoder recovers lost frames
from the next buffered packet's DRED extension rather than falling back
to PLC comfort noise.

Protocol: bool dred = 11 added to AudioConfig (backward-compatible,
defaults false). C ABI: int dred added to vc_audio_config. Encoder:
OPUS_SET_DRED_DURATION(2) when dred=true. Decoder: OpusDREDDecoder +
per-stream OpusDRED scratch pre-allocated off the RT thread;
JitterBuffer::try_copy_front_payload peeks at the next packet without
popping on every PLC step; opus_decoder_dred_decode reconstructs the
lost frame if DRED data is present, otherwise falls back to PLC.

New test: test_dred_toggle (22/22 ctest green).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-20 13:40:47 +02:00
parent de2c253199
commit fdcd8d1427
13 changed files with 412 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
core/src/audio/audio_engine.cpp
View File

@@ -127,6 +127,17 @@ std::optional<uint32_t> JitterBuffer::peek_front_ts() const {
return buf_.begin()->first;
}
size_t JitterBuffer::try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz) {
std::unique_lock lk(mu_, std::try_to_lock);
if (!lk || buf_.empty()) return 0;
auto it = buf_.begin();
if (it->first != expected_ts) return 0;
const auto& payload = it->second.payload;
size_t n = std::min(payload.size(), max_sz);
std::memcpy(out, payload.data(), n);
return n;
}
void JitterBuffer::reset() {
std::lock_guard lk(mu_);
buf_.clear();
@@ -147,6 +158,9 @@ AudioEngine::~AudioEngine() {
context_inited_ = false;
}
#endif
#ifdef VOICECAT_HAS_OPUS
if (dred_dec_) { opus_dred_decoder_destroy(dred_dec_); dred_dec_ = nullptr; }
#endif
}
#ifdef VOICECAT_HAS_AUDIO
@@ -167,6 +181,12 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
capture_cb_ = std::move(capture_cb);
frame_samples_ = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
running_.store(true, std::memory_order_release);
#ifdef VOICECAT_HAS_OPUS
if (!dred_dec_) {
int err = 0;
dred_dec_ = opus_dred_decoder_create(&err); // null on failure — DRED silently disabled
}
#endif
#ifdef VOICECAT_HAS_AUDIO
// Pre-allocate capture accumulators before the devices start so on_capture / on_loopback
@@ -412,7 +432,16 @@ bool AudioEngine::get_stream_state(uint32_t ssrc, float& gain, bool& mute, bool&
void AudioEngine::remove_stream(uint32_t ssrc) {
std::lock_guard lk(streams_mu_);
streams_.erase(ssrc);
auto it = streams_.find(ssrc);
if (it != streams_.end()) {
#ifdef VOICECAT_HAS_OPUS
if (it->second.dred_state_) {
opus_dred_free(it->second.dred_state_);
it->second.dred_state_ = nullptr;
}
#endif
streams_.erase(it);
}
}
std::vector<std::pair<uint32_t, bool>> AudioEngine::poll_talk_transitions() {
@@ -453,6 +482,12 @@ void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
int frame_samples = stream.decoder.frame_samples();
if (frame_samples <= 0) frame_samples = static_cast<int>(p.sample_rate / 1000 * p.frame_ms);
stream.init_ring(channels, frame_samples);
// DRED: pre-allocate per-stream scratch (no RT-thread allocation). 4000 bytes > max Opus pkt.
stream.dred_payload_scratch_.assign(4000, 0);
if (!stream.dred_state_) {
int err = 0;
stream.dred_state_ = opus_dred_alloc(&err); // null on failure — falls back to PLC
}
}
#endif
@@ -560,7 +595,35 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
sizeof(int16_t));
n = frame_samples;
} else {
n = stream.decoder.decode(nullptr, 0, stream.decode_scratch.data(), frame_samples);
// Try DRED recovery: if the next packet is already in the jitter buffer,
// parse its DRED extension and reconstruct the lost frame with it — producing
// better quality than PLC comfort noise. Falls back to PLC on any failure.
n = -1;
#ifdef VOICECAT_HAS_OPUS
if (dred_dec_ && stream.dred_state_) {
uint32_t next_ts = stream.playout_ts + static_cast<uint32_t>(frame_samples);
size_t psz = stream.jitter.try_copy_front_payload(
next_ts, stream.dred_payload_scratch_.data(),
stream.dred_payload_scratch_.size());
if (psz > 0) {
int dred_end = 0;
int ret = opus_dred_parse(
dred_dec_, stream.dred_state_,
stream.dred_payload_scratch_.data(),
static_cast<opus_int32>(psz),
frame_samples, static_cast<opus_int32>(params_.sample_rate),
&dred_end, 0);
if (ret > 0)
n = stream.decoder.decode_dred(stream.dred_state_, 0,
stream.decode_scratch.data(),
frame_samples);
}
}
#endif
if (n <= 0) {
n = stream.decoder.decode(nullptr, 0, stream.decode_scratch.data(),
frame_samples);
}
if (n > 0) stream.plc_samples_since_real += n; // track PLC streak
}
if (n <= 0) break; // decoder error/exhausted PLC; rest of this period stays silent

16
core/src/audio/audio_engine.h
View File

@@ -60,6 +60,11 @@ class JitterBuffer {
// AudioEngine::on_playback). Uses try_lock — never blocks the real-time callback.
std::optional<uint32_t> peek_front_ts() const;
// If the front frame's timestamp == expected_ts, copies its raw Opus payload into out
// (caller-allocated, max_sz bytes). Returns bytes copied, or 0 (lock miss / wrong ts /
// empty). Caller pre-allocates out to avoid RT-thread allocation. Uses try_lock.
size_t try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz);
uint32_t target_depth_ms()const { return target_depth_ms_.load(); }
uint32_t packets_lost() const { return lost_.load(); }
void reset();
@@ -389,6 +394,13 @@ class AudioEngine {
bool noise_reduction_enabled = false;
std::unique_ptr<ApmProcessor> recv_ns;
// DRED: pre-allocated scratch for loss recovery. dred_state_ is per-stream; see
// AudioEngine::dred_dec_ (shared). Allocated in init_recv_stream(); freed in remove_stream().
#ifdef VOICECAT_HAS_OPUS
::OpusDRED* dred_state_ = nullptr;
#endif
std::vector<uint8_t> dred_payload_scratch_; // pre-sized to 4000 bytes
// M3: talk-indicator edge detection (docs/voice.md §7) — updated by push_recv_frame
// (already off the real-time audio thread), polled by poll_talk_transitions().
std::atomic<int64_t> last_voice_ms{0};
@@ -452,6 +464,10 @@ class AudioEngine {
int frame_samples_ = 960; // 20 ms @48 kHz
#ifdef VOICECAT_HAS_OPUS
::OpusDREDDecoder* dred_dec_ = nullptr; // shared DRED decoder; null if unsupported
#endif
static constexpr int64_t kTalkHangoverMs = 300;
};