diff --git a/CLAUDE.md b/CLAUDE.md index 761a4e5..db41b76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ and what's next* read [`PROGRESS.md`](PROGRESS.md); for *design* read [`docs/`]( > server-mute, channel CRUD, in-app account management, disconnect/keepalive/reaper. Windows > WinForms C# client shipped (M4). **macOS AppKit client shipped** — `VoiceCatMac.xcodeproj` > at `clients/apple/macOS/`. **iOS SwiftUI client shipped** — `VoiceCatiOS.xcodeproj` at -> `clients/apple/iOS/`. `ctest --preset dev` green — 26/26 tests. +> `clients/apple/iOS/`. `ctest --preset dev` green — 27/27 tests. > External PCM feed/tap API (`vc_stream_feed_pcm` + `vc_set_pcm_sink`) shipped. > **Screen-audio sharing shipped on macOS (ScreenCaptureKit) and iOS (ReplayKit Broadcast > Upload Extension → host App Group ring → `vc_stream_feed_pcm`).** See [`PROGRESS.md`](PROGRESS.md). diff --git a/PROGRESS.md b/PROGRESS.md index b26e4aa..a917614 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -10,6 +10,25 @@ up instantly. Newest status at the top. ## ▶ Where we left off / next action +- **Done (2026-06-22):** **Fixed growing voice latency (jitter-buffer depth ratchet).** Symptom: + end-to-end latency grew to multiple seconds and "drifted backward," reset only by leaving/ + rejoining voice (DTX/FEC/DRED on, 10% loss). Root cause was **not** the codec settings (10% loss + is just an `OPUS_SET_PACKET_LOSS_PERC` encoder hint; FEC/DRED add no standing latency) but the + receiver playout logic in `core/src/audio/audio_engine.cpp`: the playout clock free-ran in real + time while the sender omitted silence from its timestamps and set **no header flags at all**, and + the only correction snapped the clock to the *oldest* buffered frame (could only *add* latency) — + with `target_depth_ms_` computed but never enforced, so latency could only grow or be reset. + **Fix:** bounded-depth playout — (re)seed to the *leading edge* (newest frame) on start/marker/ + starve, and **frame-skip catch-up** that trims a backlog beyond `target + hysteresis` (the missing + downward force). Plus hardening: adaptive late-drop window, talkspurt `kFlagMarker`/`kFlagDtx` + now actually stamped by the sender (`client.cpp` send path) and consumed on recv, EWMA outlier + rejection (silence gaps/stragglers no longer poison the estimate), duplicate counting, ring- + underrun diagnostics (`stream_underruns`/`stream_duplicates`). New regression test + `tests/test_jitter_depth.cpp` asserts depth stays bounded (<200 ms) while arrivals outrun playout + for ~4 s. `ctest --preset dev` green — **27/27**. Docs: `docs/voice.md` §5 rewritten. + - **Next (manual E2E):** two clients in a channel, DTX/FEC/DRED on — talk in alternating bursts + for several minutes and confirm latency stays low/stable (no backward drift, no rejoin needed). + - **Windows done / Apple awaiting Mac build (2026-06-22):** **Event sound effects + optional text-to-speech for all clients.** Clients now play a cue per session event and can optionally speak it (TTS off by default; when on it announces joins/leaves and reads message/PM bodies). diff --git a/core/src/audio/audio_engine.cpp b/core/src/audio/audio_engine.cpp index 4713ac2..f05509d 100644 --- a/core/src/audio/audio_engine.cpp +++ b/core/src/audio/audio_engine.cpp @@ -18,13 +18,18 @@ int64_t now_ms() { .count(); } -// Playout-clock re-sync thresholds (samples @ 48 kHz). The playout clock advances every callback +// Bounded-depth playout targeting (samples @ 48 kHz). The playout clock advances every callback // via the PLC path in on_playback, while the sender's frame timestamps only advance while it is -// actually transmitting — so they drift across VAD/PTT silence gaps and late joins. Both are kept -// under JitterBuffer::kLateDropSamples (500 ms) so the clock is snapped back before frames would -// begin to be dropped-as-late, which is what produced the "talk indicator lit, no audio" silence. -constexpr int32_t kResyncAheadSamples = 48000 * 200 / 1000; // clock 200 ms ahead → re-seed -constexpr int32_t kResyncBehindSamples = 48000 * 500 / 1000; // clock 500 ms behind → re-seed +// actually transmitting (silence is omitted) — so the clock drifts away from the stream timeline +// across VAD/PTT/DTX gaps and late joins. Rather than snap the clock to the *oldest* buffered +// frame (which could only add standing latency and never trim it — the cause of the "latency keeps +// drifting backward, fixed by rejoin" bug), on_playback keeps the clock a bounded `target` behind +// the *newest* arrival and frame-skips to catch up when the backlog grows. These are the floor and +// hysteresis for that target; the steady-state target itself comes from JitterBuffer's adaptive +// estimate (target_depth_samples()). +constexpr int32_t kMinDepthSamples = 48000 * 40 / 1000; // floor for the catch-up target depth +constexpr int32_t kCatchupSamples = 48000 * 60 / 1000; // skip when depth > target + 60 ms +constexpr int32_t kStarveSamples = 48000 * 120 / 1000; // reseed when clock 120 ms past newest // PLC cap: after this many consecutive samples of pure packet-loss concealment (no real // packet decoded), stop calling opus_decode(nullptr,0,...) and emit silence instead. Opus @@ -79,22 +84,37 @@ bool hex_decode_device_id(const std::string& hex, ma_device_id* out) { void JitterBuffer::push(Frame f) { std::lock_guard lk(mu_); - uint32_t ts = f.timestamp; + uint32_t ts = f.timestamp; + bool marker = f.marker; - // Jitter estimation (EWMA of inter-arrival gap vs expected gap). + // Jitter estimation (EWMA of inter-arrival gap vs the expected per-frame gap). Skip + // silence-gap outliers — a talkspurt restart (marker) or any gap far larger than a frame + // (DTX/VAD/PTT silence) is not jitter; counting it would spike the estimate and inflate the + // target depth for the rest of the call. The sender omits silence from its timestamp, so a + // restart can also arrive "behind" (negative gap) — also an outlier. if (!first_push_) { - uint32_t arrived_gap = ts - last_push_ts_; - uint32_t expected_gap = 960; // 20 ms @48k; TODO: derive from params - uint32_t diff = (arrived_gap > expected_gap) ? (arrived_gap - expected_gap) - : (expected_gap - arrived_gap); - jitter_est_ = (jitter_est_ * 7 + diff) / 8; - uint32_t depth = std::clamp(jitter_est_ * 2 + 960u, 960u, 48000u * 200u / 1000u); - target_depth_ms_.store(depth * 1000u / 48000u, std::memory_order_relaxed); + int32_t gap = static_cast(ts - last_push_ts_); + bool outlier = marker || gap <= 0 || + gap > static_cast(expected_gap_samples_ * 8); + if (!outlier) { + uint32_t diff = (static_cast(gap) > expected_gap_samples_) + ? (static_cast(gap) - expected_gap_samples_) + : (expected_gap_samples_ - static_cast(gap)); + jitter_est_ = (jitter_est_ * 7 + diff) / 8; + uint32_t depth = std::clamp(jitter_est_ * 2 + expected_gap_samples_, + expected_gap_samples_, 48000u * 200u / 1000u); + target_depth_ms_.store(depth * 1000u / 48000u, std::memory_order_relaxed); + } } last_push_ts_ = ts; first_push_ = false; - buf_.emplace(ts, std::move(f)); + // Track the leading edge (newest timestamp), wrap-safe. + if (!have_newest_ || static_cast(ts - newest_ts_) > 0) newest_ts_ = ts; + have_newest_ = true; + + auto res = buf_.emplace(ts, std::move(f)); + if (!res.second) dup_.fetch_add(1, std::memory_order_relaxed); // duplicate timestamp } std::optional JitterBuffer::pop(uint32_t playout_ts) { @@ -106,8 +126,14 @@ std::optional JitterBuffer::pop(uint32_t playout_ts) { auto it = buf_.begin(); uint32_t ts = it->first; - // Drop frames that are too old (> 500 ms late). - if (static_cast(playout_ts - ts) > static_cast(kLateDropSamples)) { + // Drop frames that are too old to play. The window tracks the (adaptive) target depth plus a + // margin so it never undercuts DRED's next-packet lookahead, floored/capped at the fixed + // 500 ms bound. Without this the only downward force on latency was a snap that *added* it. + constexpr uint32_t kLateMarginSamples = 48000u * 200u / 1000u; // +200 ms over target depth + constexpr uint32_t kLateDropFloor = 48000u * 200u / 1000u; // never drop earlier than 200 ms + uint32_t late_window = std::clamp(target_depth_samples() + kLateMarginSamples, + kLateDropFloor, kLateDropSamples); + if (static_cast(playout_ts - ts) > static_cast(late_window)) { lost_.fetch_add(1, std::memory_order_relaxed); buf_.erase(it); return std::nullopt; @@ -127,6 +153,23 @@ std::optional JitterBuffer::peek_front_ts() const { return buf_.begin()->first; } +std::optional JitterBuffer::peek_back_ts() const { + std::unique_lock lk(mu_, std::try_to_lock); + if (!lk || !have_newest_) return std::nullopt; + return newest_ts_; +} + +void JitterBuffer::drop_before(uint32_t ts) { + std::lock_guard lk(mu_); + while (!buf_.empty()) { + auto it = buf_.begin(); // oldest + if (static_cast(ts - it->first) > 0) + buf_.erase(it); // strictly before the new playout point — stale + else + break; + } +} + size_t JitterBuffer::try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz) { std::unique_lock lk(mu_, std::try_to_lock); if (!lk || buf_.empty()) return 0; @@ -142,7 +185,10 @@ void JitterBuffer::reset() { std::lock_guard lk(mu_); buf_.clear(); lost_.store(0); - first_push_ = true; + dup_.store(0); + first_push_ = true; + have_newest_ = false; + jitter_est_ = 0; } // ── AudioEngine ────────────────────────────────────────────────────────────── @@ -444,6 +490,7 @@ void AudioEngine::push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f) { std::lock_guard lk(streams_mu_); auto& s = streams_[ssrc]; s.last_voice_ms.store(now_ms(), std::memory_order_relaxed); + if (f.marker) s.pending_marker = true; // talkspurt start → force playout reseed s.jitter.push(std::move(f)); } @@ -520,6 +567,29 @@ uint32_t AudioEngine::stream_target_depth_ms(uint32_t ssrc) const { return (it != streams_.end()) ? it->second.jitter.target_depth_ms() : 40; } +uint32_t AudioEngine::stream_duplicates(uint32_t ssrc) const { + std::lock_guard lk(streams_mu_); + auto it = streams_.find(ssrc); + return (it != streams_.end()) ? it->second.jitter.duplicates() : 0; +} + +uint64_t AudioEngine::stream_underruns(uint32_t ssrc) const { + std::lock_guard lk(streams_mu_); + auto it = streams_.find(ssrc); + return (it != streams_.end()) + ? it->second.underruns.load(std::memory_order_relaxed) + : 0; +} + +int32_t AudioEngine::stream_playout_depth_samples(uint32_t ssrc) const { + std::lock_guard lk(streams_mu_); + auto it = streams_.find(ssrc); + if (it == streams_.end() || !it->second.playout_started) return 0; + auto newest = it->second.jitter.peek_back_ts(); + if (!newest) return 0; + return static_cast(*newest - it->second.playout_ts); +} + #ifdef VOICECAT_HAS_OPUS void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p, uint32_t user_id, uint32_t stream_id) { @@ -534,6 +604,9 @@ void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p, int frame_samples = stream.decoder.frame_samples(); if (frame_samples <= 0) frame_samples = static_cast(p.sample_rate / 1000 * p.frame_ms); stream.init_ring(channels, frame_samples); + // The expected inter-arrival gap = the sender's frame size in samples @48 kHz; the jitter + // EWMA and silence-gap outlier rejection key off it (defaults to 20 ms otherwise). + stream.jitter.set_expected_gap(static_cast(frame_samples)); // DRED: pre-allocate per-stream scratch (no RT-thread allocation). 4000 bytes > max Opus pkt. stream.dred_payload_scratch_.assign(4000, 0); if (!stream.dred_state_) { @@ -620,21 +693,31 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) { const int dec_channels = std::max(1, stream.decoder.channels()); const int frame_samples = stream.decoder.frame_samples(); - // Seed / re-sync the playout clock to the arriving stream. playout_ts advances every - // callback (via the PLC path below) independently of whether the sender is transmitting, - // so across a late join or any VAD/PTT silence gap it drifts away from the sender's frame - // timestamps without bound. Left uncorrected, the divergence eventually exceeds the jitter - // buffer's late-drop window and every real frame is dropped-as-late (clock ahead) or - // never-due (clock behind) — permanent silence even though frames keep arriving (the talk - // indicator, driven by push_recv_frame, stays lit). Snap to the earliest buffered frame on - // the first frame and whenever the clock has drifted too far; this both seeds startup and - // recovers after every silence gap. u32 subtraction via int32_t handles timestamp wrap. - if (auto front_ts = stream.jitter.peek_front_ts()) { - int32_t drift = static_cast(stream.playout_ts - *front_ts); - if (!stream.playout_started || drift > kResyncAheadSamples || - drift < -kResyncBehindSamples) { - stream.playout_ts = *front_ts; + // Bound the playout latency against the stream's leading edge (newest buffered frame). + // playout_ts free-runs every callback (PLC path below) while the sender omits silence from + // its timestamps, so the two diverge across late joins and VAD/PTT/DTX gaps. Two cases, + // both measured against the *newest* frame — never the oldest, whose snap could only ever + // *add* standing latency (the ratchet that caused the growing-latency bug): + // • (re)seed to the leading edge when starting, on a talkspurt marker, or when the clock + // has run past the newest frame (starved after silence). depth becomes ~0 — no + // artificial prebuffer, so latency stays as low as the old path; buffered frames still + // play in order (pop() returns oldest-first), they just stop being held back. + // • frame-skip catch-up: if the backlog has grown beyond target+hysteresis (clock drift, + // bursty arrival, reordering), fast-forward to leave exactly `target` buffered and drop + // the now-stale frames. This is the missing downward force that bounds latency. + // u32 subtraction via int32_t handles timestamp wrap. + const int32_t target = + std::max(static_cast(stream.jitter.target_depth_samples()), + kMinDepthSamples); + if (auto newest = stream.jitter.peek_back_ts()) { + int32_t depth = static_cast(*newest - stream.playout_ts); // wrap-safe + if (!stream.playout_started || stream.pending_marker || depth < -kStarveSamples) { + stream.playout_ts = *newest; stream.playout_started = true; + stream.pending_marker = false; + } else if (depth > target + kCatchupSamples) { + stream.playout_ts = *newest - static_cast(target); + stream.jitter.drop_before(stream.playout_ts); } } @@ -713,6 +796,12 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) { stream.playout_ts += static_cast(n); } + // Diagnostic: the decode loop couldn't keep the ring fed for this hardware period while + // the stream was actively playing out — a genuine underrun (decoder error / exhausted + // PLC), distinct from ordinary single-packet loss the loop conceals in place. + if (stream.playout_started && stream.ring_count < frames) + stream.underruns.fetch_add(1, std::memory_order_relaxed); + const float g = stream.gain; int16_t frame_buf[2]; for (ma_uint32 i = 0; i < frames; ++i) { diff --git a/core/src/audio/audio_engine.h b/core/src/audio/audio_engine.h index e08ce56..8a9c9d8 100644 --- a/core/src/audio/audio_engine.h +++ b/core/src/audio/audio_engine.h @@ -46,6 +46,7 @@ class JitterBuffer { uint64_t seq; uint32_t timestamp; bool fec_present; + bool marker = false; // kFlagMarker: first frame of a talkspurt std::vector payload; }; @@ -61,28 +62,49 @@ class JitterBuffer { // AudioEngine::on_playback). Uses try_lock — never blocks the real-time callback. std::optional peek_front_ts() const; + // Timestamp of the newest (latest) buffered frame — the stream's leading edge. Lets the + // playout clock keep a bounded depth behind the arriving stream and catch up (frame-skip) + // when the backlog grows; see AudioEngine::on_playback. try_lock — never blocks the RT thread. + std::optional peek_back_ts() const; + // If the front frame's timestamp == expected_ts, copies its raw Opus payload into out // (caller-allocated, max_sz bytes). Returns bytes copied, or 0 (lock miss / wrong ts / // empty). Caller pre-allocates out to avoid RT-thread allocation. Uses try_lock. size_t try_copy_front_payload(uint32_t expected_ts, uint8_t* out, size_t max_sz); - uint32_t target_depth_ms()const { return target_depth_ms_.load(); } - uint32_t packets_lost() const { return lost_.load(); } + // Erase all buffered frames whose timestamp is strictly before `ts` (oldest-first). Used by + // the playout catch-up so a forward clock jump doesn't flood the decode loop with stale frames. + void drop_before(uint32_t ts); + + uint32_t target_depth_ms() const { return target_depth_ms_.load(); } + uint32_t target_depth_samples() const { return target_depth_ms_.load() * 48; } // @48 kHz + uint32_t packets_lost() const { return lost_.load(); } + uint32_t duplicates() const { return dup_.load(); } + + // Set the expected inter-arrival gap (= sender frame size in samples @48 kHz) so the jitter + // EWMA and silence-gap outlier rejection are correct for non-20 ms channels. Call at init. + void set_expected_gap(uint32_t samples) { + if (samples > 0) expected_gap_samples_ = samples; + } void reset(); private: - static constexpr uint32_t kLateDropSamples = 48000 / 2; // 500 ms @48 kHz + static constexpr uint32_t kLateDropSamples = 48000 / 2; // 500 ms @48 kHz (hard floor/cap) mutable std::mutex mu_; std::map buf_; // keyed by timestamp (u32 wraps are handled below) std::atomic target_depth_ms_{40}; std::atomic lost_{0}; + std::atomic dup_{0}; // Jitter estimation (EWMA). - uint32_t last_push_ts_ = 0; // local clock estimate on last push - uint32_t jitter_est_ = 0; // EWMA jitter in samples - bool first_push_ = true; + uint32_t last_push_ts_ = 0; // sender timestamp on last push + uint32_t jitter_est_ = 0; // EWMA jitter in samples + uint32_t expected_gap_samples_ = 960; // expected inter-arrival gap (frame size @48 kHz) + uint32_t newest_ts_ = 0; // latest buffered timestamp (leading edge) + bool have_newest_ = false; + bool first_push_ = true; }; // ── AudioParams ────────────────────────────────────────────────────────────── @@ -192,6 +214,13 @@ class AudioEngine { // Get stats for a remote stream's jitter buffer. uint32_t stream_packets_lost(uint32_t ssrc) const; uint32_t stream_target_depth_ms(uint32_t ssrc) const; + uint32_t stream_duplicates(uint32_t ssrc) const; + uint64_t stream_underruns(uint32_t ssrc) const; + + // TEST-ONLY: current playout depth in samples (newest buffered ts - playout_ts), i.e. the + // standing latency held in the jitter buffer. Used by test_jitter_depth to assert the + // bounded-depth invariant. Returns 0 if the stream is unknown or not yet playing out. + int32_t stream_playout_depth_samples(uint32_t ssrc) const; #ifdef VOICECAT_HAS_OPUS // Configure the Opus decoder for an incoming ssrc (must be called before @@ -422,6 +451,16 @@ class AudioEngine { // dropped/never-due (silent playback). false until the first frame seeds it (on_playback). bool playout_started = false; + // Set by push_recv_frame when a kFlagMarker (talkspurt-start) frame arrives; consumed by + // on_playback to force an immediate playout-clock reseed at the new talkspurt, so the + // bounded-depth target is re-established cleanly across silence gaps. See on_playback. + bool pending_marker = false; + + // Diagnostic: times the decode/playback ring underran (produced silence because the + // jitter buffer had nothing due) while the stream was actively playing out — i.e. the + // "frames arriving but silent / latency starved" signal. Polled via stream_underruns(). + std::atomic underruns{0}; + // PLC cap (defense-in-depth): consecutive samples produced by packet-loss // concealment since the last real decoded frame. Reset to 0 on every real frame. // When it exceeds kPlcCapSamples (audio_engine.cpp), on_playback stops calling diff --git a/core/src/core/client.cpp b/core/src/core/client.cpp index 88cf91c..55e7462 100644 --- a/core/src/core/client.cpp +++ b/core/src/core/client.cpp @@ -904,6 +904,7 @@ void vc_client::run_udp_recv() { jf.seq = hdr.seq; jf.timestamp = hdr.timestamp; jf.fec_present = (hdr.flags & voicecat::net::kFlagFecPresent) != 0; + jf.marker = (hdr.flags & voicecat::net::kFlagMarker) != 0; jf.payload = std::move(plain); audio_engine_.push_recv_frame(hdr.ssrc, std::move(jf)); } @@ -1097,6 +1098,18 @@ void vc_client::encode_and_send_frame(LocalStream& ls, const int16_t* pcm, int s hdr.timestamp = ls.timestamp; ls.timestamp += static_cast(samples); + // Talkspurt marker: first frame overall, or the first after a transmission gap longer than a + // few frame intervals (VAD/PTT closed, or DTX silence). The sender omits silence from the + // timestamp, so this is how the receiver knows to reseed its playout clock (see on_playback). + const int64_t now_ms = client_now_ms(); + const int64_t frame_ms = std::max(1, samples / 48); // @48 kHz + if (ls.last_send_ms < 0 || (now_ms - ls.last_send_ms) > frame_ms * 3) + hdr.flags |= voicecat::net::kFlagMarker; + ls.last_send_ms = now_ms; + // DTX: Opus emits a 1–2 byte comfort-noise packet when it gates silence. Flag it so the + // receiver can treat it as such (informational; the bounded-depth playout handles timing). + if (opus_len <= 2) hdr.flags |= voicecat::net::kFlagDtx; + uint8_t header_bytes[voicecat::net::kVoiceHeaderSize]; voicecat::net::serialize_header(hdr, header_bytes); diff --git a/core/src/core/client.h b/core/src/core/client.h index 3aa7342..f61b5e9 100644 --- a/core/src/core/client.h +++ b/core/src/core/client.h @@ -240,6 +240,13 @@ struct vc_client { std::atomic last_capture_ms{0}; bool talking = false; + // Talkspurt marker: the sender's `timestamp` omits VAD/PTT/DTX silence, so the receiver + // can't tell a continuation from a post-silence restart. encode_and_send_frame stamps + // kFlagMarker on the first frame after a transmission gap (detected via last_send_ms) so + // the receiver reseeds its playout clock cleanly. -1 = no frame sent yet (first frame is + // always a marker). + int64_t last_send_ms = -1; + // Device-enumeration follow-up: the device this stream's capture should use ("" = // default). Only meaningful for VC_STREAM_MIC today (the real capture device); set via // vc_set_input_device. Opaque id from AudioEngine::enumerate_devices — see diff --git a/docs/voice.md b/docs/voice.md index 5994b28..140eade 100644 --- a/docs/voice.md +++ b/docs/voice.md @@ -143,16 +143,28 @@ Layered, all configurable per channel: ## 5. Jitter buffer -Each receiver keeps an **adaptive jitter buffer per ssrc**. +Each receiver keeps an **adaptive jitter buffer per ssrc** with **bounded-depth playout** +(`core/src/audio/audio_engine.cpp` — `JitterBuffer` + `AudioEngine::on_playback`). - Frames are inserted by `timestamp`; playback reads in order at the device callback rate. -- Target depth adapts to observed network jitter between a configurable **min/max latency** - (channel-level "stability vs latency" knob). A "low-latency" channel runs a shallow - buffer; a "stable" channel runs deeper. -- Late frames past the playout point are dropped; gaps are filled by FEC (if the next frame +- **The playout clock is always bounded against the stream's *leading edge* (newest buffered + frame), never re-synced to the oldest.** The clock free-runs at the playback hardware rate, + while the sender omits VAD/PTT/DTX silence from its timestamps, so the two diverge across gaps + and late joins. Two corrections keep latency bounded: + - **(Re)seed to the leading edge** on first frame, on a talkspurt `marker`, or when the clock + has run past the newest frame (starved after silence). No artificial prebuffer — latency + starts as low as possible; buffered frames still play oldest-first. + - **Frame-skip catch-up:** when the backlog grows past `target + hysteresis` (clock drift, + bursty arrival, reordering), fast-forward the clock to leave `target` buffered and drop the + now-stale frames. This is the downward force that prevents latency from ratcheting upward. +- `target` is the adaptive jitter estimate (EWMA of inter-arrival gap vs. the per-frame gap), + floored; silence gaps and reordered stragglers are rejected as outliers so they don't inflate + it. The late-drop window tracks `target` (floored/capped at 500 ms). +- Late frames past the playout point are dropped; gaps are filled by DRED (if the next frame arrived) or PLC. -- The `marker` flag (start of talkspurt) lets the buffer resynchronize cleanly after - silence/DTX without accumulating drift. +- The `marker` flag (start of talkspurt) — set by the sender on the first frame after a + transmission gap — lets the buffer reseed cleanly after silence/DTX without accumulating drift. +- Diagnostics per stream: `packets_lost`, `duplicates`, `underruns`, `target_depth_ms`. ``` incoming (out of order) ──▶ [ reorder by ts | adaptive depth ] ──▶ Opus decode ──▶ mixer diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f73a540..a2c50ff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,15 @@ if(VOICECAT_USE_VCPKG_DEPS) target_include_directories(test_plc_cap PRIVATE ${VC_TEST_INTERNAL_INCLUDES}) add_test(NAME plc_cap COMMAND test_plc_cap) + # Jitter-buffer bounded depth: across many talkspurt/silence cycles with a compressed sender + # timeline + reordered stragglers, playout latency must stay bounded (no backward drift / + # ratchet). White-box AudioEngine test, no server needed. + add_executable(test_jitter_depth test_jitter_depth.cpp) + target_link_libraries(test_jitter_depth PRIVATE voicecat::voicecat) + target_compile_features(test_jitter_depth PRIVATE cxx_std_20) + target_include_directories(test_jitter_depth PRIVATE ${VC_TEST_INTERNAL_INCLUDES}) + add_test(NAME jitter_depth COMMAND test_jitter_depth) + # External playback (iOS VPIO): the mixer-timer thread drives decode+mix with NO hardware # device and delivers the final mix to the mixed-output sink. White-box AudioEngine test. add_executable(test_external_playback test_external_playback.cpp) diff --git a/tests/test_jitter_depth.cpp b/tests/test_jitter_depth.cpp new file mode 100644 index 0000000..f3458c7 --- /dev/null +++ b/tests/test_jitter_depth.cpp @@ -0,0 +1,144 @@ +/* + * test_jitter_depth — verifies the bounded-depth playout in AudioEngine::on_playback. + * + * Regression guard for the "latency keeps drifting backward, fixed only by rejoining" bug. The + * sender omits VAD/PTT/DTX silence from its timestamps (a compressed timeline), while the + * receiver's playout clock free-runs in real time. The old logic re-synced the playout clock to + * the *oldest* buffered frame and could only ever *add* standing latency (a reordered/late frame + * snapped the clock backward), with nothing to trim it — so latency ratcheted up across talkspurt + * gaps. The fix keeps the clock a bounded `target` behind the *newest* arrival and frame-skips to + * catch up, so depth stays bounded no matter the trigger. + * + * This drives many talkspurt/silence cycles with a compressed timeline plus a reordered straggler + * each cycle (which previously snapped the clock backward), and asserts the buffered depth + * (newest_ts - playout_ts) stays bounded while audio keeps playing. White-box via mix_for_test + * (no audio hardware needed), same pattern as test_plc_cap. + */ +#include +#include +#include +#include + +#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS) + +#include "audio/audio_engine.h" +#include "codec/opus_codec.h" + +static int g_failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL [%s:%d]: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +static double rms(const int16_t* pcm, int n) { + double sum = 0.0; + for (int i = 0; i < n; ++i) sum += static_cast(pcm[i]) * pcm[i]; + return std::sqrt(sum / n); +} + +int main() { + voicecat::audio::AudioEngine engine; + voicecat::audio::AudioParams p; + p.sample_rate = 48000; + p.capture_channels = 1; + p.playback_channels = 2; + p.frame_ms = 20; + CHECK(engine.start(p)); // no capture_cb — headless safe + + voicecat::codec::OpusParams op; + op.sample_rate = 48000; + op.frame_ms = 20; + op.stereo = false; + const int frame_samples = voicecat::codec::opus_frame_samples(op); // 960 + + // A loud sine, encoded once, reused for every pushed frame. + voicecat::codec::OpusEncoder enc; + CHECK(enc.init(op)); + std::vector sine(static_cast(frame_samples)); + for (int i = 0; i < frame_samples; ++i) { + float t = static_cast(i) / 48000.0f; + sine[i] = static_cast(std::sin(2.0f * 3.14159265f * 440.0f * t) * 20000.0f); + } + uint8_t opus_buf[1500]; + const int opus_len = enc.encode(sine.data(), frame_samples, opus_buf, sizeof(opus_buf)); + CHECK(opus_len > 0); + + const uint32_t ssrc = 1; + engine.init_recv_stream(ssrc, op, /*user_id=*/0, /*stream_id=*/0); + + const uint32_t pb_frames = 480; // 10 ms hardware period + const int out_n = static_cast(pb_frames) * 2; // stereo interleaved + std::vector out(static_cast(out_n), 0); + + auto mix_n = [&](int n) { + for (int i = 0; i < n; ++i) engine.mix_for_test(out.data(), pb_frames); + }; + auto push = [&](uint32_t ts, bool marker) { + voicecat::audio::JitterBuffer::Frame f; + f.seq = 0; + f.timestamp = ts; + f.fec_present = false; + f.marker = marker; + f.payload.assign(opus_buf, opus_buf + opus_len); + engine.push_recv_frame(ssrc, std::move(f)); + }; + + uint32_t ts = 1000; // arbitrary non-zero start + int32_t max_depth = 0; + double last_voice_rms = 0.0; + + // Seed the stream (first frame is a talkspurt marker, like a real resume). + push(ts, /*marker=*/true); + ts += static_cast(frame_samples); + mix_n(1); + + // Drive the producer FASTER than the consumer: push one 960-sample frame per step but drain + // only 480 samples (one pb_frames callback) — i.e. arrivals outrun playout by ~480 samples a + // step, exactly the clock-drift / bursty-arrival condition that made latency ratchet up. Also + // inject a reordered straggler periodically (the old backward-snap trigger). The bounded-depth + // catch-up must keep the standing latency from growing without limit. Pre-fix (no catch-up, + // snap-to-oldest) the depth would climb to ~hundreds of frames here. + const int kSteps = 400; + const uint32_t kStraggler = 48000u * 250u / 1000u; // 250 ms behind the leading edge + for (int s = 0; s < kSteps; ++s) { + push(ts, /*marker=*/false); + ts += static_cast(frame_samples); + if (s % 25 == 12) push(ts - kStraggler, /*marker=*/false); // reordered straggler + mix_n(1); // drain only 480 of the 960 produced — producer outruns consumer + int32_t d = engine.stream_playout_depth_samples(ssrc); + if (d > max_depth) max_depth = d; + last_voice_rms = std::max(last_voice_rms, rms(out.data(), out_n)); + } + + std::printf("jitter_depth: max_depth=%d samples (%.0f ms) voice_rms=%.1f\n", max_depth, + static_cast(max_depth) * 1000.0 / 48000.0, last_voice_rms); + + // Bounded: with catch-up the standing latency stays near the adaptive target, well under + // 200 ms even though arrivals outran playout for 400 steps (~4 s of pushed audio). + CHECK(max_depth > 0); // playout ran / depth observed + CHECK(max_depth < static_cast(48000 * 200 / 1000)); // bounded (was unbounded pre-fix) + CHECK(last_voice_rms > 1.0); // audio keeps playing + + engine.remove_stream(ssrc); + engine.stop(); + enc.destroy(); + + if (g_failures == 0) { + std::printf("jitter_depth: all checks passed\n"); + return 0; + } + std::printf("jitter_depth: %d failure(s)\n", g_failures); + return 1; +} + +#else + +int main() { + std::printf("jitter_depth: SKIP (VOICECAT_HAS_AUDIO or VOICECAT_HAS_OPUS not defined)\n"); + return 0; +} + +#endif diff --git a/tests/test_plc_cap.cpp b/tests/test_plc_cap.cpp index a2afc61..3ab773c 100644 --- a/tests/test_plc_cap.cpp +++ b/tests/test_plc_cap.cpp @@ -110,11 +110,14 @@ int main() { } CHECK(energy == 0); // capped PLC = silence - // 5) Resumption: push a fresh real frame — PLC streak resets, audio returns. + // 5) Resumption: push a fresh real frame — PLC streak resets, audio returns. marker=true is + // what the real sender stamps on the first frame after a silence (talkspurt restart); it + // makes the playout clock reseed to this leading edge immediately (no prebuffer delay). voicecat::audio::JitterBuffer::Frame f2; f2.seq = 1; - f2.timestamp = 200000; // far ahead — playout-clock re-sync snaps to it + f2.timestamp = 200000; // far ahead — playout-clock reseeds to it f2.fec_present = false; + f2.marker = true; f2.payload.assign(opus_buf, opus_buf + opus_len); engine.push_recv_frame(ssrc, std::move(f2));