fix(audio): reframe send path to channel frame_ms; pin codec to 48 kHz
The AudioEngine capture clock is fixed at 48 kHz / 20 ms (960-sample frames), but a channel may set any Opus frame_ms (2.5..60 ms, voice.md §3) and the server enforces it unclamped. on_capture_frame handed the engine's 960-sample frame straight to an encoder configured for the channel's window: frame_ms > 20 was silently ignored, and frame_ms < 20 broke entirely (receiver sized its decode buffer too small -> OPUS_BUFFER_TOO_SMALL -> dead audio). Affected the hardware mic and vc_stream_feed_pcm alike. Reframe each captured/fed block to ls.frame_samples via a per-LocalStream accumulator (pre-sized at announce, no RT-thread alloc) before encode_and_send_frame; the 20 ms case stays a zero-copy fast path. Also pin the codec to 48 kHz in opus_params_from_audio_config — it was honoring a non-48k effective sample_rate against a 48 kHz PCM clock. New ctest frame_ms_reframe covers 40 ms (accumulate) and 10 ms (split) feed->encode->relay->decode->sink round trips. ctest --preset dev 25/25. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -944,7 +944,12 @@ int64_t client_now_ms() {
|
||||
// M2 gap where mode/dtx/complexity/application were silently dropped.
|
||||
voicecat::codec::OpusParams opus_params_from_audio_config(const voicecat::v1::AudioConfig& a) {
|
||||
voicecat::codec::OpusParams p;
|
||||
p.sample_rate = a.sample_rate() ? a.sample_rate() : 48000;
|
||||
// Opus always runs at 48 kHz internally (docs/voice.md §3): the whole AudioEngine clock is
|
||||
// 48 kHz and external PCM is fed at 48 kHz, so the codec must match regardless of what a
|
||||
// channel advertises. Honoring a non-48k effective sample_rate here would create an encoder
|
||||
// expecting e.g. 16k PCM while being fed 48k frames — wrong pitch/duration. The wire
|
||||
// sample_rate field mainly tags narrowband intent; Opus handles that via bitrate at 48k.
|
||||
p.sample_rate = 48000;
|
||||
p.bitrate_bps = a.bitrate_bps() ? a.bitrate_bps() : 24000;
|
||||
p.frame_ms = a.frame_ms() ? a.frame_ms() : 20;
|
||||
p.stereo = (a.mode() == voicecat::v1::MODE_STEREO);
|
||||
@@ -1023,6 +1028,45 @@ void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples, int
|
||||
// Updated only after the gate above, so a VAD/PTT-closed frame never shows as "talking".
|
||||
ls.last_capture_ms.store(client_now_ms(), std::memory_order_relaxed);
|
||||
|
||||
// The AudioEngine clock is fixed at 48 kHz / 20 ms, so `samples` is always 960. The encoder,
|
||||
// however, wants ls.frame_samples per call, which the channel's frame_ms can make smaller
|
||||
// (480 @10ms) or larger (1920 @40ms). Reframe to that size before encoding (docs/voice.md §3).
|
||||
const int target = ls.frame_samples;
|
||||
if (samples == target) {
|
||||
// Fast path — the common 20 ms channel: encode the engine frame directly, no buffering.
|
||||
encode_and_send_frame(ls, pcm, samples, channels, fd);
|
||||
return;
|
||||
}
|
||||
|
||||
// Reframe: accumulate engine frames and emit target-sized chunks. encode_accum/upmix_scratch
|
||||
// were pre-sized in handle_stream_announce_result, so no allocation happens here.
|
||||
const int ch = std::max(1, channels);
|
||||
if (ls.accum_channels != ch) { // mono/stereo source change — never mix the two
|
||||
ls.accum_count = 0;
|
||||
ls.accum_channels = ch;
|
||||
}
|
||||
const size_t add = static_cast<size_t>(samples) * static_cast<size_t>(ch);
|
||||
const size_t chunk = static_cast<size_t>(target) * static_cast<size_t>(ch);
|
||||
if (ls.accum_count + add > ls.encode_accum.size()) return; // capacity guard (shouldn't trip)
|
||||
std::memcpy(ls.encode_accum.data() + ls.accum_count, pcm, add * sizeof(int16_t));
|
||||
ls.accum_count += add;
|
||||
|
||||
size_t off = 0;
|
||||
while (ls.accum_count - off >= chunk) {
|
||||
encode_and_send_frame(ls, ls.encode_accum.data() + off, target, ch, fd);
|
||||
off += chunk;
|
||||
}
|
||||
if (off > 0) { // shift the sub-frame remainder to the front
|
||||
const size_t rem = ls.accum_count - off;
|
||||
if (rem > 0)
|
||||
std::memmove(ls.encode_accum.data(), ls.encode_accum.data() + off,
|
||||
rem * sizeof(int16_t));
|
||||
ls.accum_count = rem;
|
||||
}
|
||||
}
|
||||
|
||||
void vc_client::encode_and_send_frame(LocalStream& ls, const int16_t* pcm, int samples,
|
||||
int channels, int fd) {
|
||||
uint8_t opus_buf[1500];
|
||||
int opus_len;
|
||||
if (channels == 2) {
|
||||
@@ -1033,13 +1077,13 @@ void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples, int
|
||||
// Mono capture (mic, or loopback on a mono channel, or test injection) on a channel
|
||||
// configured for stereo — upmix L=R so the stream is still a spec-correct stereo Opus
|
||||
// bitstream. (Mic stays mono in v1 — no stereo capture device — but a stereo channel
|
||||
// requires a stereo bitstream, hence the upmix.)
|
||||
std::vector<int16_t> stereo_pcm(static_cast<size_t>(samples) * 2);
|
||||
// requires a stereo bitstream, hence the upmix.) upmix_scratch is pre-sized at announce.
|
||||
int16_t* st = ls.upmix_scratch.data();
|
||||
for (int i = 0; i < samples; ++i) {
|
||||
stereo_pcm[i * 2] = pcm[i];
|
||||
stereo_pcm[i * 2 + 1] = pcm[i];
|
||||
st[i * 2] = pcm[i];
|
||||
st[i * 2 + 1] = pcm[i];
|
||||
}
|
||||
opus_len = ls.encoder.encode(stereo_pcm.data(), samples, opus_buf, sizeof(opus_buf));
|
||||
opus_len = ls.encoder.encode(st, samples, opus_buf, sizeof(opus_buf));
|
||||
} else {
|
||||
opus_len = ls.encoder.encode(pcm, samples, opus_buf, sizeof(opus_buf));
|
||||
}
|
||||
@@ -1231,6 +1275,18 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
|
||||
return;
|
||||
}
|
||||
|
||||
// Pre-size the reframe buffers used by on_capture_frame when the channel's frame_ms
|
||||
// differs from the engine's 20 ms (frame_samples != 960). Worst case the accumulator
|
||||
// holds one sub-frame remainder (< frame_samples) plus one engine block (960 samples),
|
||||
// interleaved over up to 2 channels — so (frame_samples + 960) * 2. upmix_scratch sizes
|
||||
// one frame_samples chunk as stereo. Allocated here (control thread), never on the RT
|
||||
// audio thread (architecture.md §3).
|
||||
const size_t fs = ls.frame_samples;
|
||||
ls.encode_accum.assign((fs + 960) * 2, 0);
|
||||
ls.upmix_scratch.assign(fs * 2, 0);
|
||||
ls.accum_count = 0;
|
||||
ls.accum_channels = 0;
|
||||
|
||||
ls.active.store(true, std::memory_order_release);
|
||||
self_uid = self_user_id_;
|
||||
emit_stream_id = ls.stream_id;
|
||||
|
||||
@@ -255,6 +255,19 @@ struct vc_client {
|
||||
// core's WASAPI loopback device. Set from vc_stream_desc::external_feed at stream_start
|
||||
// time and checked in handle_stream_announce_result / stream_stop.
|
||||
bool external_feed = false;
|
||||
|
||||
// Reframe buffer: the AudioEngine clock is fixed at 48 kHz / 20 ms, so capture/feed
|
||||
// always delivers 960-sample frames — but the channel's frame_ms (docs/voice.md §3)
|
||||
// can be 2.5…60 ms, so the encoder needs frame_samples per call (480 @10ms, 1920 @40ms,
|
||||
// …). on_capture_frame accumulates the engine's 960-sample frames here and emits
|
||||
// frame_samples-sized chunks. The 20 ms case (frame_samples == 960) bypasses this
|
||||
// entirely (fast path). Pre-sized at announce; never resized on the audio thread
|
||||
// (architecture.md §3 — no RT-thread allocation). encode_accum holds interleaved int16
|
||||
// at accum_channels; upmix_scratch is the pre-sized mono→stereo upmix target.
|
||||
std::vector<int16_t> encode_accum;
|
||||
size_t accum_count = 0; // flat samples currently buffered
|
||||
int accum_channels = 0; // channel count of buffered data; resets on change
|
||||
std::vector<int16_t> upmix_scratch;
|
||||
};
|
||||
mutable std::mutex local_streams_mu_;
|
||||
std::unordered_map<int, LocalStream> local_streams_; // keyed by vc_stream_kind
|
||||
@@ -337,6 +350,12 @@ struct vc_client {
|
||||
// capture_cb passed to audio_engine_.start(): encode + seal + send one frame for the
|
||||
// given local stream `kind` (M3: multiple concurrent local streams are possible).
|
||||
void on_capture_frame(int kind, const int16_t* pcm, int samples, int channels);
|
||||
// Encode one frame of exactly ls.frame_samples samples-per-channel (upmixing mono→stereo
|
||||
// for a stereo channel as needed), seal it, and send it over UDP, advancing ls.timestamp.
|
||||
// Called by on_capture_frame for each frame_samples-sized chunk. Assumes local_streams_mu_
|
||||
// is held and the send gate/crypto checks have already passed.
|
||||
void encode_and_send_frame(LocalStream& ls, const int16_t* pcm, int samples, int channels,
|
||||
int fd);
|
||||
// Inspect a User proto's streams and wire up any new remote ssrc into audio_engine_,
|
||||
// emitting VC_EVENT_STREAM_STARTED/STOPPED as streams appear/disappear.
|
||||
void sync_remote_streams(const voicecat::v1::User& user);
|
||||
|
||||
Reference in New Issue
Block a user