Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(audio): stereo screen-audio loopback capture on Windows

Browse Source 
start_loopback_capture hardcoded channels=1, forcing miniaudio to downmix the system's stereo mix to mono before the encoder saw it -- on_capture_frame then upmixed L=R to produce fake stereo. Now the loopback device opens in the channel's mode (stereo when the channel is stereo), CaptureCallback carries an explicit channels param so the encoder gets real interleaved L/R, and a mono fallback covers unusual render endpoints. New test_loopback_stereo_capture asserts L!=R end-to-end; 18/18 ctest green.
This commit is contained in:
Talon
2026-06-17 23:27:59 +02:00
parent a88656f2fa
commit cccf085a87
7 changed files with 258 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
core/src/audio/audio_engine.cpp
View File

@@ -140,7 +140,11 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
#ifdef VOICECAT_HAS_AUDIO
// Pre-allocate capture accumulators before the devices start so on_capture / on_loopback
// never allocate on the RT thread. count=0 means "empty"; the buf is sized to exactly one
// encoder frame so a memcpy into it can never overrun.
// encoder frame so a memcpy into it can never overrun. The mic accumulator is mono
// (params_.capture_channels, always 1 in v1 — no stereo mic). The loopback accumulator is
// sized mono here as a safe default and re-sized to frame_samples_*channels in
// start_loopback_capture() once the screen stream's channel mode is known (off the RT
// thread, before the loopback device is started).
capture_accum_.buf.assign(static_cast<size_t>(frame_samples_), 0);
capture_accum_.count = 0;
loopback_accum_.buf.assign(static_cast<size_t>(frame_samples_), 0);
@@ -288,7 +292,7 @@ void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
frame[i] = tap->ring[(r + i) % kInjectCapSamples];
tap->read.store(r + frame_samples_, std::memory_order_release);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_, 1);
}
}
@@ -391,7 +395,8 @@ void AudioEngine::on_capture(const int16_t* pcm, ma_uint32 frames) {
src += copy;
remaining -= copy;
if (capture_accum_.count == frame_samples_) {
capture_cb_(0, capture_accum_.buf.data(), frame_samples_);
capture_cb_(0, capture_accum_.buf.data(), frame_samples_,
static_cast<int>(params_.capture_channels));
capture_accum_.count = 0;
}
}
@@ -507,32 +512,39 @@ void AudioEngine::loopback_data_cb(ma_device* dev, void* /*out*/, const void* in
void AudioEngine::on_loopback(const int16_t* pcm, ma_uint32 frames) {
// Same accumulation as on_capture — the WASAPI loopback render endpoint's callback
// period is also hardware-driven and may not match frame_samples_.
// period is also hardware-driven and may not match frame_samples_. PCM here is
// interleaved across loopback_channels_ (1 = mono downmix, 2 = stereo L/R) — the
// accumulator was sized to frame_samples_*loopback_channels_ in start_loopback_capture,
// so a memcpy into it can never overrun. capture_cb_ receives samples-per-channel
// (frame_samples_) and the channel count explicitly.
if (!capture_cb_ || frame_samples_ <= 0) return;
const int ch = std::max(1, loopback_channels_);
const int16_t* src = pcm;
auto remaining = static_cast<int>(frames);
auto remaining = static_cast<int>(frames) * ch;
const int full = frame_samples_ * ch;
while (remaining > 0) {
int space = frame_samples_ - loopback_accum_.count;
int space = full - loopback_accum_.count;
int copy = std::min(remaining, space);
std::memcpy(loopback_accum_.buf.data() + loopback_accum_.count, src,
static_cast<size_t>(copy) * sizeof(int16_t));
loopback_accum_.count += copy;
src += copy;
remaining -= copy;
if (loopback_accum_.count == frame_samples_) {
capture_cb_(loopback_kind_, loopback_accum_.buf.data(), frame_samples_);
if (loopback_accum_.count == full) {
capture_cb_(loopback_kind_, loopback_accum_.buf.data(), frame_samples_, ch);
loopback_accum_.count = 0;
}
}
}
bool AudioEngine::start_loopback_capture(int kind) {
bool AudioEngine::start_loopback_capture(int kind, int channels) {
if (loopback_started_) return false; // already running; stop_loopback_capture() first
int ch = std::max(1, channels);
loopback_accum_.count = 0; // discard any partial frame from a previous loopback session
ma_device_config cfg = ma_device_config_init(ma_device_type_loopback);
cfg.capture.format = ma_format_s16;
cfg.capture.channels = 1; // let miniaudio's converter remix from the system's mix format
cfg.capture.channels = static_cast<ma_uint32>(ch);
cfg.sampleRate = params_.sample_rate;
cfg.dataCallback = loopback_data_cb;
cfg.pUserData = this;
@@ -543,14 +555,31 @@ bool AudioEngine::start_loopback_capture(int kind) {
// along with everything else playing — an accepted self-echo-loop characteristic of
// desktop-audio capture, not a bug.
if (ma_device_init(nullptr, &cfg, &loopback_device_) != MA_SUCCESS) return false;
if (ma_device_init(nullptr, &cfg, &loopback_device_) != MA_SUCCESS) {
// Fallback: some unusual render endpoints may reject channels=2 even though WASAPI
// shared mode normally remixes transparently. Retry once at mono (mirror the playback
// device's fallback in start()) rather than leaving screen-audio capture dead.
if (ch != 1) {
ch = 1;
cfg.capture.channels = 1;
if (ma_device_init(nullptr, &cfg, &loopback_device_) != MA_SUCCESS) return false;
} else {
return false;
}
}
if (ma_device_start(&loopback_device_) != MA_SUCCESS) {
ma_device_uninit(&loopback_device_);
return false;
}
loopback_kind_ = kind;
loopback_started_ = true;
// Size the accumulator to one full encoder frame at the actual opened channel count.
// Done here (off the RT thread) before any callback can fire — satisfies architecture.md
// §3 (no allocation on RT threads). ma_device_start() above is what arms the callback.
loopback_channels_ = ch;
loopback_accum_.buf.assign(static_cast<size_t>(frame_samples_ * ch), 0);
loopback_accum_.count = 0;
loopback_kind_ = kind;
loopback_started_ = true;
return true;
}
@@ -561,14 +590,14 @@ void AudioEngine::stop_loopback_capture() {
loopback_started_ = false;
}
#else // !VOICECAT_HAS_LOOPBACK
bool AudioEngine::start_loopback_capture(int /*kind*/) { return false; }
bool AudioEngine::start_loopback_capture(int /*kind*/, int /*channels*/) { return false; }
void AudioEngine::stop_loopback_capture() {}
#endif // VOICECAT_HAS_LOOPBACK
#endif // VOICECAT_HAS_AUDIO
#ifndef VOICECAT_HAS_AUDIO
bool AudioEngine::start_loopback_capture(int /*kind*/) { return false; }
bool AudioEngine::start_loopback_capture(int /*kind*/, int /*channels*/) { return false; }
void AudioEngine::stop_loopback_capture() {}
#endif

61
core/src/audio/audio_engine.h
View File

@@ -105,10 +105,14 @@ class AudioEngine {
public:
// Callback type for encoded capture frames ready to be sent. `kind` identifies which
// local stream this PCM belongs to (a vc_stream_kind value; 0 = MIC for the real capture
// device, which is always the "primary" tap). M3: multiple concurrent local streams are
// possible (e.g. MIC + SCREEN_AUDIO), each fed via its own injection tap (see
// inject_capture) since there is only one real hardware capture device.
using CaptureCallback = std::function<void(int kind, const int16_t* pcm, int samples)>;
// device, which is always the "primary" tap). `channels` is the channel count of the PCM
// buffer (1 = mono, 2 = stereo interleaved) — the mic capture device is mono in v1, but
// the WASAPI loopback path (SCREEN_AUDIO) captures in the channel's mode when stereo, so
// the encoder sees real interleaved L/R PCM rather than a mono upmix. M3: multiple
// concurrent local streams are possible (e.g. MIC + SCREEN_AUDIO), each fed via its own
// injection tap (see inject_capture) since there is only one real hardware capture device.
using CaptureCallback = std::function<void(int kind, const int16_t* pcm, int samples,
int channels)>;
AudioEngine();
~AudioEngine();
@@ -131,8 +135,11 @@ class AudioEngine {
// Real desktop-audio loopback capture (Windows/WASAPI only, VOICECAT_HAS_LOOPBACK). Feeds
// `kind`'s capture_cb_ directly, same pattern as the real mic capture device — NOT routed
// through inject_capture()'s test-only ring. No-op (returns false) when unsupported.
bool start_loopback_capture(int kind);
// through inject_capture()'s test-only ring. `channels` is the channel count to open the
// loopback device with (1 = mono downmix of the system mix, 2 = stereo capture when the
// channel is configured stereo); the accumulator and capture_cb_ invocation are shaped to
// match. No-op (returns false) when unsupported.
bool start_loopback_capture(int kind, int channels);
void stop_loopback_capture();
// Inject synthetic PCM directly into the capture pipeline (bypasses real device).
@@ -193,13 +200,52 @@ class AudioEngine {
src += copy;
remaining -= copy;
if (capture_accum_.count == frame_samples_) {
cb(0, capture_accum_.buf.data(), frame_samples_);
cb(0, capture_accum_.buf.data(), frame_samples_, 1);
capture_accum_.count = 0;
}
}
}
#endif
#ifdef VOICECAT_HAS_LOOPBACK
// TEST-ONLY — drives the loopback accumulator directly with an explicit callback, the
// loopback analogue of feed_capture_for_test. Self-contained: sizes the accumulator and
// sets loopback_channels_ itself, so it works on headless CI where start_loopback_capture
// can't init a real WASAPI device. `channels` selects mono (1) or interleaved stereo (2).
// PCM is interleaved L/R when channels==2. Invokes `cb` once per full frame_samples_
// per-channel chunk, with the channel count passed through so the encoder branch in
// on_capture_frame sees real stereo (channels==2) rather than a mono upmix.
void feed_loopback_for_test(const int16_t* pcm, int frames_per_channel, int channels,
const CaptureCallback& cb) {
if (frame_samples_ <= 0 || !cb) return;
const int ch = std::max(1, channels);
const int full = frame_samples_ * ch;
// Size the accumulator for the requested channel count (off the RT thread; this is a
// test-only path). start_loopback_capture() does the same sizing when it opens a real
// device, but on headless CI that init fails — so do it here too.
if (static_cast<int>(loopback_accum_.buf.size()) != full) {
loopback_accum_.buf.assign(static_cast<size_t>(full), 0);
loopback_accum_.count = 0;
}
loopback_channels_ = ch;
const int16_t* src = pcm;
auto remaining = frames_per_channel * ch;
while (remaining > 0) {
int space = full - loopback_accum_.count;
int copy = std::min(remaining, space);
std::memcpy(loopback_accum_.buf.data() + loopback_accum_.count, src,
static_cast<size_t>(copy) * sizeof(int16_t));
loopback_accum_.count += copy;
src += copy;
remaining -= copy;
if (loopback_accum_.count == full) {
cb(loopback_kind_, loopback_accum_.buf.data(), frame_samples_, ch);
loopback_accum_.count = 0;
}
}
}
#endif
private:
#ifdef VOICECAT_HAS_AUDIO
static void capture_data_cb(ma_device*, void*, const void*, ma_uint32);
@@ -222,6 +268,7 @@ class AudioEngine {
ma_device loopback_device_{};
bool loopback_started_ = false;
int loopback_kind_ = 0;
int loopback_channels_ = 1; // channel count the loopback device was opened with
#endif
#endif

28
core/src/core/client.cpp
View File

@@ -852,7 +852,7 @@ voicecat::v1::Channel channel_from_vc(const vc_channel_info& c) {
} // namespace
void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples) {
void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples, int channels) {
std::lock_guard lk(local_streams_mu_);
auto it = local_streams_.find(kind);
if (it == local_streams_.end() || !it->second.active.load(std::memory_order_acquire)) return;
@@ -888,9 +888,15 @@ void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples) {
uint8_t opus_buf[1500];
int opus_len;
if (ls.effective_params.stereo) {
// Capture is always mono in M3 (no stereo capture device); upmix L=R so a
// channel configured for stereo still gets a real, spec-correct stereo Opus stream.
if (channels == 2) {
// Real interleaved stereo PCM (SCREEN_AUDIO loopback on a stereo channel) — encode
// directly, no upmix. `samples` is samples-per-channel, as OpusEncoder::encode expects.
opus_len = ls.encoder.encode(pcm, samples, opus_buf, sizeof(opus_buf));
} else if (ls.effective_params.stereo) {
// Mono capture (mic, or loopback on a mono channel, or test injection) on a channel
// configured for stereo — upmix L=R so the stream is still a spec-correct stereo Opus
// bitstream. (Mic stays mono in v1 — no stereo capture device — but a stereo channel
// requires a stereo bitstream, hence the upmix.)
std::vector<int16_t> stereo_pcm(static_cast<size_t>(samples) * 2);
for (int i = 0; i < samples; ++i) {
stereo_pcm[i * 2] = pcm[i];
@@ -942,8 +948,8 @@ void vc_client::ensure_audio_running() {
auto it = local_streams_.find(static_cast<int>(VC_STREAM_MIC));
if (it != local_streams_.end()) p.capture_device_id = it->second.capture_device_id;
}
audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples) {
on_capture_frame(kind, pcm, samples);
audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples, int channels) {
on_capture_frame(kind, pcm, samples, channels);
});
}
@@ -1049,6 +1055,7 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
uint32_t emit_stream_id;
bool ok_to_emit = false;
int kind;
int loopback_channels = 1; // only meaningful for SCREEN_AUDIO; set under the lock
{
std::lock_guard lk(local_streams_mu_);
auto pit = pending_announce_kind_.find(req_id);
@@ -1087,11 +1094,18 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
mic_vad_ = voicecat::audio::ApmProcessor::create_vad(
vad_threshold_.load(std::memory_order_relaxed));
}
// SCREEN_AUDIO loopback opens the WASAPI device in the channel's mode: stereo capture
// when the channel is stereo (real L/R, no downmix), mono otherwise. Captured under
// the lock alongside the rest of the LocalStream setup; used below after unlock.
if (kind == static_cast<int>(VC_STREAM_SCREEN_AUDIO)) {
loopback_channels = ls.effective_params.stereo ? 2 : 1;
}
}
ensure_audio_running();
if (kind == static_cast<int>(VC_STREAM_SCREEN_AUDIO)) {
audio_engine_.start_loopback_capture(kind);
audio_engine_.start_loopback_capture(kind, loopback_channels);
}
if (ok_to_emit) {

2
core/src/core/client.h
View File

@@ -272,7 +272,7 @@ struct vc_client {
void run_udp_recv();
// capture_cb passed to audio_engine_.start(): encode + seal + send one frame for the
// given local stream `kind` (M3: multiple concurrent local streams are possible).
void on_capture_frame(int kind, const int16_t* pcm, int samples);
void on_capture_frame(int kind, const int16_t* pcm, int samples, int channels);
// Inspect a User proto's streams and wire up any new remote ssrc into audio_engine_,
// emitting VC_EVENT_STREAM_STARTED/STOPPED as streams appear/disappear.
void sync_remote_streams(const voicecat::v1::User& user);