feat: device enumeration, VAD/PTT input gate, stereo playback, WASAPI loopback
Closes the three items PROGRESS.md's M3 section explicitly carried forward as out of scope: - Device enumeration (vc_list_devices) + input device selection (vc_set_input_device), backed by AudioEngine::enumerate_devices() via miniaudio's ma_context_get_devices. Device ids are opaque hex-encoded ma_device_id strings. - VAD/PTT send-side input gate (vc_set_input_mode, vc_set_push_to_talk). webrtc-audio-processing (the originally-planned APM) has no working Windows/MSVC build upstream (GCC-only Meson, unfinished MinGW support, hard abseil-cpp dependency), so VAD is a new lightweight, dependency-free energy/RMS processor (EnergyVadProcessor) behind the existing ApmProcessor interface. Gating is MIC-only; SCREEN_AUDIO/AUX_DEVICE always bypass it. - True stereo playback: AudioEngine's mixer and output device now carry stereo end-to-end (mono streams upmix L=R) instead of downmixing decoded stereo streams to mono before mixing. - Real WASAPI loopback capture for SCREEN_AUDIO (Windows-only, via miniaudio's loopback device type), replacing test-only injection as the production capture path. Also: vccli gains --list-devices, --input-device, --input-mode, and --share-screen-audio flags, plus a stdin command loop (ptt on/off, mode vad/ptt) for manual verification. New test_vad_ptt_devices.cpp covers all four items (ABI-level + a white-box AudioEngine stereo-mix check). Docs updated to match: voice.md, roadmap.md (decision-log entry superseding the original webrtc-audio-processing choice), tech-stack.md, README.md, architecture.md, CLAUDE.md, PROGRESS.md. Still explicitly out of scope, documented not silently dropped: real webrtc-audio-processing/AEC (no AEC/NS/AGC exists at all yet), macOS/iOS SCREEN_AUDIO capture, process-specific loopback, and a pre-existing RT-thread rule violation in the capture path that predates this work. Verified: ctest 12/12 green across 3 consecutive full-suite runs (both dev and m1-dev presets build clean); test_vad_ptt_devices passed 5 consecutive standalone runs; manually verified live (vccli --list-devices against real hardware, vccli --voice --input-mode vad streaming without incident). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,18 @@
|
||||
#include "audio/apm_processor.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
|
||||
namespace voicecat::audio {
|
||||
|
||||
namespace {
|
||||
int64_t steady_now_ms() {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// ── ApmPassthrough ────────────────────────────────────────────────────────────
|
||||
// No-op: returns true (VAD always open), does not modify PCM.
|
||||
// Replaced by WebrtcApmProcessor when VOICECAT_HAS_APM is defined.
|
||||
@@ -11,11 +22,47 @@ class ApmPassthrough final : public ApmProcessor {
|
||||
bool process_capture(int16_t*, int, int) override { return true; }
|
||||
};
|
||||
|
||||
// ── EnergyVadProcessor ──────────────────────────────────────────────────────
|
||||
// Lightweight, dependency-free energy/RMS VAD — see apm_processor.h's create_vad() doc comment
|
||||
// for why this exists instead of a real APM. No AEC (process_render is a no-op); doesn't modify
|
||||
// the PCM it's given, only inspects it.
|
||||
class EnergyVadProcessor final : public ApmProcessor {
|
||||
public:
|
||||
EnergyVadProcessor(float rms_threshold, int64_t hang_time_ms)
|
||||
: threshold_(rms_threshold), hang_time_ms_(hang_time_ms) {}
|
||||
|
||||
void process_render(const int16_t*, int, int) override {}
|
||||
|
||||
bool process_capture(int16_t* pcm, int samples, int /*sample_rate*/) override {
|
||||
if (samples > 0) {
|
||||
double sum_sq = 0.0;
|
||||
for (int i = 0; i < samples; ++i) {
|
||||
double s = static_cast<double>(pcm[i]) / 32768.0;
|
||||
sum_sq += s * s;
|
||||
}
|
||||
double rms = std::sqrt(sum_sq / samples);
|
||||
if (rms >= threshold_) last_voice_ms_ = steady_now_ms();
|
||||
}
|
||||
return (steady_now_ms() - last_voice_ms_) < hang_time_ms_;
|
||||
}
|
||||
|
||||
private:
|
||||
float threshold_;
|
||||
int64_t hang_time_ms_;
|
||||
int64_t last_voice_ms_ = 0; // epoch start -> gate begins closed until first loud frame
|
||||
};
|
||||
|
||||
std::unique_ptr<ApmProcessor> ApmProcessor::create() {
|
||||
#ifdef VOICECAT_HAS_APM
|
||||
// TODO(M3): return std::make_unique<WebrtcApmProcessor>();
|
||||
// TODO: return std::make_unique<WebrtcApmProcessor>(); — see create_vad()'s doc comment for
|
||||
// why this isn't wired up yet (no working Windows/MSVC build upstream).
|
||||
#endif
|
||||
return std::make_unique<ApmPassthrough>();
|
||||
}
|
||||
|
||||
std::unique_ptr<ApmProcessor> ApmProcessor::create_vad(float rms_threshold,
|
||||
int64_t hang_time_ms) {
|
||||
return std::make_unique<EnergyVadProcessor>(rms_threshold, hang_time_ms);
|
||||
}
|
||||
|
||||
} // namespace voicecat::audio
|
||||
|
||||
@@ -24,8 +24,24 @@ class ApmProcessor {
|
||||
// Returns false → caller should skip encode/send (silence gate).
|
||||
virtual bool process_capture(int16_t* pcm, int samples, int sample_rate) = 0;
|
||||
|
||||
// Factory: returns a real APM if VOICECAT_HAS_APM is defined, else a passthrough.
|
||||
// Factory: returns a real APM if VOICECAT_HAS_APM is defined, else a passthrough. Used for
|
||||
// recv-side per-stream noise reduction (docs/voice.md §10) — gating doesn't apply there, so
|
||||
// this stays a passthrough until a real APM/NS backend exists (still inert; see
|
||||
// PROGRESS.md). Do not use this for the send-side VAD gate — see create_vad() below.
|
||||
static std::unique_ptr<ApmProcessor> create();
|
||||
|
||||
// Factory for the send-side input gate (docs/voice.md §11): a lightweight, dependency-free
|
||||
// energy/RMS VAD with configurable threshold + hang-time. webrtc-audio-processing (the
|
||||
// originally-planned APM) has no working Windows/MSVC build upstream (GCC-only Meson build,
|
||||
// unfinished MinGW support, hard abseil-cpp dependency — see PROGRESS.md), so this is the
|
||||
// real v1 implementation behind the same ApmProcessor interface, not a passthrough. No AEC
|
||||
// — process_render() is a no-op here; that's a real limitation versus the originally-planned
|
||||
// APM, not just a deferred VAD.
|
||||
// rms_threshold: normalized 0.0-1.0 RMS-of-int16-range; default ~0.025.
|
||||
// hang_time_ms: how long the gate stays open after the last loud frame; default 300 ms
|
||||
// (matches AudioEngine's kTalkHangoverMs so "talking" and "gate open" agree).
|
||||
static std::unique_ptr<ApmProcessor> create_vad(float rms_threshold = 0.025f,
|
||||
int64_t hang_time_ms = 300);
|
||||
};
|
||||
|
||||
} // namespace voicecat::audio
|
||||
|
||||
@@ -17,6 +17,43 @@ int64_t now_ms() {
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// device_id encoding (DeviceInfo::id / AudioParams::*_device_id): a hex string of the raw
|
||||
// ma_device_id bytes. Opaque on purpose — names aren't guaranteed unique, and this is the only
|
||||
// stable handle miniaudio accepts back for device selection. Internal contract only; never
|
||||
// exposed as anything other than an opaque round-tripped string at the C ABI boundary.
|
||||
std::string hex_encode_device_id(const ma_device_id& id) {
|
||||
static constexpr char kHex[] = "0123456789abcdef";
|
||||
const auto* bytes = reinterpret_cast<const uint8_t*>(&id);
|
||||
std::string out;
|
||||
out.reserve(sizeof(ma_device_id) * 2);
|
||||
for (size_t i = 0; i < sizeof(ma_device_id); ++i) {
|
||||
out.push_back(kHex[bytes[i] >> 4]);
|
||||
out.push_back(kHex[bytes[i] & 0xF]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool hex_decode_device_id(const std::string& hex, ma_device_id* out) {
|
||||
if (hex.size() != sizeof(ma_device_id) * 2) return false;
|
||||
std::memset(out, 0, sizeof(ma_device_id));
|
||||
auto* bytes = reinterpret_cast<uint8_t*>(out);
|
||||
auto nibble = [](char c) -> int {
|
||||
if (c >= '0' && c <= '9') return c - '0';
|
||||
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
|
||||
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
|
||||
return -1;
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(ma_device_id); ++i) {
|
||||
int hi = nibble(hex[i * 2]);
|
||||
int lo = nibble(hex[i * 2 + 1]);
|
||||
if (hi < 0 || lo < 0) return false;
|
||||
bytes[i] = static_cast<uint8_t>((hi << 4) | lo);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
} // namespace
|
||||
|
||||
// ── JitterBuffer ─────────────────────────────────────────────────────────────
|
||||
@@ -88,13 +125,17 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// ── Capture device ──────────────────────────────────────────────────────
|
||||
ma_device_id cap_id{};
|
||||
bool have_cap_id = !p.capture_device_id.empty() &&
|
||||
hex_decode_device_id(p.capture_device_id, &cap_id);
|
||||
|
||||
ma_device_config cap_cfg = ma_device_config_init(ma_device_type_capture);
|
||||
cap_cfg.capture.format = ma_format_s16;
|
||||
cap_cfg.capture.channels = p.channels;
|
||||
cap_cfg.capture.channels = p.capture_channels;
|
||||
cap_cfg.sampleRate = p.sample_rate;
|
||||
cap_cfg.dataCallback = capture_data_cb;
|
||||
cap_cfg.pUserData = this;
|
||||
cap_cfg.capture.pDeviceID = nullptr; // always default for now
|
||||
cap_cfg.capture.pDeviceID = have_cap_id ? &cap_id : nullptr; // null = default device
|
||||
|
||||
if (ma_device_init(nullptr, &cap_cfg, &capture_device_) == MA_SUCCESS) {
|
||||
if (ma_device_start(&capture_device_) == MA_SUCCESS) {
|
||||
@@ -105,13 +146,17 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
|
||||
}
|
||||
|
||||
// ── Playback device ─────────────────────────────────────────────────────
|
||||
ma_device_id pb_id{};
|
||||
bool have_pb_id = !p.playback_device_id.empty() &&
|
||||
hex_decode_device_id(p.playback_device_id, &pb_id);
|
||||
|
||||
ma_device_config pb_cfg = ma_device_config_init(ma_device_type_playback);
|
||||
pb_cfg.playback.format = ma_format_s16;
|
||||
pb_cfg.playback.channels = p.channels;
|
||||
pb_cfg.playback.channels = p.playback_channels;
|
||||
pb_cfg.sampleRate = p.sample_rate;
|
||||
pb_cfg.dataCallback = playback_data_cb;
|
||||
pb_cfg.pUserData = this;
|
||||
pb_cfg.playback.pDeviceID = nullptr;
|
||||
pb_cfg.playback.pDeviceID = have_pb_id ? &pb_id : nullptr;
|
||||
|
||||
if (ma_device_init(nullptr, &pb_cfg, &playback_device_) == MA_SUCCESS) {
|
||||
if (ma_device_start(&playback_device_) == MA_SUCCESS) {
|
||||
@@ -119,12 +164,57 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
|
||||
} else {
|
||||
ma_device_uninit(&playback_device_);
|
||||
}
|
||||
} else if (p.playback_channels != 1) {
|
||||
// Fallback: some unusual hardware may not accept the requested channel count even
|
||||
// though WASAPI shared mode normally remixes transparently. Retry once at mono rather
|
||||
// than leaving playback dead.
|
||||
pb_cfg.playback.channels = 1;
|
||||
if (ma_device_init(nullptr, &pb_cfg, &playback_device_) == MA_SUCCESS) {
|
||||
if (ma_device_start(&playback_device_) == MA_SUCCESS) {
|
||||
playback_started_ = true;
|
||||
params_.playback_channels = 1;
|
||||
} else {
|
||||
ma_device_uninit(&playback_device_);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<DeviceInfo> AudioEngine::enumerate_devices(bool capture) {
|
||||
std::vector<DeviceInfo> result;
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
ma_context ctx;
|
||||
if (ma_context_init(nullptr, 0, nullptr, &ctx) != MA_SUCCESS) return result;
|
||||
|
||||
ma_device_info* playback_infos = nullptr;
|
||||
ma_uint32 playback_count = 0;
|
||||
ma_device_info* capture_infos = nullptr;
|
||||
ma_uint32 capture_count = 0;
|
||||
|
||||
if (ma_context_get_devices(&ctx, &playback_infos, &playback_count, &capture_infos,
|
||||
&capture_count) == MA_SUCCESS) {
|
||||
ma_device_info* infos = capture ? capture_infos : playback_infos;
|
||||
ma_uint32 count = capture ? capture_count : playback_count;
|
||||
result.reserve(count);
|
||||
for (ma_uint32 i = 0; i < count; ++i) {
|
||||
DeviceInfo d;
|
||||
d.id = hex_encode_device_id(infos[i].id);
|
||||
d.name = infos[i].name;
|
||||
d.is_default = infos[i].isDefault != 0;
|
||||
result.push_back(std::move(d));
|
||||
}
|
||||
}
|
||||
|
||||
ma_context_uninit(&ctx);
|
||||
#else
|
||||
(void)capture;
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
return result;
|
||||
}
|
||||
|
||||
void AudioEngine::stop() {
|
||||
if (!running_.exchange(false)) return;
|
||||
|
||||
@@ -263,22 +353,25 @@ void AudioEngine::playback_data_cb(ma_device* dev, void* out,
|
||||
}
|
||||
|
||||
void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
|
||||
std::memset(out, 0, frames * params_.channels * sizeof(int16_t));
|
||||
const uint32_t pb_channels = params_.playback_channels;
|
||||
std::memset(out, 0, frames * pb_channels * sizeof(int16_t));
|
||||
|
||||
std::unique_lock lk(streams_mu_, std::try_to_lock);
|
||||
if (!lk) return; // contended: emit silence this period
|
||||
|
||||
#ifdef VOICECAT_HAS_OPUS
|
||||
std::vector<int32_t> mix(frames * params_.channels, 0);
|
||||
std::vector<int32_t> mix(frames * pb_channels, 0);
|
||||
|
||||
for (auto& [ssrc, stream] : streams_) {
|
||||
if (stream.mute || !stream.decoder.valid()) continue;
|
||||
|
||||
// M3: a stream's Opus channel count (mono/stereo, per-channel AudioConfig) may differ
|
||||
// from the engine-wide playback channel count (always mono in M3 — see PROGRESS.md).
|
||||
// A stream's Opus channel count (mono/stereo, per-channel AudioConfig) may differ from
|
||||
// the engine-wide playback channel count (now genuinely stereo — see docs/voice.md §8).
|
||||
// Decode into a buffer sized for the *decoder's* channel count (opus_decode's
|
||||
// frame_size parameter is samples-per-channel, not total samples — pass `frames`,
|
||||
// not `frames * channels`), then convert at this boundary.
|
||||
// not `frames * channels`), then convert at this mix boundary: true stereo decode
|
||||
// output is mixed in directly (no downmix); mono decode output is upmixed L=R (mirrors
|
||||
// the capture-side upmix in vc_client::on_capture_frame).
|
||||
int dec_channels = std::max(1, stream.decoder.channels());
|
||||
auto maybe_frame = stream.jitter.pop(stream.playout_ts);
|
||||
std::vector<int16_t> pcm(frames * static_cast<ma_uint32>(dec_channels));
|
||||
@@ -302,20 +395,26 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
|
||||
|
||||
float g = stream.gain;
|
||||
for (int i = 0; i < n; ++i) {
|
||||
// Downmix decoder output to the engine's mono accumulator if needed
|
||||
// (average L/R); upmix is unnecessary since the mix buffer is per-channel.
|
||||
int32_t sample = (dec_channels == 2)
|
||||
? (static_cast<int32_t>(pcm[i * 2]) +
|
||||
static_cast<int32_t>(pcm[i * 2 + 1])) / 2
|
||||
: static_cast<int32_t>(pcm[i]);
|
||||
for (uint32_t c = 0; c < params_.channels; ++c)
|
||||
mix[i * params_.channels + c] += static_cast<int32_t>(sample * g);
|
||||
if (dec_channels == 2) {
|
||||
int32_t l = static_cast<int32_t>(static_cast<float>(pcm[i * 2]) * g);
|
||||
int32_t r = static_cast<int32_t>(static_cast<float>(pcm[i * 2 + 1]) * g);
|
||||
if (pb_channels == 2) {
|
||||
mix[i * 2] += l;
|
||||
mix[i * 2 + 1] += r;
|
||||
} else {
|
||||
mix[i] += (l + r) / 2; // playback device fell back to mono
|
||||
}
|
||||
} else {
|
||||
int32_t sample = static_cast<int32_t>(static_cast<float>(pcm[i]) * g);
|
||||
for (uint32_t c = 0; c < pb_channels; ++c)
|
||||
mix[i * pb_channels + c] += sample; // upmix mono -> all playback channels
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.playout_ts += frames;
|
||||
}
|
||||
|
||||
for (ma_uint32 i = 0; i < frames * params_.channels; ++i)
|
||||
for (ma_uint32 i = 0; i < frames * pb_channels; ++i)
|
||||
out[i] = static_cast<int16_t>(std::clamp(mix[i], -32768, 32767));
|
||||
#else
|
||||
(void)out;
|
||||
@@ -323,6 +422,62 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef VOICECAT_HAS_LOOPBACK
|
||||
void AudioEngine::loopback_data_cb(ma_device* dev, void* /*out*/, const void* in,
|
||||
ma_uint32 frame_count) {
|
||||
auto* self = static_cast<AudioEngine*>(dev->pUserData);
|
||||
self->on_loopback(static_cast<const int16_t*>(in), frame_count);
|
||||
}
|
||||
|
||||
void AudioEngine::on_loopback(const int16_t* pcm, ma_uint32 frames) {
|
||||
// Same pattern as the real mic capture callback (on_capture) — call capture_cb_ directly,
|
||||
// NOT through inject_capture()'s test-only ring (see audio_engine.h).
|
||||
if (capture_cb_) capture_cb_(loopback_kind_, pcm, static_cast<int>(frames));
|
||||
}
|
||||
|
||||
bool AudioEngine::start_loopback_capture(int kind) {
|
||||
if (loopback_started_) return false; // already running; stop_loopback_capture() first
|
||||
|
||||
ma_device_config cfg = ma_device_config_init(ma_device_type_loopback);
|
||||
cfg.capture.format = ma_format_s16;
|
||||
cfg.capture.channels = 1; // let miniaudio's converter remix from the system's mix format
|
||||
cfg.sampleRate = params_.sample_rate;
|
||||
cfg.dataCallback = loopback_data_cb;
|
||||
cfg.pUserData = this;
|
||||
// pDeviceID left null: captures the default render endpoint (docs/voice.md §9). Windows
|
||||
// 10 2004+ supports process-specific loopback via AUDIOCLIENT_ACTIVATION_PARAMS, but
|
||||
// miniaudio's loopback mode only exposes whole-device capture — a future enhancement, not
|
||||
// this pass. Whole-device loopback inherently captures this app's own incoming voice mix
|
||||
// along with everything else playing — an accepted self-echo-loop characteristic of
|
||||
// desktop-audio capture, not a bug.
|
||||
|
||||
if (ma_device_init(nullptr, &cfg, &loopback_device_) != MA_SUCCESS) return false;
|
||||
if (ma_device_start(&loopback_device_) != MA_SUCCESS) {
|
||||
ma_device_uninit(&loopback_device_);
|
||||
return false;
|
||||
}
|
||||
|
||||
loopback_kind_ = kind;
|
||||
loopback_started_ = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void AudioEngine::stop_loopback_capture() {
|
||||
if (!loopback_started_) return;
|
||||
ma_device_stop(&loopback_device_);
|
||||
ma_device_uninit(&loopback_device_);
|
||||
loopback_started_ = false;
|
||||
}
|
||||
#else // !VOICECAT_HAS_LOOPBACK
|
||||
bool AudioEngine::start_loopback_capture(int /*kind*/) { return false; }
|
||||
void AudioEngine::stop_loopback_capture() {}
|
||||
#endif // VOICECAT_HAS_LOOPBACK
|
||||
|
||||
#endif // VOICECAT_HAS_AUDIO
|
||||
|
||||
#ifndef VOICECAT_HAS_AUDIO
|
||||
bool AudioEngine::start_loopback_capture(int /*kind*/) { return false; }
|
||||
void AudioEngine::stop_loopback_capture() {}
|
||||
#endif
|
||||
|
||||
} // namespace voicecat::audio
|
||||
|
||||
@@ -75,10 +75,21 @@ class JitterBuffer {
|
||||
// ── AudioParams ──────────────────────────────────────────────────────────────
|
||||
struct AudioParams {
|
||||
uint32_t sample_rate = 48000;
|
||||
uint32_t channels = 1;
|
||||
uint32_t capture_channels = 1; // no stereo capture device (mic) in this pass
|
||||
uint32_t playback_channels = 2; // true stereo output (see audio_engine.cpp on_playback)
|
||||
uint32_t frame_ms = 20;
|
||||
std::string capture_device_id; // "" = default
|
||||
std::string playback_device_id; // "" = default
|
||||
std::string capture_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices
|
||||
std::string playback_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices
|
||||
};
|
||||
|
||||
// One enumerated device, returned by AudioEngine::enumerate_devices(). `id` is an internal,
|
||||
// opaque hex-encoded ma_device_id — callers must always round-trip an id that came from
|
||||
// enumerate_devices(); never construct one by hand (names aren't guaranteed unique, so the id
|
||||
// is the only stable handle miniaudio accepts back for device selection).
|
||||
struct DeviceInfo {
|
||||
std::string id;
|
||||
std::string name;
|
||||
bool is_default = false;
|
||||
};
|
||||
|
||||
// ── AudioEngine ──────────────────────────────────────────────────────────────
|
||||
@@ -105,6 +116,18 @@ class AudioEngine {
|
||||
|
||||
bool running() const { return running_.load(std::memory_order_acquire); }
|
||||
|
||||
// Enumerate input or output devices (for UI pickers / vc_list_devices). Static: works
|
||||
// before any AudioEngine instance is running (device pickers need to populate pre-connect).
|
||||
// Inits a throwaway ma_context if VOICECAT_HAS_AUDIO; returns {} otherwise. See DeviceInfo
|
||||
// above for the `id` encoding contract.
|
||||
static std::vector<DeviceInfo> enumerate_devices(bool capture);
|
||||
|
||||
// Real desktop-audio loopback capture (Windows/WASAPI only, VOICECAT_HAS_LOOPBACK). Feeds
|
||||
// `kind`'s capture_cb_ directly, same pattern as the real mic capture device — NOT routed
|
||||
// through inject_capture()'s test-only ring. No-op (returns false) when unsupported.
|
||||
bool start_loopback_capture(int kind);
|
||||
void stop_loopback_capture();
|
||||
|
||||
// Inject synthetic PCM directly into the capture pipeline (bypasses real device).
|
||||
// Thread-safe; can be called from any thread including tests. `kind` selects which local
|
||||
// stream's injection tap to feed (each gets its own ring buffer); the 2-arg overload
|
||||
@@ -139,6 +162,13 @@ class AudioEngine {
|
||||
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p);
|
||||
#endif
|
||||
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// TEST-ONLY — exposes the playback mixer without a real ma_device, so tests can verify
|
||||
// stereo mixing end-to-end (no audio hardware needed). Same logic the real playback
|
||||
// callback uses; safe to call any time after start() (no ma_device touched).
|
||||
void mix_for_test(int16_t* out, uint32_t frames) { on_playback(out, frames); }
|
||||
#endif
|
||||
|
||||
private:
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
static void capture_data_cb(ma_device*, void*, const void*, ma_uint32);
|
||||
@@ -150,6 +180,18 @@ class AudioEngine {
|
||||
ma_device playback_device_{};
|
||||
bool capture_started_ = false;
|
||||
bool playback_started_ = false;
|
||||
|
||||
#ifdef VOICECAT_HAS_LOOPBACK
|
||||
// Desktop-audio loopback capture (SCREEN_AUDIO) — own lifecycle, decoupled from
|
||||
// capture_device_/playback_device_ start/stop (a screen-share can start/stop independently
|
||||
// of the mic and of whether anything is currently playing back).
|
||||
static void loopback_data_cb(ma_device*, void*, const void*, ma_uint32);
|
||||
void on_loopback(const int16_t* pcm, ma_uint32 frames);
|
||||
|
||||
ma_device loopback_device_{};
|
||||
bool loopback_started_ = false;
|
||||
int loopback_kind_ = 0;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
AudioParams params_{};
|
||||
|
||||
@@ -657,11 +657,26 @@ void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples) {
|
||||
// playing while the user's mic is muted (docs §M3 scope decision).
|
||||
if (kind == static_cast<int>(VC_STREAM_MIC) &&
|
||||
self_mic_muted_.load(std::memory_order_acquire)) return;
|
||||
|
||||
// Send-side input gate (docs/voice.md §11) — MIC only. SCREEN_AUDIO/AUX_DEVICE always
|
||||
// bypass this: gating a screen-share on the user's own voice activity would silently drop
|
||||
// shared music/video audio whenever the user isn't talking, which defeats the feature.
|
||||
if (kind == static_cast<int>(VC_STREAM_MIC)) {
|
||||
if (current_input_mode_.load(std::memory_order_acquire) == VC_INPUT_PUSH_TO_TALK) {
|
||||
if (!ptt_active_.load(std::memory_order_acquire)) return; // gate closed
|
||||
} else if (mic_vad_) {
|
||||
// EnergyVadProcessor never writes through the pointer (see apm_processor.cpp); the
|
||||
// const_cast is safe and avoids splitting ApmProcessor's interface just for this.
|
||||
if (!mic_vad_->process_capture(const_cast<int16_t*>(pcm), samples, 48000)) return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!media_send_crypto_) return;
|
||||
int fd = udp_fd_.load(std::memory_order_acquire);
|
||||
if (fd == -1) return;
|
||||
|
||||
auto& ls = it->second;
|
||||
// Updated only after the gate above, so a VAD/PTT-closed frame never shows as "talking".
|
||||
ls.last_capture_ms.store(client_now_ms(), std::memory_order_relaxed);
|
||||
|
||||
uint8_t opus_buf[1500];
|
||||
@@ -711,9 +726,15 @@ void vc_client::on_capture_frame(int kind, const int16_t* pcm, int samples) {
|
||||
void vc_client::ensure_audio_running() {
|
||||
if (audio_engine_.running()) return;
|
||||
voicecat::audio::AudioParams p;
|
||||
p.sample_rate = 48000;
|
||||
p.channels = 1;
|
||||
p.frame_ms = 20;
|
||||
p.sample_rate = 48000;
|
||||
p.capture_channels = 1; // no stereo capture device in this pass
|
||||
p.playback_channels = 2; // true stereo output (audio_engine.cpp on_playback)
|
||||
p.frame_ms = 20;
|
||||
{
|
||||
std::lock_guard lk(local_streams_mu_);
|
||||
auto it = local_streams_.find(static_cast<int>(VC_STREAM_MIC));
|
||||
if (it != local_streams_.end()) p.capture_device_id = it->second.capture_device_id;
|
||||
}
|
||||
audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples) {
|
||||
on_capture_frame(kind, pcm, samples);
|
||||
});
|
||||
@@ -818,11 +839,12 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
|
||||
uint32_t self_uid;
|
||||
uint32_t emit_stream_id;
|
||||
bool ok_to_emit = false;
|
||||
int kind;
|
||||
{
|
||||
std::lock_guard lk(local_streams_mu_);
|
||||
auto pit = pending_announce_kind_.find(req_id);
|
||||
if (pit == pending_announce_kind_.end()) return; // stray/duplicate — ignore
|
||||
int kind = pit->second;
|
||||
kind = pit->second;
|
||||
pending_announce_kind_.erase(pit);
|
||||
|
||||
auto sit = local_streams_.find(kind);
|
||||
@@ -849,9 +871,18 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
|
||||
self_uid = self_user_id_;
|
||||
emit_stream_id = ls.stream_id;
|
||||
ok_to_emit = true;
|
||||
|
||||
// Construct the MIC VAD once, here on io_thread_ (not the RT capture callback) — see
|
||||
// client.h's comment on mic_vad_.
|
||||
if (kind == static_cast<int>(VC_STREAM_MIC) && !mic_vad_) {
|
||||
mic_vad_ = voicecat::audio::ApmProcessor::create_vad();
|
||||
}
|
||||
}
|
||||
|
||||
ensure_audio_running();
|
||||
if (kind == static_cast<int>(VC_STREAM_SCREEN_AUDIO)) {
|
||||
audio_engine_.start_loopback_capture(kind);
|
||||
}
|
||||
|
||||
if (ok_to_emit) {
|
||||
vc_event ev{};
|
||||
@@ -865,14 +896,22 @@ void vc_client::handle_stream_announce_result(uint64_t req_id,
|
||||
vc_result vc_client::stream_stop(uint32_t stream_id) {
|
||||
if (state_net_.load(std::memory_order_acquire) != VC_STATE_CONNECTED) return VC_ERR_NOT_CONNECTED;
|
||||
|
||||
int stopped_kind = -1;
|
||||
{
|
||||
std::lock_guard lk(local_streams_mu_);
|
||||
for (auto& [k, ls] : local_streams_) {
|
||||
if (ls.stream_id == stream_id) { stopped_kind = k; break; }
|
||||
}
|
||||
LocalStream* ls = find_local_stream_by_id(stream_id);
|
||||
if (!ls || !ls->active.load(std::memory_order_acquire)) return VC_ERR_INVALID_ARG;
|
||||
ls->active.store(false, std::memory_order_release);
|
||||
ls->encoder.destroy();
|
||||
}
|
||||
|
||||
if (stopped_kind == static_cast<int>(VC_STREAM_SCREEN_AUDIO)) {
|
||||
audio_engine_.stop_loopback_capture();
|
||||
}
|
||||
|
||||
voicecat::v1::Envelope req;
|
||||
req.set_request_id(next_req_id_++);
|
||||
req.mutable_stream_stop()->set_stream_id(stream_id);
|
||||
@@ -894,9 +933,38 @@ vc_client::LocalStream* vc_client::find_local_stream_by_id(uint32_t stream_id) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
vc_result vc_client::set_input_device(uint32_t, const char*) { return VC_ERR_NOT_IMPLEMENTED; }
|
||||
vc_result vc_client::set_input_mode(vc_input_mode) { return VC_ERR_NOT_IMPLEMENTED; }
|
||||
vc_result vc_client::set_push_to_talk(bool) { return VC_ERR_NOT_IMPLEMENTED; }
|
||||
vc_result vc_client::set_input_device(uint32_t stream_id, const char* device_id) {
|
||||
int kind = -1;
|
||||
{
|
||||
std::lock_guard lk(local_streams_mu_);
|
||||
LocalStream* ls = find_local_stream_by_id(stream_id);
|
||||
if (!ls) return VC_ERR_INVALID_ARG;
|
||||
ls->capture_device_id = device_id ? device_id : "";
|
||||
for (auto& [k, entry] : local_streams_) {
|
||||
if (&entry == ls) { kind = k; break; }
|
||||
}
|
||||
}
|
||||
// Only the real capture device (MIC) is affected by device selection — SCREEN_AUDIO uses
|
||||
// loopback capture (no input device to pick) and AUX_DEVICE isn't backed by a real device
|
||||
// path yet. Restart the engine unconditionally when it's already running so the new device
|
||||
// id takes effect; AudioEngine doesn't expose a getter for "is this the same device" so we
|
||||
// don't try to skip the restart when it happens to be a no-op change.
|
||||
if (kind == static_cast<int>(VC_STREAM_MIC) && audio_engine_.running()) {
|
||||
audio_engine_.stop();
|
||||
ensure_audio_running();
|
||||
}
|
||||
return VC_OK;
|
||||
}
|
||||
|
||||
vc_result vc_client::set_input_mode(vc_input_mode mode) {
|
||||
current_input_mode_.store(mode, std::memory_order_release);
|
||||
return VC_OK;
|
||||
}
|
||||
|
||||
vc_result vc_client::set_push_to_talk(bool active) {
|
||||
ptt_active_.store(active, std::memory_order_release);
|
||||
return VC_OK;
|
||||
}
|
||||
|
||||
vc_result vc_client::set_self_mute(bool mic_muted, bool deafened) {
|
||||
if (state_net_.load(std::memory_order_acquire) != VC_STATE_CONNECTED) return VC_ERR_NOT_CONNECTED;
|
||||
@@ -984,10 +1052,31 @@ vc_result vc_client::test_inject_capture(uint32_t stream_id, const int16_t* pcm,
|
||||
return VC_OK;
|
||||
}
|
||||
|
||||
vc_result vc_client::list_devices(vc_device_kind, vc_device_list* out) {
|
||||
vc_result vc_client::list_devices(vc_device_kind kind, vc_device_list* out) {
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
// Works in any connection state — device pickers need to populate pre-connect.
|
||||
auto devices = voicecat::audio::AudioEngine::enumerate_devices(kind == VC_DEVICE_INPUT);
|
||||
|
||||
auto* items = new vc_device[devices.size()];
|
||||
for (size_t i = 0; i < devices.size(); ++i) {
|
||||
const auto& d = devices[i];
|
||||
auto* id = new char[d.id.size() + 1];
|
||||
auto* name = new char[d.name.size() + 1];
|
||||
std::memcpy(id, d.id.c_str(), d.id.size() + 1);
|
||||
std::memcpy(name, d.name.c_str(), d.name.size() + 1);
|
||||
items[i].id = id;
|
||||
items[i].name = name;
|
||||
items[i].is_default = d.is_default ? 1 : 0;
|
||||
}
|
||||
out->items = items;
|
||||
out->count = devices.size();
|
||||
return VC_OK;
|
||||
#else
|
||||
(void)kind;
|
||||
out->items = nullptr;
|
||||
out->count = 0;
|
||||
return VC_ERR_NOT_IMPLEMENTED;
|
||||
#endif
|
||||
}
|
||||
|
||||
void vc_client::run_talk_timer() {
|
||||
|
||||
@@ -147,9 +147,16 @@ struct vc_client {
|
||||
// vc_get_stream_audio_config() has something to read back for our own streams.
|
||||
voicecat::codec::OpusParams effective_params;
|
||||
|
||||
// Talk-indicator edge detection (docs/voice.md §7) — updated in on_capture_frame.
|
||||
// Talk-indicator edge detection (docs/voice.md §7) — updated in on_capture_frame, after
|
||||
// the VAD/PTT gate so a gated-closed frame doesn't show as "talking".
|
||||
std::atomic<int64_t> last_capture_ms{0};
|
||||
bool talking = false;
|
||||
|
||||
// Device-enumeration follow-up: the device this stream's capture should use ("" =
|
||||
// default). Only meaningful for VC_STREAM_MIC today (the real capture device); set via
|
||||
// vc_set_input_device. Opaque id from AudioEngine::enumerate_devices — see
|
||||
// audio_engine.h's DeviceInfo doc comment.
|
||||
std::string capture_device_id;
|
||||
};
|
||||
mutable std::mutex local_streams_mu_;
|
||||
std::unordered_map<int, LocalStream> local_streams_; // keyed by vc_stream_kind
|
||||
@@ -174,6 +181,14 @@ struct vc_client {
|
||||
std::atomic<bool> self_mic_muted_{false};
|
||||
std::atomic<bool> self_deafened_{false};
|
||||
|
||||
// Follow-up to M3: send-side input gate (docs/voice.md §11). MIC-only — SCREEN_AUDIO/
|
||||
// AUX_DEVICE are never gated (see PROGRESS.md for the rationale). Pure local state, no
|
||||
// protocol traffic. mic_vad_ is constructed once the MIC stream's StreamAnnounceResult
|
||||
// lands (handle_stream_announce_result, on io_thread_ — not the RT capture callback).
|
||||
std::atomic<vc_input_mode> current_input_mode_{VC_INPUT_VOICE_ACTIVATION};
|
||||
std::atomic<bool> ptt_active_{false};
|
||||
std::unique_ptr<voicecat::audio::ApmProcessor> mic_vad_;
|
||||
|
||||
// teardown_voice() is called both from run_io()'s own cleanup (on the io_thread_, when
|
||||
// the read loop exits) and from disconnect() (on the caller's thread) -- without
|
||||
// serializing those two call sites, both can see udp_thread_/talk_timer_thread_ as
|
||||
|
||||
@@ -140,8 +140,12 @@ vc_result vc_list_devices(vc_client* c, vc_device_kind kind, vc_device_list* out
|
||||
}
|
||||
|
||||
void vc_free_device_list(vc_device_list* list) {
|
||||
if (list == nullptr) return;
|
||||
/* Stub: no allocation yet. Real impl frees list->items here. */
|
||||
if (list == nullptr || list->items == nullptr) return;
|
||||
for (size_t i = 0; i < list->count; ++i) {
|
||||
delete[] list->items[i].id;
|
||||
delete[] list->items[i].name;
|
||||
}
|
||||
delete[] list->items;
|
||||
list->items = nullptr;
|
||||
list->count = 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user