Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: external PCM feed/tap API (vc_stream_feed_pcm + vc_set_pcm_sink)

Browse Source 
Promotes vc_test_inject_capture (mono-only, TEST-ONLY) to a public,
stereo-capable production API and adds a symmetric PCM tap on the
receive side. Enables ReplayKit (iOS), ScreenCaptureKit (macOS), bots,
soundboards, and custom clients — all without a hardware audio device.

Core C++:
- voicecat.h: new vc_stream_feed_pcm, vc_pcm_sink_cb typedef,
  vc_set_pcm_sink; vc_test_inject_capture kept as deprecated alias
- audio_engine: stereo-aware inject_capture (channels param + ring
  reset on channel-count change); atomic pcm_sink_ fired per decoded
  frame in on_playback; RemoteStream carries user_id/stream_id for
  RT-safe sink metadata; init_recv_stream takes user_id+stream_id
- client.cpp: stream_feed_pcm / set_pcm_sink implementations;
  sync_remote_streams passes user_id/stream_id to init_recv_stream
- voicecat.cpp: trampolines + channels=1/2 validation

Tests: test_external_pcm (headless, 3 sub-tests: mono round-trip,
stereo feed L≠R, sink metadata+disable). ctest 23/23.

Swift: feedPcm / setPcmSink in VoiceCatClient.swift + 4 XCTest
smoke tests (ExternalPcmTests.swift).

C#: StreamFeedPcm / SetPcmSink in VoiceCatClient.cs + NativeMethods.cs
(vc_stream_feed_pcm unsafe P/Invoke, VcPcmSinkCallback delegate,
vc_set_pcm_sink via nint) + 4 xUnit smoke tests (ExternalPcmTests.cs).

Docs: architecture.md §4 new subsection, voice.md §9 updated
(macOS/iOS now reference vc_stream_feed_pcm), protocol.md §8 explicit
no-protocol-change note, roadmap.md M5 entry.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-20 17:52:09 +02:00
parent 540ec13a63
commit 615d2a8e5f
21 changed files with 891 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
core/src/audio/audio_engine.cpp
View File

@@ -359,7 +359,11 @@ bool AudioEngine::resume() {
#endif
}
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t samples_per_channel,
int channels) {
const int ch = std::max(1, channels);
const size_t n = samples_per_channel * static_cast<size_t>(ch);
InjectTap* tap;
{
std::lock_guard lk(inject_mu_);
@@ -371,26 +375,38 @@ void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
tap = slot.get();
}
// If the channel count changed, reset the ring to avoid mixing mono and stereo samples.
if (tap->channels != ch) {
tap->write.store(0, std::memory_order_relaxed);
tap->read.store(0, std::memory_order_relaxed);
tap->channels = ch;
}
size_t w = tap->write.load(std::memory_order_relaxed);
for (size_t i = 0; i < n; ++i)
tap->ring[(w + i) % kInjectCapSamples] = pcm[i];
tap->write.store(w + n, std::memory_order_release);
// Fire capture_cb_ for each complete frame now available.
// Fire capture_cb_ for each complete frame (frame_samples_ * ch flat samples).
const size_t frame_flat = static_cast<size_t>(frame_samples_) * static_cast<size_t>(ch);
while (true) {
size_t r = tap->read.load(std::memory_order_relaxed);
size_t avail = tap->write.load(std::memory_order_acquire) - r;
if (avail < static_cast<size_t>(frame_samples_)) break;
if (avail < frame_flat) break;
std::vector<int16_t> frame(frame_samples_);
for (int i = 0; i < frame_samples_; ++i)
std::vector<int16_t> frame(frame_flat);
for (size_t i = 0; i < frame_flat; ++i)
frame[i] = tap->ring[(r + i) % kInjectCapSamples];
tap->read.store(r + frame_samples_, std::memory_order_release);
tap->read.store(r + frame_flat, std::memory_order_release);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_, 1);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_, ch);
}
}
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
inject_capture(kind, pcm, n, 1);
}
void AudioEngine::push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f) {
std::lock_guard lk(streams_mu_);
auto& s = streams_[ssrc];
@@ -472,9 +488,12 @@ uint32_t AudioEngine::stream_target_depth_ms(uint32_t ssrc) const {
}
#ifdef VOICECAT_HAS_OPUS
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p,
uint32_t user_id, uint32_t stream_id) {
std::lock_guard lk(streams_mu_);
auto& stream = streams_[ssrc];
stream.user_id = user_id;
stream.stream_id = stream_id;
stream.decoder.init(p);
// Ring must be sized for this decoder's actual channel/frame-size — see RemoteStream::ring
// comment in audio_engine.h for why this can't just be the playback callback's frame count.
@@ -491,6 +510,11 @@ void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
}
#endif
void AudioEngine::set_pcm_sink(PcmSink cb, void* user) {
pcm_sink_user_.store(user, std::memory_order_relaxed);
pcm_sink_.store(cb, std::memory_order_release);
}
#ifdef VOICECAT_HAS_AUDIO
void AudioEngine::capture_data_cb(ma_device* dev, void* /*out*/,
@@ -634,6 +658,19 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
stream.recv_ns->process_capture(stream.decode_scratch.data(), n,
static_cast<int>(params_.sample_rate));
// PCM sink: deliver decoded per-stream audio to external consumer (bots,
// transcription, recording) before it enters the hardware mix. Atomic relaxed-
// load is safe on the RT thread — the fn-ptr and user-ptr are independent
// pointer-sized values written together by set_pcm_sink (release store).
if (auto sink = pcm_sink_.load(std::memory_order_relaxed)) {
sink(pcm_sink_user_.load(std::memory_order_relaxed),
stream.user_id, stream.stream_id,
stream.decode_scratch.data(),
static_cast<size_t>(n),
static_cast<uint32_t>(dec_channels),
params_.sample_rate);
}
stream.push_ring(stream.decode_scratch.data(), static_cast<size_t>(n));
stream.playout_ts += static_cast<uint32_t>(n);
}

27
core/src/audio/audio_engine.h
View File

@@ -190,10 +190,22 @@ class AudioEngine {
#ifdef VOICECAT_HAS_OPUS
// Configure the Opus decoder for an incoming ssrc (must be called before
// push_recv_frame for that ssrc). Thread-safe.
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p);
// push_recv_frame for that ssrc). user_id/stream_id identify the source for the
// pcm_sink_ callback. Thread-safe.
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p,
uint32_t user_id, uint32_t stream_id);
#endif
// External PCM tap: callback fired once per decoded Opus frame per remote stream, on the
// playback (RT) thread. Matching signature to vc_pcm_sink_cb (cast at the C-ABI boundary).
// Pass nullptr to disable. Thread-safe (atomic store; the RT read is relaxed-load).
using PcmSink = void(*)(void*, uint32_t, uint32_t, const int16_t*, size_t, uint32_t, uint32_t);
void set_pcm_sink(PcmSink cb, void* user);
// External PCM feed overload: stereo-aware variant of inject_capture. samples_per_channel
// is samples per channel; total samples written = samples_per_channel * channels.
void inject_capture(int kind, const int16_t* pcm, size_t samples_per_channel, int channels);
#ifdef VOICECAT_HAS_AUDIO
// TEST-ONLY — exposes the playback mixer without a real ma_device, so tests can verify
// stereo mixing end-to-end (no audio hardware needed). Same logic the real playback
@@ -364,6 +376,7 @@ class AudioEngine {
std::vector<int16_t> ring; // circular, size = kInjectCapSamples
std::atomic<size_t> write{0};
std::atomic<size_t> read{0};
int channels{1}; // channel count last written; resets ring on change
};
std::mutex inject_mu_;
std::unordered_map<int, std::unique_ptr<InjectTap>> inject_taps_;
@@ -405,6 +418,11 @@ class AudioEngine {
#endif
std::vector<uint8_t> dred_payload_scratch_; // pre-sized to 4000 bytes
// Source identity: stored at init_recv_stream() so the pcm_sink_ callback can receive
// (user_id, stream_id) without a separate map lookup from the RT playback thread.
uint32_t user_id = 0;
uint32_t stream_id = 0;
// M3: talk-indicator edge detection (docs/voice.md §7) — updated by push_recv_frame
// (already off the real-time audio thread), polled by poll_talk_transitions().
std::atomic<int64_t> last_voice_ms{0};
@@ -468,6 +486,11 @@ class AudioEngine {
int frame_samples_ = 960; // 20 ms @48 kHz
// External PCM tap: atomic fn-ptr + user-ptr pair. Written by set_pcm_sink (any thread);
// read by on_playback (RT thread) via relaxed load — safe for pointer-sized atomics.
std::atomic<PcmSink> pcm_sink_{nullptr};
std::atomic<void*> pcm_sink_user_{nullptr};
#ifdef VOICECAT_HAS_OPUS
::OpusDREDDecoder* dred_dec_ = nullptr; // shared DRED decoder; null if unsupported
#endif