Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: external PCM feed/tap API (vc_stream_feed_pcm + vc_set_pcm_sink)

Browse Source 
Promotes vc_test_inject_capture (mono-only, TEST-ONLY) to a public,
stereo-capable production API and adds a symmetric PCM tap on the
receive side. Enables ReplayKit (iOS), ScreenCaptureKit (macOS), bots,
soundboards, and custom clients — all without a hardware audio device.

Core C++:
- voicecat.h: new vc_stream_feed_pcm, vc_pcm_sink_cb typedef,
  vc_set_pcm_sink; vc_test_inject_capture kept as deprecated alias
- audio_engine: stereo-aware inject_capture (channels param + ring
  reset on channel-count change); atomic pcm_sink_ fired per decoded
  frame in on_playback; RemoteStream carries user_id/stream_id for
  RT-safe sink metadata; init_recv_stream takes user_id+stream_id
- client.cpp: stream_feed_pcm / set_pcm_sink implementations;
  sync_remote_streams passes user_id/stream_id to init_recv_stream
- voicecat.cpp: trampolines + channels=1/2 validation

Tests: test_external_pcm (headless, 3 sub-tests: mono round-trip,
stereo feed L≠R, sink metadata+disable). ctest 23/23.

Swift: feedPcm / setPcmSink in VoiceCatClient.swift + 4 XCTest
smoke tests (ExternalPcmTests.swift).

C#: StreamFeedPcm / SetPcmSink in VoiceCatClient.cs + NativeMethods.cs
(vc_stream_feed_pcm unsafe P/Invoke, VcPcmSinkCallback delegate,
vc_set_pcm_sink via nint) + 4 xUnit smoke tests (ExternalPcmTests.cs).

Docs: architecture.md §4 new subsection, voice.md §9 updated
(macOS/iOS now reference vc_stream_feed_pcm), protocol.md §8 explicit
no-protocol-change note, roadmap.md M5 entry.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-20 17:52:09 +02:00
parent 540ec13a63
commit 615d2a8e5f
21 changed files with 891 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
core/include/voicecat.h
View File

@@ -413,6 +413,45 @@ VC_API vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const
* VC_ERR_INVALID_ARG if stream_id is unknown or channels is not 1 or 2. */
VC_API vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels);
/* ── External PCM feed/tap ─────────────────────────────────────────────────── */
/* External PCM feed — production-grade API for driving a local stream's encode pipeline
* with caller-supplied PCM instead of (or in addition to) a hardware capture device. The
* stream must already be started (vc_stream_start). The core frames, encodes (Opus), seals
* (AEAD), and sends (UDP) the provided samples exactly as it would mic/loopback audio.
*
* samples_per_channel : samples per channel (e.g. 960 for 20 ms @ 48 kHz).
* channels : 1 (mono) or 2 (stereo interleaved L/R). VC_ERR_INVALID_ARG otherwise.
*
* Use cases: ReplayKit Broadcast Extension (iOS), ScreenCaptureKit (macOS), bots (TTS /
* music / relay), soundboards, DAW integration. Works for any stream kind (MIC /
* SCREEN_AUDIO / AUX_DEVICE). Thread-safe; may be called from any thread.
*
* Replaces vc_test_inject_capture (deprecated alias, see below). */
VC_API vc_result vc_stream_feed_pcm(vc_client* c, uint32_t stream_id,
const int16_t* pcm, size_t samples_per_channel,
uint32_t channels);
/* External PCM tap — receive decoded remote audio as int16 PCM per stream, before it is
* summed into the hardware mix. The callback fires on the audio playback thread once per
* decoded Opus frame (typically every 20 ms) for each active remote stream:
*
* cb(user, user_id, stream_id, pcm, samples_per_channel, channels, sample_rate)
*
* user_id / stream_id : identify the sender (same values as VC_EVENT_STREAM_STARTED).
* pcm : decoded int16 PCM, interleaved when channels == 2.
* samples_per_channel : samples per channel for this frame (typically 960 @ 48 kHz).
* channels : 1 or 2, matching the sender's stream configuration.
* sample_rate : always 48000 in the current implementation.
*
* Pass cb = NULL to disable (default: disabled; hardware playback only).
* The callback MUST NOT block, lock, or allocate — copy what you need and return.
* PCM is still delivered to the hardware playback device regardless (dual output). */
typedef void (*vc_pcm_sink_cb)(void* user, uint32_t user_id, uint32_t stream_id,
const int16_t* pcm, size_t samples_per_channel,
uint32_t channels, uint32_t sample_rate);
VC_API vc_result vc_set_pcm_sink(vc_client* c, vc_pcm_sink_cb cb, void* user);
/* ── Text ─────────────────────────────────────────────────────────────────── */
VC_API vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id,
const char* utf8);

53
core/src/audio/audio_engine.cpp
View File

@@ -359,7 +359,11 @@ bool AudioEngine::resume() {
#endif
}
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t samples_per_channel,
int channels) {
const int ch = std::max(1, channels);
const size_t n = samples_per_channel * static_cast<size_t>(ch);
InjectTap* tap;
{
std::lock_guard lk(inject_mu_);
@@ -371,26 +375,38 @@ void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
tap = slot.get();
}
// If the channel count changed, reset the ring to avoid mixing mono and stereo samples.
if (tap->channels != ch) {
tap->write.store(0, std::memory_order_relaxed);
tap->read.store(0, std::memory_order_relaxed);
tap->channels = ch;
}
size_t w = tap->write.load(std::memory_order_relaxed);
for (size_t i = 0; i < n; ++i)
tap->ring[(w + i) % kInjectCapSamples] = pcm[i];
tap->write.store(w + n, std::memory_order_release);
// Fire capture_cb_ for each complete frame now available.
// Fire capture_cb_ for each complete frame (frame_samples_ * ch flat samples).
const size_t frame_flat = static_cast<size_t>(frame_samples_) * static_cast<size_t>(ch);
while (true) {
size_t r = tap->read.load(std::memory_order_relaxed);
size_t avail = tap->write.load(std::memory_order_acquire) - r;
if (avail < static_cast<size_t>(frame_samples_)) break;
if (avail < frame_flat) break;
std::vector<int16_t> frame(frame_samples_);
for (int i = 0; i < frame_samples_; ++i)
std::vector<int16_t> frame(frame_flat);
for (size_t i = 0; i < frame_flat; ++i)
frame[i] = tap->ring[(r + i) % kInjectCapSamples];
tap->read.store(r + frame_samples_, std::memory_order_release);
tap->read.store(r + frame_flat, std::memory_order_release);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_, 1);
if (capture_cb_) capture_cb_(kind, frame.data(), frame_samples_, ch);
}
}
void AudioEngine::inject_capture(int kind, const int16_t* pcm, size_t n) {
inject_capture(kind, pcm, n, 1);
}
void AudioEngine::push_recv_frame(uint32_t ssrc, JitterBuffer::Frame f) {
std::lock_guard lk(streams_mu_);
auto& s = streams_[ssrc];
@@ -472,9 +488,12 @@ uint32_t AudioEngine::stream_target_depth_ms(uint32_t ssrc) const {
}
#ifdef VOICECAT_HAS_OPUS
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p,
uint32_t user_id, uint32_t stream_id) {
std::lock_guard lk(streams_mu_);
auto& stream = streams_[ssrc];
stream.user_id = user_id;
stream.stream_id = stream_id;
stream.decoder.init(p);
// Ring must be sized for this decoder's actual channel/frame-size — see RemoteStream::ring
// comment in audio_engine.h for why this can't just be the playback callback's frame count.
@@ -491,6 +510,11 @@ void AudioEngine::init_recv_stream(uint32_t ssrc, const codec::OpusParams& p) {
}
#endif
void AudioEngine::set_pcm_sink(PcmSink cb, void* user) {
pcm_sink_user_.store(user, std::memory_order_relaxed);
pcm_sink_.store(cb, std::memory_order_release);
}
#ifdef VOICECAT_HAS_AUDIO
void AudioEngine::capture_data_cb(ma_device* dev, void* /*out*/,
@@ -634,6 +658,19 @@ void AudioEngine::on_playback(int16_t* out, ma_uint32 frames) {
stream.recv_ns->process_capture(stream.decode_scratch.data(), n,
static_cast<int>(params_.sample_rate));
// PCM sink: deliver decoded per-stream audio to external consumer (bots,
// transcription, recording) before it enters the hardware mix. Atomic relaxed-
// load is safe on the RT thread — the fn-ptr and user-ptr are independent
// pointer-sized values written together by set_pcm_sink (release store).
if (auto sink = pcm_sink_.load(std::memory_order_relaxed)) {
sink(pcm_sink_user_.load(std::memory_order_relaxed),
stream.user_id, stream.stream_id,
stream.decode_scratch.data(),
static_cast<size_t>(n),
static_cast<uint32_t>(dec_channels),
params_.sample_rate);
}
stream.push_ring(stream.decode_scratch.data(), static_cast<size_t>(n));
stream.playout_ts += static_cast<uint32_t>(n);
}

27
core/src/audio/audio_engine.h
View File

@@ -190,10 +190,22 @@ class AudioEngine {
#ifdef VOICECAT_HAS_OPUS
// Configure the Opus decoder for an incoming ssrc (must be called before
// push_recv_frame for that ssrc). Thread-safe.
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p);
// push_recv_frame for that ssrc). user_id/stream_id identify the source for the
// pcm_sink_ callback. Thread-safe.
void init_recv_stream(uint32_t ssrc, const codec::OpusParams& p,
uint32_t user_id, uint32_t stream_id);
#endif
// External PCM tap: callback fired once per decoded Opus frame per remote stream, on the
// playback (RT) thread. Matching signature to vc_pcm_sink_cb (cast at the C-ABI boundary).
// Pass nullptr to disable. Thread-safe (atomic store; the RT read is relaxed-load).
using PcmSink = void(*)(void*, uint32_t, uint32_t, const int16_t*, size_t, uint32_t, uint32_t);
void set_pcm_sink(PcmSink cb, void* user);
// External PCM feed overload: stereo-aware variant of inject_capture. samples_per_channel
// is samples per channel; total samples written = samples_per_channel * channels.
void inject_capture(int kind, const int16_t* pcm, size_t samples_per_channel, int channels);
#ifdef VOICECAT_HAS_AUDIO
// TEST-ONLY — exposes the playback mixer without a real ma_device, so tests can verify
// stereo mixing end-to-end (no audio hardware needed). Same logic the real playback
@@ -364,6 +376,7 @@ class AudioEngine {
std::vector<int16_t> ring; // circular, size = kInjectCapSamples
std::atomic<size_t> write{0};
std::atomic<size_t> read{0};
int channels{1}; // channel count last written; resets ring on change
};
std::mutex inject_mu_;
std::unordered_map<int, std::unique_ptr<InjectTap>> inject_taps_;
@@ -405,6 +418,11 @@ class AudioEngine {
#endif
std::vector<uint8_t> dred_payload_scratch_; // pre-sized to 4000 bytes
// Source identity: stored at init_recv_stream() so the pcm_sink_ callback can receive
// (user_id, stream_id) without a separate map lookup from the RT playback thread.
uint32_t user_id = 0;
uint32_t stream_id = 0;
// M3: talk-indicator edge detection (docs/voice.md §7) — updated by push_recv_frame
// (already off the real-time audio thread), polled by poll_talk_transitions().
std::atomic<int64_t> last_voice_ms{0};
@@ -468,6 +486,11 @@ class AudioEngine {
int frame_samples_ = 960; // 20 ms @48 kHz
// External PCM tap: atomic fn-ptr + user-ptr pair. Written by set_pcm_sink (any thread);
// read by on_playback (RT thread) via relaxed load — safe for pointer-sized atomics.
std::atomic<PcmSink> pcm_sink_{nullptr};
std::atomic<void*> pcm_sink_user_{nullptr};
#ifdef VOICECAT_HAS_OPUS
::OpusDREDDecoder* dred_dec_ = nullptr; // shared DRED decoder; null if unsupported
#endif

23
core/src/core/client.cpp
View File

@@ -1109,7 +1109,7 @@ void vc_client::sync_remote_streams(const voicecat::v1::User& user) {
remote_streams_[ssrc] = {user.id(), si.stream_id()};
voicecat::codec::OpusParams p = opus_params_from_audio_config(si.audio());
audio_engine_.init_recv_stream(ssrc, p);
audio_engine_.init_recv_stream(ssrc, p, user.id(), si.stream_id());
bool muted = self_deafened_.load(std::memory_order_acquire) ||
server_deafened_.load(std::memory_order_acquire);
audio_engine_.set_stream_mute(ssrc, muted);
@@ -1490,7 +1490,8 @@ vc_result vc_client::get_stream_audio_config(uint32_t user_id, uint32_t stream_i
return VC_ERR_INVALID_ARG;
}
vc_result vc_client::test_inject_capture(uint32_t stream_id, const int16_t* pcm, size_t samples) {
vc_result vc_client::stream_feed_pcm(uint32_t stream_id, const int16_t* pcm,
size_t samples_per_channel, uint32_t channels) {
int kind = -1;
{
std::lock_guard lk(local_streams_mu_);
@@ -1502,10 +1503,20 @@ vc_result vc_client::test_inject_capture(uint32_t stream_id, const int16_t* pcm,
}
}
if (kind < 0) return VC_ERR_INVALID_ARG;
audio_engine_.inject_capture(kind, pcm, samples);
audio_engine_.inject_capture(kind, pcm, samples_per_channel, static_cast<int>(channels));
return VC_OK;
}
vc_result vc_client::set_pcm_sink(vc_pcm_sink_cb cb, void* user) {
audio_engine_.set_pcm_sink(
reinterpret_cast<voicecat::audio::AudioEngine::PcmSink>(cb), user);
return VC_OK;
}
vc_result vc_client::test_inject_capture(uint32_t stream_id, const int16_t* pcm, size_t samples) {
return stream_feed_pcm(stream_id, pcm, samples, 1);
}
vc_result vc_client::list_devices(vc_device_kind kind, vc_device_list* out) {
#ifdef VOICECAT_HAS_AUDIO
// Works in any connection state — device pickers need to populate pre-connect.
@@ -1876,6 +1887,12 @@ vc_result vc_client::list_devices(vc_device_kind, vc_device_list* out) {
vc_result vc_client::get_stream_audio_config(uint32_t, uint32_t, vc_audio_config*) {
return VC_ERR_NOT_IMPLEMENTED;
}
vc_result vc_client::stream_feed_pcm(uint32_t, const int16_t*, size_t, uint32_t) {
return VC_ERR_NOT_IMPLEMENTED;
}
vc_result vc_client::set_pcm_sink(vc_pcm_sink_cb, void*) {
return VC_ERR_NOT_IMPLEMENTED;
}
vc_result vc_client::test_inject_capture(uint32_t, const int16_t*, size_t) {
return VC_ERR_NOT_IMPLEMENTED;
}

10
core/src/core/client.h
View File

@@ -81,7 +81,15 @@ struct vc_client {
vc_result get_stream_audio_config(uint32_t user_id, uint32_t stream_id,
vc_audio_config* out);
// TEST-ONLY (see voicecat.h) — inject synthetic PCM into a local stream's encode pipeline.
// External PCM feed (see voicecat.h: vc_stream_feed_pcm). Production API for driving a
// local stream's encode pipeline without a hardware capture device. channels = 1 or 2.
vc_result stream_feed_pcm(uint32_t stream_id, const int16_t* pcm,
size_t samples_per_channel, uint32_t channels);
// External PCM sink (see voicecat.h: vc_set_pcm_sink). Delegates to AudioEngine.
vc_result set_pcm_sink(vc_pcm_sink_cb cb, void* user);
// TEST-ONLY (see voicecat.h) — deprecated alias for stream_feed_pcm(..., channels=1).
vc_result test_inject_capture(uint32_t stream_id, const int16_t* pcm, size_t samples);
// M5: moderation & admin.

14
core/src/voicecat.cpp
View File

@@ -143,7 +143,19 @@ vc_result vc_get_stream_audio_config(vc_client* c, uint32_t user_id, uint32_t st
vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const int16_t* pcm,
size_t samples) {
if (c == nullptr || pcm == nullptr) return VC_ERR_INVALID_ARG;
return c->test_inject_capture(stream_id, pcm, samples);
return c->stream_feed_pcm(stream_id, pcm, samples, 1);
}
vc_result vc_stream_feed_pcm(vc_client* c, uint32_t stream_id, const int16_t* pcm,
size_t samples_per_channel, uint32_t channels) {
if (c == nullptr || pcm == nullptr) return VC_ERR_INVALID_ARG;
if (channels != 1 && channels != 2) return VC_ERR_INVALID_ARG;
return c->stream_feed_pcm(stream_id, pcm, samples_per_channel, channels);
}
vc_result vc_set_pcm_sink(vc_client* c, vc_pcm_sink_cb cb, void* user) {
if (c == nullptr) return VC_ERR_INVALID_ARG;
return c->set_pcm_sink(cb, user);
}
vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels) {