Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ios): audio overhaul, Join/Leave Voice, channel-id sync fix, stereo mic capture

Browse Source 
Three iOS client problems fixed plus a new core stereo-mic capture ABI:

1. Channel-id sync bug (mic button permanently dimmed): SessionState never
   synced currentChannelId from the self user's channelId on connect, so the
   mic button (gated on currentChannelId == 0) stayed dimmed. Added
   syncSelfChannel() (mirrors macOS MainWindowController.swift:461,491,522);
   called from init/.channelList/.userJoined/.userLeft/.userUpdated/.joinResult.
   Added applyServerMuteState() + serverMuted/serverDeafened to VoiceState.

2. Join/Leave Voice button: replaced icon-only mic toggle with explicit
   text button (parity with macOS). Mute/deafen disable when not in voice.

3. IOSAudioRouter.swift (new): full AVAudioSession routing layer — input
   port selection, built-in mic orientation/polar patterns, Bluetooth
   HFP/A2DP/Off modes, Standard/Raw mic processing, stereo capture, AirPlay,
   UserDefaults persistence. AudioSessionManager delegates to it.

4. Core stereo-mic capture (append-only ABI): vc_set_capture_channels()
   lets the core open the mic device in stereo (2-ch interleaved). LocalStream
   gains capture_channels; ensure_audio_running reads it; audio_engine.cpp
   capture_accum_ + on_capture updated to channel-aware accumulation. Test
   test_stereo_mic_capture (headless, L!=R stereo round-trip). Swift wrapper
   VoiceCatClient.setCaptureChannels.

5. Settings UI rework: AVAudioSession-derived input/output tree replaces
   miniaudio device picker.

6. iOS deployment target raised to 18.0 (Package.swift + project.pbxproj).
   swift-tools-version 6.0 with swiftLanguageModes .v5.

Docs: tech-stack.md, architecture.md, voice.md, roadmap.md, building.md
updated; stale 'vc_audio_suspend/resume deferred' claims corrected.

Verified: ctest --preset dev 21/21 green; swift test 6/6 green;
xcodebuild -target VoiceCatiOS -sdk iphonesimulator BUILD SUCCEEDED.
This commit is contained in:
Talon
2026-06-19 13:17:52 +02:00
parent a10a18aebe
commit 9fc51cffc4
23 changed files with 861 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/include/voicecat.h
View File

@@ -396,6 +396,16 @@ VC_API vc_result vc_get_stream_audio_config(vc_client* c, uint32_t user_id, uint
VC_API vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const int16_t* pcm,
size_t samples);
/* Set the capture channel count for a local MIC stream (1 = mono, 2 = stereo interleaved).
* Must be called after vc_stream_start; takes effect on the next AudioEngine restart (e.g. when
* joining voice, or immediately if the engine is already running — it stops and restarts the
* capture device with the new channel count). Defaults to 1 (mono). On iOS this lets the Swift
* AVAudioSession routing layer request stereo built-in mic capture via
* setPreferredInputNumberOfChannels(2) and then tell the core to open the capture device in
* stereo. VC_ERR_INVALID_ARG if stream_id is unknown, the stream is not a MIC stream, or
* channels is not 1 or 2. */
VC_API vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels);
/* ── Text ─────────────────────────────────────────────────────────────────── */
VC_API vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id,
const char* utf8);

22
core/src/audio/audio_engine.cpp
View File

@@ -150,12 +150,13 @@ bool AudioEngine::start(const AudioParams& p, CaptureCallback capture_cb) {
#ifdef VOICECAT_HAS_AUDIO
// Pre-allocate capture accumulators before the devices start so on_capture / on_loopback
// never allocate on the RT thread. count=0 means "empty"; the buf is sized to exactly one
// encoder frame so a memcpy into it can never overrun. The mic accumulator is mono
// (params_.capture_channels, always 1 in v1 — no stereo mic). The loopback accumulator is
// encoder frame so a memcpy into it can never overrun. The mic accumulator is sized to
// frame_samples_ * capture_channels (1 = mono, 2 = stereo interleaved — set via
// vc_set_capture_channels, e.g. iOS stereo built-in mic). The loopback accumulator is
// sized mono here as a safe default and re-sized to frame_samples_*channels in
// start_loopback_capture() once the screen stream's channel mode is known (off the RT
// thread, before the loopback device is started).
capture_accum_.buf.assign(static_cast<size_t>(frame_samples_), 0);
capture_accum_.buf.assign(static_cast<size_t>(frame_samples_) * p.capture_channels, 0);
capture_accum_.count = 0;
loopback_accum_.buf.assign(static_cast<size_t>(frame_samples_), 0);
loopback_accum_.count = 0;
@@ -428,20 +429,25 @@ void AudioEngine::on_capture(const int16_t* pcm, ma_uint32 frames) {
// then fire capture_cb_. WASAPI shared mode commonly delivers 480-sample (10 ms) callbacks
// regardless of the periodSizeInFrames hint above; passing a sub-frame chunk directly to
// opus_encode() returns OPUS_BAD_ARG (negative), silently dropping every mic frame.
// PCM here is interleaved across params_.capture_channels (1 = mono, 2 = stereo L/R —
// e.g. iOS stereo built-in mic via vc_set_capture_channels) — the accumulator was sized to
// frame_samples_*capture_channels in start(), so a memcpy into it can never overrun.
// capture_cb_ receives samples-per-channel (frame_samples_) and the channel count explicitly.
if (!capture_cb_ || frame_samples_ <= 0) return;
const int ch = std::max(1u, params_.capture_channels);
const int16_t* src = pcm;
auto remaining = static_cast<int>(frames);
auto remaining = static_cast<int>(frames) * ch;
const int full = frame_samples_ * ch;
while (remaining > 0) {
int space = frame_samples_ - capture_accum_.count;
int space = full - capture_accum_.count;
int copy = std::min(remaining, space);
std::memcpy(capture_accum_.buf.data() + capture_accum_.count, src,
static_cast<size_t>(copy) * sizeof(int16_t));
capture_accum_.count += copy;
src += copy;
remaining -= copy;
if (capture_accum_.count == frame_samples_) {
capture_cb_(0, capture_accum_.buf.data(), frame_samples_,
static_cast<int>(params_.capture_channels));
if (capture_accum_.count == full) {
capture_cb_(0, capture_accum_.buf.data(), frame_samples_, ch);
capture_accum_.count = 0;
}
}

31
core/src/audio/audio_engine.h
View File

@@ -82,7 +82,7 @@ class JitterBuffer {
// ── AudioParams ──────────────────────────────────────────────────────────────
struct AudioParams {
uint32_t sample_rate = 48000;
uint32_t capture_channels = 1; // no stereo capture device (mic) in this pass
uint32_t capture_channels = 1; // mic capture: 1 = mono, 2 = stereo (set via vc_set_capture_channels)
uint32_t playback_channels = 2; // true stereo output (see audio_engine.cpp on_playback)
uint32_t frame_ms = 20;
std::string capture_device_id; // "" = default; opaque id from AudioEngine::enumerate_devices
@@ -215,6 +215,35 @@ class AudioEngine {
}
}
}
// TEST-ONLY — stereo-aware variant: drives the capture accumulator with interleaved L/R
// PCM (channels=2) or mono (channels=1). Sizes the accumulator to frame_samples_*channels
// and invokes `cb` with the channel count passed through — mirrors feed_loopback_for_test.
// Use to verify stereo mic capture (vc_set_capture_channels → on_capture's accumulator).
void feed_capture_for_test(const int16_t* pcm, int frames_per_channel, int channels,
const CaptureCallback& cb) {
if (frame_samples_ <= 0 || !cb) return;
const int ch = std::max(1, channels);
const int full = frame_samples_ * ch;
if (static_cast<int>(capture_accum_.buf.size()) != full) {
capture_accum_.buf.assign(static_cast<size_t>(full), 0);
capture_accum_.count = 0;
}
const int16_t* src = pcm;
auto remaining = frames_per_channel * ch;
while (remaining > 0) {
int space = full - capture_accum_.count;
int copy = std::min(remaining, space);
std::memcpy(capture_accum_.buf.data() + capture_accum_.count, src,
static_cast<size_t>(copy) * sizeof(int16_t));
capture_accum_.count += copy;
src += copy;
remaining -= copy;
if (capture_accum_.count == full) {
cb(0, capture_accum_.buf.data(), frame_samples_, ch);
capture_accum_.count = 0;
}
}
}
#endif
#ifdef VOICECAT_HAS_LOOPBACK

31
core/src/core/client.cpp
View File

@@ -1075,13 +1075,16 @@ void vc_client::ensure_audio_running() {
if (audio_engine_.running()) return;
voicecat::audio::AudioParams p;
p.sample_rate = 48000;
p.capture_channels = 1; // no stereo capture device in this pass
p.capture_channels = 1;
p.playback_channels = 2; // true stereo output (audio_engine.cpp on_playback)
p.frame_ms = 20;
{
std::lock_guard lk(local_streams_mu_);
auto it = local_streams_.find(static_cast<int>(VC_STREAM_MIC));
if (it != local_streams_.end()) p.capture_device_id = it->second.capture_device_id;
if (it != local_streams_.end()) {
p.capture_device_id = it->second.capture_device_id;
p.capture_channels = it->second.capture_channels;
}
}
audio_engine_.start(p, [this](int kind, const int16_t* pcm, int samples, int channels) {
on_capture_frame(kind, pcm, samples, channels);
@@ -1315,6 +1318,29 @@ vc_result vc_client::set_input_device(uint32_t stream_id, const char* device_id)
return VC_OK;
}
vc_result vc_client::set_capture_channels(uint32_t stream_id, uint32_t channels) {
if (channels != 1 && channels != 2) return VC_ERR_INVALID_ARG;
int kind = -1;
{
std::lock_guard lk(local_streams_mu_);
LocalStream* ls = find_local_stream_by_id(stream_id);
if (!ls) return VC_ERR_INVALID_ARG;
ls->capture_channels = channels;
for (auto& [k, entry] : local_streams_) {
if (&entry == ls) { kind = k; break; }
}
}
// Only the real capture device (MIC) is affected by channel count — SCREEN_AUDIO uses
// loopback capture (channel count driven by the channel's stereo mode, not this setter)
// and AUX_DEVICE isn't backed by a real device path yet. Restart the engine so the capture
// device re-opens with the new channel count.
if (kind == static_cast<int>(VC_STREAM_MIC) && audio_engine_.running()) {
audio_engine_.stop();
ensure_audio_running();
}
return VC_OK;
}
vc_result vc_client::set_input_mode(vc_input_mode mode) {
current_input_mode_.store(mode, std::memory_order_release);
return VC_OK;
@@ -1815,6 +1841,7 @@ vc_result vc_client::leave_channel() { return VC_ERR_NOT_
vc_result vc_client::stream_start(const vc_stream_desc&, uint32_t*) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::stream_stop(uint32_t) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::set_input_device(uint32_t, const char*) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::set_capture_channels(uint32_t, uint32_t) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::set_input_mode(vc_input_mode) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::set_vad_threshold(float) { return VC_ERR_NOT_IMPLEMENTED; }
vc_result vc_client::set_push_to_talk(bool) { return VC_ERR_NOT_IMPLEMENTED; }

6
core/src/core/client.h
View File

@@ -49,6 +49,7 @@ struct vc_client {
vc_result stream_start(const vc_stream_desc& desc, uint32_t* out_stream_id);
vc_result stream_stop(uint32_t stream_id);
vc_result set_input_device(uint32_t stream_id, const char* device_id);
vc_result set_capture_channels(uint32_t stream_id, uint32_t channels);
vc_result set_input_mode(vc_input_mode mode);
vc_result set_vad_threshold(float threshold);
vc_result set_push_to_talk(bool active);
@@ -230,6 +231,11 @@ struct vc_client {
// vc_set_input_device. Opaque id from AudioEngine::enumerate_devices — see
// audio_engine.h's DeviceInfo doc comment.
std::string capture_device_id;
// Capture channel count (1 = mono, 2 = stereo interleaved). Only meaningful for
// VC_STREAM_MIC. Set via vc_set_capture_channels; read by ensure_audio_running() to
// configure AudioParams.capture_channels before the device opens. Defaults to 1 (mono).
uint32_t capture_channels = 1;
};
mutable std::mutex local_streams_mu_;
std::unordered_map<int, LocalStream> local_streams_; // keyed by vc_stream_kind

5
core/src/voicecat.cpp
View File

@@ -141,6 +141,11 @@ vc_result vc_test_inject_capture(vc_client* c, uint32_t stream_id, const int16_t
return c->test_inject_capture(stream_id, pcm, samples);
}
vc_result vc_set_capture_channels(vc_client* c, uint32_t stream_id, uint32_t channels) {
if (c == nullptr) return VC_ERR_INVALID_ARG;
return c->set_capture_channels(stream_id, channels);
}
vc_result vc_send_text(vc_client* c, vc_text_scope scope, uint32_t target_id,
const char* utf8) {
if (c == nullptr || utf8 == nullptr) return VC_ERR_INVALID_ARG;