feat(audio): stereo mic capture on Windows & macOS desktop clients
Both desktop mics were hard-mono: the core defaults capture_channels=1 and neither client ever called vc_set_capture_channels (only iOS did). Add a persisted "Stereo microphone" toggle to each client's Audio settings, applied when the mic stream starts and live via vc_set_capture_channels + vc_audio_restart. Expose both ABI calls in the Windows interop; the macOS wrapper already had them. Core fix: encode_and_send_frame now folds a stereo mic frame to mono on a mono channel - previously the channels==2 branch encoded interleaved L/R directly even on a mono channel, feeding a mono opus_encode 2x its samples (wrong pitch/garbage). Real stereo still only reaches the wire on a stereo channel; on a mono channel the mic is cleanly downmixed. Test: test_stereo_mic_mono_channel. ctest --preset dev green (28/28). Docs: voice.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
18
PROGRESS.md
18
PROGRESS.md
@@ -26,6 +26,24 @@ up instantly. Newest status at the top.
|
||||
send cushion could be reduced or removed. Shared-core change → add a test and re-verify
|
||||
desktop↔desktop stays low-latency (steady sender ⇒ ~0 arrival jitter ⇒ no regression).
|
||||
|
||||
- **Done (2026-06-23):** **Stereo mic capture on Windows & macOS desktop clients.** Both
|
||||
desktop mics were hard-mono: `ensure_audio_running()` defaults `capture_channels = 1` and
|
||||
neither client ever called `vc_set_capture_channels` (only iOS did). Added a **"Stereo
|
||||
microphone" toggle** to each client's Audio settings (off by default, persisted —
|
||||
`VoiceSettings.StereoMic` on Windows, `MainWindowController.stereoMic` /
|
||||
`voice.stereoMic` UserDefaults on macOS). It's applied to the core when the mic stream
|
||||
starts (stored on the stream before the announce round-trip, so the first device open picks
|
||||
it up) and live in settings via `vc_set_capture_channels` + `vc_audio_restart`. Exposed both
|
||||
ABI calls in the Windows interop (`NativeMethods`/`VoiceCatClient`); the macOS wrapper already
|
||||
had them. **Core fix:** `encode_and_send_frame` (`core/src/core/client.cpp`) now folds a
|
||||
stereo mic frame to mono when the channel is mono — previously the `channels == 2` branch
|
||||
encoded interleaved L/R directly even on a mono channel, feeding a mono `opus_encode` 2× its
|
||||
samples (wrong pitch / garbage). Real stereo still only reaches the wire on a **stereo
|
||||
channel** (encoder channel count = channel's Opus mode); on a mono channel the mic is cleanly
|
||||
downmixed. Test: `test_stereo_mic_mono_channel` in `tests/test_vad_ptt_devices.cpp`. Full
|
||||
`ctest --preset dev` green — 28/28. macOS Xcode build not compiled here (Windows host); the
|
||||
Swift changes follow existing `nrChanged`/`setInputDevice` patterns. Docs: voice.md §8.
|
||||
|
||||
- **Done (2026-06-23):** **Fixed iOS dual-stream / crackly mic — core opened a second
|
||||
(miniaudio) capture device alongside the AVAudioEngine tap.** Symptom: with two clients in
|
||||
a channel, the remote end heard the iOS mic **twice** and crackly. With Voice Chat + a BT
|
||||
|
||||
@@ -62,6 +62,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
static let vadThreshold = "voice.vadThreshold"
|
||||
static let inputGain = "voice.inputGain"
|
||||
static let inputNoiseReduction = "voice.inputNoiseReduction"
|
||||
static let stereoMic = "voice.stereoMic"
|
||||
static let pttKeyCode = "voice.pttKeyCode"
|
||||
static let auxEnabled = "voice.auxEnabled"
|
||||
static let auxDeviceUID = "voice.auxDeviceUID"
|
||||
@@ -80,6 +81,12 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
internal var inputNoiseReduction: Bool = false {
|
||||
didSet { UserDefaults.standard.set(inputNoiseReduction, forKey: AudioDefaults.inputNoiseReduction) }
|
||||
}
|
||||
// Capture the mic in stereo (interleaved L/R) instead of mono. Real stereo only reaches the
|
||||
// wire on a stereo channel; the core folds a stereo mic to mono on a mono channel. Applied to
|
||||
// the core when the mic stream starts (micToggleClicked) and live via SettingsWindowController.
|
||||
internal var stereoMic: Bool = false {
|
||||
didSet { UserDefaults.standard.set(stereoMic, forKey: AudioDefaults.stereoMic) }
|
||||
}
|
||||
internal var selectedInputDeviceId: String?
|
||||
|
||||
// Aux outgoing stream: a second hardware input device the client captures itself and feeds to
|
||||
@@ -163,6 +170,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
if d.object(forKey: AudioDefaults.inputNoiseReduction) != nil {
|
||||
inputNoiseReduction = d.bool(forKey: AudioDefaults.inputNoiseReduction)
|
||||
}
|
||||
stereoMic = d.bool(forKey: AudioDefaults.stereoMic)
|
||||
if d.object(forKey: AudioDefaults.pttKeyCode) != nil {
|
||||
pttKeyCode = UInt16(d.integer(forKey: AudioDefaults.pttKeyCode))
|
||||
}
|
||||
@@ -781,6 +789,9 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
if let devId = selectedInputDeviceId {
|
||||
client.setInputDevice(streamId: streamId, deviceId: devId)
|
||||
}
|
||||
// Stored on the stream before the announce round-trip completes, so the core's
|
||||
// first capture-device open (ensure_audio_running) picks up the channel count.
|
||||
client.setCaptureChannels(streamId: streamId, channels: stereoMic ? 2 : 1)
|
||||
client.setInputMode(selectedInputMode)
|
||||
if selectedInputMode == .voiceActivation {
|
||||
client.setVadThreshold(vadThresholdValue)
|
||||
|
||||
@@ -63,6 +63,12 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
private let nrCheckbox = NSButton(checkboxWithTitle: "Noise reduction (RNNoise)",
|
||||
target: nil, action: nil)
|
||||
|
||||
// Capture the mic in stereo (interleaved L/R) instead of mono. Real stereo only reaches the
|
||||
// wire on a stereo channel; the core folds a stereo mic to mono on a mono channel. Persisted
|
||||
// via MainWindowController.stereoMic.
|
||||
private let stereoMicCheckbox = NSButton(checkboxWithTitle: "Stereo microphone",
|
||||
target: nil, action: nil)
|
||||
|
||||
// Aux input stream: a second outgoing stream from another hardware input device (e.g. line-in
|
||||
// / aux), captured client-side. Device + volume only — aux is always-on. Persisted via
|
||||
// MainWindowController.auxEnabled / auxDeviceUID / auxGain.
|
||||
@@ -178,6 +184,11 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
nrCheckbox.setAccessibilityLabel("Microphone noise reduction")
|
||||
nrCheckbox.setAccessibilityHelp("RNNoise denoising of your microphone. Cleans your signal for everyone.")
|
||||
|
||||
stereoMicCheckbox.target = self
|
||||
stereoMicCheckbox.action = #selector(stereoMicChanged)
|
||||
stereoMicCheckbox.setAccessibilityLabel("Stereo microphone")
|
||||
stereoMicCheckbox.setAccessibilityHelp("Capture your microphone in stereo. Only transmitted in stereo on a stereo channel.")
|
||||
|
||||
let inputModeRow = NSStackView(views: [inputModeLabel, inputModeControl])
|
||||
inputModeRow.orientation = .horizontal
|
||||
inputModeRow.spacing = 8
|
||||
@@ -259,7 +270,7 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
volumeRow.orientation = .horizontal
|
||||
volumeRow.spacing = 8
|
||||
|
||||
let stack = NSStackView(views: [inputModeRow, vadRow, inputGainRow, nrCheckbox, pttRow, deviceRow,
|
||||
let stack = NSStackView(views: [inputModeRow, vadRow, inputGainRow, nrCheckbox, stereoMicCheckbox, pttRow, deviceRow,
|
||||
levelRow, auxHeader, auxCheckbox, auxDeviceRow, auxGainRow,
|
||||
notificationsHeader, soundsCheckbox, volumeRow,
|
||||
speechCheckbox, selfTalkCheckbox, pttSoundCheckbox])
|
||||
@@ -330,6 +341,7 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
inputGainSlider.doubleValue = Double(mc.inputGain * 100)
|
||||
updateInputGainLabel()
|
||||
nrCheckbox.state = mc.inputNoiseReduction ? .on : .off
|
||||
stereoMicCheckbox.state = mc.stereoMic ? .on : .off
|
||||
|
||||
auxCheckbox.state = mc.auxEnabled ? .on : .off
|
||||
auxGainSlider.doubleValue = Double(mc.auxGain * 100)
|
||||
@@ -390,6 +402,16 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
}
|
||||
}
|
||||
|
||||
@objc private func stereoMicChanged() {
|
||||
let on = stereoMicCheckbox.state == .on
|
||||
mainController?.stereoMic = on
|
||||
// Channel count only takes effect when the capture device (re)starts, so restart it live.
|
||||
if let mc = mainController, mc.micStreamId != 0 {
|
||||
client.setCaptureChannels(streamId: mc.micStreamId, channels: on ? 2 : 1)
|
||||
client.audioRestart()
|
||||
}
|
||||
}
|
||||
|
||||
private func updateInputGainLabel() {
|
||||
let pct = Int(inputGainSlider.doubleValue.rounded())
|
||||
inputGainValueLabel.stringValue = "\(pct)%"
|
||||
|
||||
@@ -28,6 +28,7 @@ public sealed class AudioSettingsForm : Form
|
||||
private readonly int _origVadSlider;
|
||||
private readonly int _origMicGain;
|
||||
private readonly bool _origMicNoiseReduction;
|
||||
private readonly bool _origStereoMic;
|
||||
private readonly Keys _origPttKey;
|
||||
private readonly bool _origAuxEnabled;
|
||||
private readonly string? _origAuxDeviceId;
|
||||
@@ -44,6 +45,7 @@ public sealed class AudioSettingsForm : Form
|
||||
private readonly TrackBar _trkVad;
|
||||
private readonly TrackBar _trkGain;
|
||||
private readonly CheckBox _chkNoiseReduction;
|
||||
private readonly CheckBox _chkStereoMic;
|
||||
private readonly CheckBox _chkAux;
|
||||
private readonly Label _lblAuxDevice;
|
||||
private readonly ComboBox _cboAuxDevice;
|
||||
@@ -70,6 +72,7 @@ public sealed class AudioSettingsForm : Form
|
||||
_origVadSlider = settings.VadThresholdSlider;
|
||||
_origMicGain = settings.MicGain;
|
||||
_origMicNoiseReduction = settings.MicNoiseReduction;
|
||||
_origStereoMic = settings.StereoMic;
|
||||
_origPttKey = _pttKey;
|
||||
_origAuxEnabled = settings.AuxEnabled;
|
||||
_origAuxDeviceId = settings.AuxDeviceId;
|
||||
@@ -81,7 +84,7 @@ public sealed class AudioSettingsForm : Form
|
||||
MinimizeBox = false;
|
||||
StartPosition = FormStartPosition.CenterParent;
|
||||
AutoScaleMode = AutoScaleMode.Font;
|
||||
ClientSize = new Size(420, 583);
|
||||
ClientSize = new Size(420, 611);
|
||||
|
||||
// ── Device row ────────────────────────────────────────────────────────
|
||||
var lblDevice = new Label
|
||||
@@ -217,6 +220,23 @@ public sealed class AudioSettingsForm : Form
|
||||
};
|
||||
_chkNoiseReduction.CheckedChanged += ChkNoiseReduction_CheckedChanged;
|
||||
|
||||
// ── Stereo microphone ─────────────────────────────────────────────────
|
||||
// Opens the mic capture device in stereo (interleaved L/R) instead of mono. Real stereo
|
||||
// only reaches the wire on a stereo channel; the core folds a stereo mic to mono on a mono
|
||||
// channel. Toggling while connected restarts the capture device (vc_audio_restart) so the
|
||||
// new channel count takes effect immediately.
|
||||
_chkStereoMic = new CheckBox
|
||||
{
|
||||
Text = "&Stereo microphone",
|
||||
Location = new Point(12, 364),
|
||||
AutoSize = true,
|
||||
Checked = settings.StereoMic,
|
||||
AccessibleName = "Stereo microphone",
|
||||
AccessibleDescription =
|
||||
"Capture your microphone in stereo. Only transmitted in stereo on a stereo channel.",
|
||||
};
|
||||
_chkStereoMic.CheckedChanged += ChkStereoMic_CheckedChanged;
|
||||
|
||||
// ── Aux input stream ──────────────────────────────────────────────────
|
||||
// A second outgoing stream from another hardware input device (e.g. line-in / aux),
|
||||
// captured client-side and fed to the core. Device + volume only — aux is always-on
|
||||
@@ -224,7 +244,7 @@ public sealed class AudioSettingsForm : Form
|
||||
_chkAux = new CheckBox
|
||||
{
|
||||
Text = "&Aux stream (second input device)",
|
||||
Location = new Point(12, 376),
|
||||
Location = new Point(12, 404),
|
||||
AutoSize = true,
|
||||
Checked = settings.AuxEnabled,
|
||||
AccessibleName = "Enable aux input stream",
|
||||
@@ -236,12 +256,12 @@ public sealed class AudioSettingsForm : Form
|
||||
_lblAuxDevice = new Label
|
||||
{
|
||||
Text = "Aux d&evice:",
|
||||
Location = new Point(12, 406),
|
||||
Location = new Point(12, 434),
|
||||
AutoSize = true,
|
||||
};
|
||||
_cboAuxDevice = new ComboBox
|
||||
{
|
||||
Location = new Point(12, 426),
|
||||
Location = new Point(12, 454),
|
||||
Width = 300,
|
||||
DropDownStyle = ComboBoxStyle.DropDownList,
|
||||
DisplayMember = "Name",
|
||||
@@ -254,7 +274,7 @@ public sealed class AudioSettingsForm : Form
|
||||
_btnAuxRefresh = new Button
|
||||
{
|
||||
Text = "Re&fresh",
|
||||
Location = new Point(320, 424),
|
||||
Location = new Point(320, 452),
|
||||
Size = new Size(80, 26),
|
||||
};
|
||||
_btnAuxRefresh.Click += (_, _) => LoadAuxDevices();
|
||||
@@ -262,12 +282,12 @@ public sealed class AudioSettingsForm : Form
|
||||
_lblAuxGain = new Label
|
||||
{
|
||||
Text = "Aux vo&lume:",
|
||||
Location = new Point(12, 462),
|
||||
Location = new Point(12, 490),
|
||||
AutoSize = true,
|
||||
};
|
||||
_trkAuxGain = new TrackBar
|
||||
{
|
||||
Location = new Point(12, 482),
|
||||
Location = new Point(12, 510),
|
||||
Size = new Size(200, 45),
|
||||
Minimum = 0,
|
||||
Maximum = 300,
|
||||
@@ -286,14 +306,14 @@ public sealed class AudioSettingsForm : Form
|
||||
{
|
||||
Text = "&OK",
|
||||
DialogResult = DialogResult.OK,
|
||||
Location = new Point(228, 544),
|
||||
Location = new Point(228, 572),
|
||||
Size = new Size(80, 27),
|
||||
};
|
||||
var btnCancel = new Button
|
||||
{
|
||||
Text = "&Cancel",
|
||||
DialogResult = DialogResult.Cancel,
|
||||
Location = new Point(316, 544),
|
||||
Location = new Point(316, 572),
|
||||
Size = new Size(80, 27),
|
||||
};
|
||||
|
||||
@@ -310,7 +330,7 @@ public sealed class AudioSettingsForm : Form
|
||||
lblDevice, _cboDevice, _btnRefresh,
|
||||
lblMode, _radioVad, _lblSensitivity, _trkVad,
|
||||
_radioPtt, _lblPttKey, _btnChangePtt, _radioAlwaysOn,
|
||||
lblGain, _trkGain, _chkNoiseReduction,
|
||||
lblGain, _trkGain, _chkNoiseReduction, _chkStereoMic,
|
||||
_chkAux, _lblAuxDevice, _cboAuxDevice, _btnAuxRefresh, _lblAuxGain, _trkAuxGain,
|
||||
btnOk, btnCancel,
|
||||
]);
|
||||
@@ -455,6 +475,17 @@ public sealed class AudioSettingsForm : Form
|
||||
_client.SetInputNoiseReduction(_chkNoiseReduction.Checked);
|
||||
}
|
||||
|
||||
private void ChkStereoMic_CheckedChanged(object? sender, EventArgs e)
|
||||
{
|
||||
_settings.StereoMic = _chkStereoMic.Checked;
|
||||
// Channel count only takes effect when the capture device (re)starts, so restart it live.
|
||||
if (_micStreamId != 0)
|
||||
{
|
||||
_client.SetCaptureChannels(_micStreamId, _chkStereoMic.Checked ? 2u : 1u);
|
||||
_client.AudioRestart();
|
||||
}
|
||||
}
|
||||
|
||||
private void BtnChangePtt_Click(object? sender, EventArgs e)
|
||||
{
|
||||
using var dlg = new PttKeyCaptureDialog(_pttKey);
|
||||
@@ -476,6 +507,7 @@ public sealed class AudioSettingsForm : Form
|
||||
_settings.VadThresholdSlider = _origVadSlider;
|
||||
_settings.MicGain = _origMicGain;
|
||||
_settings.MicNoiseReduction = _origMicNoiseReduction;
|
||||
_settings.StereoMic = _origStereoMic;
|
||||
_settings.PttKey = (int)_origPttKey;
|
||||
|
||||
if (_micStreamId != 0)
|
||||
@@ -486,6 +518,12 @@ public sealed class AudioSettingsForm : Form
|
||||
_client.SetVadThreshold(0.1f * (1f - (_origVadSlider - 1f) / 99f));
|
||||
_client.SetInputGain(_origMicGain / 100f);
|
||||
_client.SetInputNoiseReduction(_origMicNoiseReduction);
|
||||
// Restore capture channel count; restart the device only if it actually changed.
|
||||
if (_origStereoMic != _chkStereoMic.Checked)
|
||||
{
|
||||
_client.SetCaptureChannels(_micStreamId, _origStereoMic ? 2u : 1u);
|
||||
_client.AudioRestart();
|
||||
}
|
||||
}
|
||||
|
||||
// Aux: restore originals to settings and re-apply live (order: device + gain first so a
|
||||
|
||||
@@ -646,6 +646,7 @@ public partial class MainForm : Form
|
||||
var mode = (VcInputMode)_voiceSettings.InputMode;
|
||||
if (_voiceSettings.InputDeviceId is string devId)
|
||||
_client.SetInputDevice(streamId, devId);
|
||||
_client.SetCaptureChannels(streamId, _voiceSettings.StereoMic ? 2u : 1u);
|
||||
_client.SetInputMode(mode);
|
||||
if (mode == VcInputMode.VoiceActivation)
|
||||
_client.SetVadThreshold(VadThresholdFromSettings());
|
||||
|
||||
@@ -26,6 +26,11 @@ public sealed class VoiceSettings
|
||||
/// cleaned signal. Independent of the per-listener receive-side NR.</summary>
|
||||
public bool MicNoiseReduction { get; set; } = false;
|
||||
|
||||
/// <summary>Capture the mic in stereo (interleaved L/R) instead of mono. Off by default. Real
|
||||
/// stereo only reaches the wire on a stereo channel; on a mono channel the core folds the mic
|
||||
/// to mono. Applied when the capture device next starts (Join Voice or an audio restart).</summary>
|
||||
public bool StereoMic { get; set; } = false;
|
||||
|
||||
/// <summary>Push-to-talk key, stored as the integer value of System.Windows.Forms.Keys.</summary>
|
||||
public int PttKey { get; set; } = (int)Keys.F8;
|
||||
|
||||
|
||||
@@ -65,6 +65,12 @@ internal static partial class NativeMethods
|
||||
[LibraryImport(LibName, StringMarshalling = StringMarshalling.Utf8)]
|
||||
internal static partial VcResult vc_set_input_device(nint c, uint streamId, string? deviceId);
|
||||
|
||||
[LibraryImport(LibName)]
|
||||
internal static partial VcResult vc_set_capture_channels(nint c, uint streamId, uint channels);
|
||||
|
||||
[LibraryImport(LibName)]
|
||||
internal static partial VcResult vc_audio_restart(nint c);
|
||||
|
||||
[LibraryImport(LibName)]
|
||||
internal static partial VcResult vc_set_input_mode(nint c, VcInputMode mode);
|
||||
|
||||
|
||||
@@ -212,6 +212,19 @@ public sealed class VoiceCatClient : IDisposable
|
||||
public VcResult SetInputDevice(uint streamId, string? deviceId) =>
|
||||
NativeMethods.vc_set_input_device(_handle.DangerousGetHandle(), streamId, deviceId);
|
||||
|
||||
/// <summary>Sets the mic capture channel count (1 = mono, 2 = stereo) for the given stream.
|
||||
/// Applied when the capture device next (re)starts — call before Join Voice, or pair with an
|
||||
/// audio restart to take effect live. Real stereo only reaches the wire on a stereo channel;
|
||||
/// the core folds a stereo mic to mono on a mono channel.</summary>
|
||||
public VcResult SetCaptureChannels(uint streamId, uint channels) =>
|
||||
NativeMethods.vc_set_capture_channels(_handle.DangerousGetHandle(), streamId, channels);
|
||||
|
||||
/// <summary>Uninitializes and re-initializes the capture and playback devices on a running
|
||||
/// engine, applying pending changes (e.g. capture channel count) that only take effect on a
|
||||
/// device restart. No-op if audio isn't running.</summary>
|
||||
public VcResult AudioRestart() =>
|
||||
NativeMethods.vc_audio_restart(_handle.DangerousGetHandle());
|
||||
|
||||
public VcResult SetInputMode(VcInputMode mode) =>
|
||||
NativeMethods.vc_set_input_mode(_handle.DangerousGetHandle(), mode);
|
||||
|
||||
|
||||
@@ -1106,15 +1106,26 @@ void vc_client::encode_and_send_frame(LocalStream& ls, const int16_t* pcm, int s
|
||||
int channels, int fd) {
|
||||
uint8_t opus_buf[1500];
|
||||
int opus_len;
|
||||
if (channels == 2) {
|
||||
// Real interleaved stereo PCM (SCREEN_AUDIO loopback on a stereo channel) — encode
|
||||
// directly, no upmix. `samples` is samples-per-channel, as OpusEncoder::encode expects.
|
||||
if (channels == 2 && ls.effective_params.stereo) {
|
||||
// Real interleaved stereo PCM (a stereo mic via vc_set_capture_channels, or SCREEN_AUDIO
|
||||
// loopback) on a channel configured for stereo — encode directly, true L/R, no fold.
|
||||
// `samples` is samples-per-channel, as OpusEncoder::encode expects.
|
||||
opus_len = ls.encoder.encode(pcm, samples, opus_buf, sizeof(opus_buf));
|
||||
} else if (channels == 2) {
|
||||
// Stereo capture (stereo mic / line-in) on a MONO channel: the encoder is mono, so fold
|
||||
// L/R to mono first. Feeding interleaved pairs straight to a mono opus_encode would make
|
||||
// it read 2× the samples it should (wrong pitch / garbage). upmix_scratch is pre-sized at
|
||||
// announce and easily holds `samples` mono values. This keeps a stereo-mic toggle safe on
|
||||
// every channel: real L/R when the channel is stereo, a clean downmix when it isn't.
|
||||
int16_t* mono = ls.upmix_scratch.data();
|
||||
for (int i = 0; i < samples; ++i)
|
||||
mono[i] = static_cast<int16_t>(
|
||||
(static_cast<int32_t>(pcm[2 * i]) + static_cast<int32_t>(pcm[2 * i + 1])) / 2);
|
||||
opus_len = ls.encoder.encode(mono, samples, opus_buf, sizeof(opus_buf));
|
||||
} else if (ls.effective_params.stereo) {
|
||||
// Mono capture (mic, or loopback on a mono channel, or test injection) on a channel
|
||||
// Mono capture (mono mic, or loopback on a mono channel, or test injection) on a channel
|
||||
// configured for stereo — upmix L=R so the stream is still a spec-correct stereo Opus
|
||||
// bitstream. (Mic stays mono in v1 — no stereo capture device — but a stereo channel
|
||||
// requires a stereo bitstream, hence the upmix.) upmix_scratch is pre-sized at announce.
|
||||
// bitstream (a stereo channel requires a stereo bitstream). upmix_scratch is pre-sized at announce.
|
||||
int16_t* st = ls.upmix_scratch.data();
|
||||
for (int i = 0; i < samples; ++i) {
|
||||
st[i * 2] = pcm[i];
|
||||
|
||||
@@ -224,9 +224,20 @@ Each receiver keeps an **adaptive jitter buffer per ssrc** with **bounded-depth
|
||||
- Capture and playback run on miniaudio's real-time callbacks (WASAPI / CoreAudio / ALSA).
|
||||
Playback is genuinely stereo end-to-end. **Mic capture** is mono by default; **stereo mic
|
||||
capture** is supported via `vc_set_capture_channels(stream_id, 2)` — when enabled, the
|
||||
capture device opens in stereo (interleaved L/R) and the encoder receives real stereo PCM
|
||||
(no upmix). A mono mic frame on a stereo channel is upmixed L=R before encoding so the Opus
|
||||
bitstream is still spec-correct stereo. **Screen-audio (`SCREEN_AUDIO`) loopback** captures
|
||||
capture device opens in stereo (interleaved L/R). All native clients expose this as a
|
||||
per-user toggle (iOS in Settings; the Windows and macOS desktop clients via a "Stereo
|
||||
microphone" checkbox in Audio settings — a live toggle there restarts the capture device via
|
||||
`vc_audio_restart` so it takes effect immediately). Whether stereo actually reaches the wire
|
||||
depends on the **channel's** Opus mode, which decides the encoder's channel count — the mic's
|
||||
channel count and the channel's mode are independent knobs:
|
||||
- **stereo mic + stereo channel** → real interleaved L/R is encoded directly (no upmix).
|
||||
- **mono mic + stereo channel** → the mono frame is upmixed L=R so the Opus bitstream is
|
||||
still spec-correct stereo.
|
||||
- **stereo mic + mono channel** → the interleaved L/R is folded to mono before the mono
|
||||
encoder. (Handing interleaved pairs straight to a mono `opus_encode` would make it read 2×
|
||||
the samples it should — wrong pitch / garbage — so the fold keeps the toggle safe on any
|
||||
channel.)
|
||||
**Screen-audio (`SCREEN_AUDIO`) loopback** captures
|
||||
in the channel's mode — stereo when the channel is stereo (real interleaved L/R, no
|
||||
downmix), mono when the channel is mono — so a stereo music/screen-share channel gets
|
||||
genuine stereo end-to-end. See §9 for the platform-specific loopback mechanism.
|
||||
|
||||
@@ -662,6 +662,59 @@ static void test_stereo_mic_capture() {
|
||||
}
|
||||
#endif
|
||||
|
||||
// ── 6. Stereo mic capture on a MONO channel (downmix safety) ──────────────────
|
||||
// A stereo mic (vc_set_capture_channels=2) can be enabled while on a mono channel. The mic
|
||||
// then delivers interleaved L/R, but the channel's Opus encoder is mono. encode_and_send_frame
|
||||
// must fold L/R to mono before encoding — handing interleaved pairs straight to a mono
|
||||
// opus_encode makes it read 2× the samples it should (wrong pitch / garbage). This mirrors that
|
||||
// fold and proves the result is a valid mono bitstream that decodes to the expected averaged
|
||||
// signal, rather than half-length junk.
|
||||
#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS)
|
||||
static void test_stereo_mic_mono_channel() {
|
||||
voicecat::codec::OpusParams mono_params;
|
||||
mono_params.stereo = false; // mono channel — encoder is mono
|
||||
mono_params.application = voicecat::codec::OpusApplication::Voip;
|
||||
mono_params.bitrate_bps = 64000;
|
||||
const int frame_samples = voicecat::codec::opus_frame_samples(mono_params);
|
||||
|
||||
voicecat::codec::OpusEncoder enc;
|
||||
CHECK(enc.init(mono_params));
|
||||
|
||||
// Loud left, silent right — folding (L+R)/2 yields a half-amplitude tone on every sample.
|
||||
std::vector<int16_t> interleaved(static_cast<size_t>(frame_samples) * 2);
|
||||
for (int i = 0; i < frame_samples; ++i) {
|
||||
float t = static_cast<float>(i) / 48000.0f;
|
||||
interleaved[i * 2] = static_cast<int16_t>(std::sin(2.0f * 3.14159265f * 440.0f * t) * 20000.0f);
|
||||
interleaved[i * 2 + 1] = 0;
|
||||
}
|
||||
|
||||
// Fold exactly as encode_and_send_frame does for a stereo frame on a mono channel.
|
||||
std::vector<int16_t> folded(frame_samples);
|
||||
for (int i = 0; i < frame_samples; ++i)
|
||||
folded[i] = static_cast<int16_t>(
|
||||
(static_cast<int32_t>(interleaved[i * 2]) + static_cast<int32_t>(interleaved[i * 2 + 1])) / 2);
|
||||
|
||||
uint8_t opus_buf[1500];
|
||||
int opus_len = enc.encode(folded.data(), frame_samples, opus_buf, sizeof(opus_buf));
|
||||
CHECK(opus_len > 0);
|
||||
|
||||
// Decode mono and verify a full-length frame with real energy survived (a garbage half-read
|
||||
// would either fail to decode the full frame_samples or come back near-silent / wrong length).
|
||||
voicecat::codec::OpusDecoder dec;
|
||||
CHECK(dec.init(mono_params));
|
||||
std::vector<int16_t> decoded(frame_samples, 0);
|
||||
int dec_samples = dec.decode(opus_buf, opus_len, decoded.data(), frame_samples);
|
||||
CHECK(dec_samples == frame_samples);
|
||||
|
||||
int64_t energy = 0;
|
||||
for (int i = 0; i < frame_samples; ++i) energy += std::abs(static_cast<int>(decoded[i]));
|
||||
CHECK(energy > static_cast<int64_t>(frame_samples) * 500); // clearly audible, not silence
|
||||
|
||||
std::printf("test_stereo_mic_mono_channel: ok (opus_len=%d, energy=%lld)\n",
|
||||
opus_len, static_cast<long long>(energy));
|
||||
}
|
||||
#endif
|
||||
|
||||
int main() {
|
||||
test_device_enumeration();
|
||||
#if defined(VOICECAT_HAS_AUDIO) && defined(VOICECAT_HAS_OPUS)
|
||||
@@ -670,6 +723,7 @@ int main() {
|
||||
test_loopback_stereo_capture();
|
||||
#endif
|
||||
test_stereo_mic_capture();
|
||||
test_stereo_mic_mono_channel();
|
||||
test_playout_resync();
|
||||
#endif
|
||||
#ifdef VOICECAT_HAS_AUDIO
|
||||
|
||||
Reference in New Issue
Block a user