diff --git a/PROGRESS.md b/PROGRESS.md index 6ffe878..7d9e5aa 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -148,6 +148,21 @@ up instantly. Newest status at the top. SUCCEEDED (sim slice still arm64-only → simulator run N/A). Next: on-device check of the drill-down + compose box + unified timeline. +- **Done (2026-06-22):** **Windows exclude mode is now a real native exclude + self-echo + removal.** The "All apps except selected" mode previously captured the *complement of a frozen + app snapshot* in INCLUDE mode (missed late-launched apps, system sounds; wasted captures on + silent windows). It now opens a **single `ProcessLoopbackCapture` in EXCLUDE mode** + (`AUDIOCLIENT_PROCESS_LOOPBACK_MODE_EXCLUDE_TARGET_PROCESS_TREE`) of the one chosen app — true + system-mix-minus-one, dynamic. `AppAudioPickerDialog` enforces single-selection in exclude + mode (the API takes one target PID). Added an **"Exclude VoiceCat's own audio (prevents echo)"** + checkbox (default on, entire-desktop only) that routes the desktop capture through the same + EXCLUDE path targeting `Environment.ProcessId`, killing the whole-device self-echo loop. + Touched `ProcessAudioMixer.cs` (`ResolveCaptures`), `AppAudioPickerDialog.cs`, `MainForm.cs`, + `AudioSessionEnumerator.cs` (`EntireDesktop(bool ExcludeSelf)`); docs in voice.md §9. No C++ / + ABI changes. `dotnet build` clean. **Still to verify on-device:** exclude actually silences + the chosen app while the rest plays, late-launched apps appear without restart, and the + self-exclude checkbox removes the echo. + - **Done (2026-06-21):** **Screen-audio sharing on macOS + iOS.** macOS uses ScreenCaptureKit (`ScreenAudioCapture.swift`) → `vc_stream_feed_pcm`; iOS uses a ReplayKit Broadcast Upload Extension (`VoiceCatBroadcast`) that forwards captured `.audioApp` PCM through a shared App diff --git a/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs b/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs index 4e146b3..137df88 100644 --- a/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs +++ b/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs @@ -6,7 +6,10 @@ namespace VoiceCat.App.Audio; // ── Scope types (mirror macOS ScreenAudioScope / ScreenAudioSelection) ──────── public abstract record AppAudioScope; -public sealed record EntireDesktop : AppAudioScope; +// ExcludeSelf=true captures the whole system render mix minus VoiceCat's own process tree +// (kills the self-echo loop). Routed through the external-feed mixer as a single EXCLUDE +// capture; ExcludeSelf=false keeps the core's whole-device loopback path. +public sealed record EntireDesktop(bool ExcludeSelf = false) : AppAudioScope; public sealed record OnlyApps(IReadOnlyList Pids, IReadOnlyList Names) : AppAudioScope; public sealed record AllExceptApps(IReadOnlyList Pids, IReadOnlyList Names) : AppAudioScope; diff --git a/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs b/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs index 04ed01c..5fa9f4a 100644 --- a/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs +++ b/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs @@ -22,15 +22,14 @@ public sealed class ProcessAudioMixer : IDisposable private VoiceCatClient? _client; private uint _streamId; - public void Start(AppAudioScope scope, VoiceCatClient client, uint streamId, - IReadOnlyList allApps) + public void Start(AppAudioScope scope, VoiceCatClient client, uint streamId) { if (_running) return; _client = client; _streamId = streamId; - var pids = ResolvePids(scope, allApps); - if (pids.Count == 0) + var specs = ResolveCaptures(scope); + if (specs.Count == 0) { // nothing to capture — scope resolved to empty set return; @@ -38,14 +37,15 @@ public sealed class ProcessAudioMixer : IDisposable lock (_frameLock) { - _latestFrames = new List(new short[pids.Count][]); + _latestFrames = new List(new short[specs.Count][]); _activeChannels = Channels; } - for (int i = 0; i < pids.Count; i++) + for (int i = 0; i < specs.Count; i++) { int captureIndex = i; - var cap = new ProcessLoopbackCapture(pids[i], ProcessLoopbackCapture.Mode.Include); + var (pid, mode) = specs[i]; + var cap = new ProcessLoopbackCapture(pid, mode); cap.PcmFrameReady += (pcm, spc, ch) => OnCaptureFrame(captureIndex, pcm, ch); _captures.Add(cap); } @@ -117,18 +117,21 @@ public sealed class ProcessAudioMixer : IDisposable // ── Helpers ─────────────────────────────────────────────────────────────── - // For "AllExcept": enumerate all running audio apps and exclude the specified ones. - // For "OnlyApps": use their PIDs directly. - private static List ResolvePids(AppAudioScope scope, IReadOnlyList allApps) + // Resolve the scope to the WASAPI captures to open: + // OnlyApps → one INCLUDE capture per selected process tree. + // AllExceptApps → one EXCLUDE capture of the single selected process tree, which + // natively captures the whole system render mix minus that tree + // (dynamic — apps launched later are included automatically). + private static List<(int pid, ProcessLoopbackCapture.Mode mode)> ResolveCaptures(AppAudioScope scope) { return scope switch { - OnlyApps o => [.. o.Pids], - AllExceptApps a => - allApps - .Where(app => !a.Pids.Contains(app.Pid)) - .Select(app => app.Pid) - .ToList(), + OnlyApps o => o.Pids.Select(p => (p, ProcessLoopbackCapture.Mode.Include)).ToList(), + AllExceptApps a when a.Pids.Count > 0 => + [(a.Pids[0], ProcessLoopbackCapture.Mode.Exclude)], + // Entire desktop minus VoiceCat itself: EXCLUDE our own process tree. + EntireDesktop { ExcludeSelf: true } => + [(Environment.ProcessId, ProcessLoopbackCapture.Mode.Exclude)], _ => [], }; } diff --git a/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs b/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs index ac06e38..3b800f0 100644 --- a/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs +++ b/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs @@ -12,6 +12,7 @@ public sealed class AppAudioPickerDialog : Form private readonly RadioButton _rdoAll; private readonly RadioButton _rdoOnly; private readonly RadioButton _rdoExcept; + private readonly CheckBox _chkExcludeSelf; private readonly ListView _appList; private readonly Label _lblApps; @@ -50,26 +51,42 @@ public sealed class AppAudioPickerDialog : Form _rdoOnly.CheckedChanged += OnModeChanged; _rdoExcept.CheckedChanged += OnModeChanged; + // ── Self-exclude (echo prevention) ─────────────────────────────────── + // Captures the whole desktop minus VoiceCat's own playback, so the channel + // doesn't echo back the voices you're already hearing. Only applies to the + // entire-desktop mode (the per-app modes already exclude this app's tree). + _chkExcludeSelf = new CheckBox + { + Text = "E&xclude VoiceCat's own audio (prevents echo)", + Checked = true, + Location = new Point(28, 84), + Size = new Size(344, 20), + TabIndex = 3, + }; + // ── App list ──────────────────────────────────────────────────────── _lblApps = new Label { Text = "Apps with active audio sessions:", AutoSize = true, - Location = new Point(12, 90), + Location = new Point(12, 112), Visible = false, - TabIndex = 3, + TabIndex = 4, }; _appList = new ListView { - Location = new Point(12, 112), - Size = new Size(360, 180), + Location = new Point(12, 134), + Size = new Size(360, 158), CheckBoxes = true, View = View.List, Visible = false, - TabIndex = 4, + TabIndex = 5, FullRowSelect = true, }; + // Exclude mode supports only one target process tree (WASAPI EXCLUDE takes a single + // PID), so enforce a single check while "All apps except selected" is active. + _appList.ItemCheck += OnAppItemCheck; // ── Buttons ───────────────────────────────────────────────────────── var btnShare = new Button @@ -78,7 +95,7 @@ public sealed class AppAudioPickerDialog : Form DialogResult = DialogResult.OK, Location = new Point(216, 308), Size = new Size(75, 27), - TabIndex = 5, + TabIndex = 6, }; var btnCancel = new Button { @@ -86,7 +103,7 @@ public sealed class AppAudioPickerDialog : Form DialogResult = DialogResult.Cancel, Location = new Point(297, 308), Size = new Size(75, 27), - TabIndex = 6, + TabIndex = 7, }; btnShare.Click += OnShareClick; @@ -94,7 +111,8 @@ public sealed class AppAudioPickerDialog : Form CancelButton = btnCancel; AutoScaleMode = AutoScaleMode.Font; ClientSize = new Size(384, 348); - Controls.AddRange([_rdoAll, _rdoOnly, _rdoExcept, _lblApps, _appList, btnShare, btnCancel]); + Controls.AddRange([_rdoAll, _rdoOnly, _rdoExcept, _chkExcludeSelf, _lblApps, _appList, + btnShare, btnCancel]); FormBorderStyle = FormBorderStyle.FixedDialog; MaximizeBox = false; MinimizeBox = false; @@ -121,16 +139,56 @@ public sealed class AppAudioPickerDialog : Form bool showList = _rdoOnly.Checked || _rdoExcept.Checked; _lblApps.Visible = showList; _appList.Visible = showList; + _lblApps.Text = _rdoExcept.Checked + ? "App to exclude (everything else is shared):" + : "Apps with active audio sessions:"; + + // Self-exclude only applies to entire-desktop; the per-app modes already exclude + // this app's own tree (Include) or spend the single EXCLUDE slot on the chosen app. + _chkExcludeSelf.Enabled = _rdoAll.Checked; if (showList && _appList.Items.Count == 0) RefreshAppList(); + + // Switching into exclude mode: collapse any multi-selection down to a single item. + if (_rdoExcept.Checked) + KeepSingleCheck(firstCheckedOnly: true); + } + + // In exclude mode the list behaves like radio buttons: checking one item clears the rest. + private bool _suppressItemCheck; + + private void OnAppItemCheck(object? sender, ItemCheckEventArgs e) + { + if (_suppressItemCheck || !_rdoExcept.Checked || e.NewValue != CheckState.Checked) + return; + + _suppressItemCheck = true; + foreach (ListViewItem item in _appList.Items) + if (item.Index != e.Index && item.Checked) + item.Checked = false; + _suppressItemCheck = false; + } + + // Leave at most one item checked (the first), unchecking the rest. + private void KeepSingleCheck(bool firstCheckedOnly) + { + _suppressItemCheck = true; + bool kept = false; + foreach (ListViewItem item in _appList.Items) + { + if (!item.Checked) continue; + if (firstCheckedOnly && !kept) { kept = true; continue; } + item.Checked = false; + } + _suppressItemCheck = false; } private void OnShareClick(object? sender, EventArgs e) { if (_rdoAll.Checked) { - ChosenScope = new EntireDesktop(); + ChosenScope = new EntireDesktop(_chkExcludeSelf.Checked); return; } @@ -160,7 +218,4 @@ public sealed class AppAudioPickerDialog : Form ? new OnlyApps(checkedPids, checkedNames) : new AllExceptApps(checkedPids, checkedNames); } - - /// All apps that were visible in the list when the user clicked Share. - public IReadOnlyList VisibleApps => _apps; } diff --git a/clients/windows/VoiceCat.App/Forms/MainForm.cs b/clients/windows/VoiceCat.App/Forms/MainForm.cs index f4bfe22..07f85d6 100644 --- a/clients/windows/VoiceCat.App/Forms/MainForm.cs +++ b/clients/windows/VoiceCat.App/Forms/MainForm.cs @@ -640,7 +640,7 @@ public partial class MainForm : Form var scope = picker.ChosenScope; - if (scope is EntireDesktop) + if (scope is EntireDesktop { ExcludeSelf: false }) { // Existing whole-device WASAPI loopback path — core handles it. var (result, streamId) = _client.StartStream(VcStreamKind.ScreenAudio, "Desktop audio"); @@ -654,7 +654,8 @@ public partial class MainForm : Form } else { - // Per-app path: suppress core loopback, C# mixer feeds PCM. + // External-feed path: suppress core loopback, C# mixer feeds PCM. Covers the + // per-app modes and "entire desktop except VoiceCat" (single EXCLUDE of self). var (result, streamId) = _client.StartStreamExternalFeed(VcStreamKind.ScreenAudio, "App audio"); if (result != VcResult.Ok) { @@ -664,12 +665,13 @@ public partial class MainForm : Form _screenStreamId = streamId; _screenMixer = new ProcessAudioMixer(); - _screenMixer.Start(scope, _client, streamId, picker.VisibleApps); + _screenMixer.Start(scope, _client, streamId); string desc = scope switch { + EntireDesktop => "entire desktop (excluding VoiceCat)", OnlyApps o => $"only {o.Pids.Count} app(s)", - AllExceptApps a => $"all except {a.Pids.Count} app(s)", + AllExceptApps a => $"all except {a.Names[0]}", _ => "apps", }; AddActivity($"Sharing screen audio: {desc}"); diff --git a/docs/voice.md b/docs/voice.md index 0ede2c1..787e95c 100644 --- a/docs/voice.md +++ b/docs/voice.md @@ -301,7 +301,7 @@ normal stream; only the *source* is platform-specific. | Platform | Mechanism | Notes | |----------|-----------|-------| -| **Windows** | **WASAPI loopback** capture of the default render endpoint (via miniaudio's loopback mode) | **Implemented.** Captures in the channel's mode — stereo (interleaved L/R) when the channel is stereo, mono when the channel is mono — so a stereo music/screen-share channel gets genuine stereo end-to-end (no downmix). Whole-device capture, not process-specific — it inherently captures this app's own incoming voice mix along with everything else playing (an accepted self-echo-loop characteristic of desktop-audio capture, not a bug). Windows 10 2004+'s process-specific loopback (`AUDIOCLIENT_ACTIVATION_PARAMS`) would avoid this but miniaudio doesn't expose it — a future enhancement. | +| **Windows** | **WASAPI loopback** (whole-device, via miniaudio) **or WASAPI process loopback** (`AUDIOCLIENT_ACTIVATION_PARAMS`, Win10 2004+) for per-app / self-exclude | **Implemented.** Default *entire desktop* uses miniaudio's whole-device loopback in the channel's mode — stereo (interleaved L/R) when the channel is stereo, mono when mono — so a stereo channel gets genuine stereo end-to-end (no downmix). It inherently captures this app's own incoming voice mix (self-echo). The **per-app modes and the "exclude VoiceCat's own audio" option** instead drive `ProcessLoopbackCapture` (process-specific INCLUDE/EXCLUDE) through the external-feed mixer (`vc_stream_feed_pcm`, `external_feed=1`), which avoids self-echo and supports true "everything except". See below. | | **macOS** | **ScreenCaptureKit** system-audio capture (macOS 13+) | **Implemented** (`clients/apple/macOS/VoiceCatMac/Audio/ScreenAudioCapture.swift`). OS requires screen-recording permission; capture happens in the main app. An `SCStream` with `capturesAudio` + `excludesCurrentProcessAudio` delivers audio `CMSampleBuffer`s; Swift converts Float32 → int16 (in the channel's mono/stereo mode) and calls `vc_stream_feed_pcm` — no miniaudio loopback device involved (`VOICECAT_HAS_LOOPBACK` is Windows-only). **Supports per-app audio selection** — see below. | | **iOS** | **ReplayKit Broadcast Upload Extension** (the Discord mechanism) | **Implemented.** See below — separate process, App Group, ~50 MB cap (fine for audio-only). ReplayKit only ever delivers the *mixed* system stream as `.audioApp`, so **per-app filtering / VoiceOver exclusion is not possible on iOS** (it has no per-app granularity, unlike ScreenCaptureKit). | @@ -321,6 +321,25 @@ the speech-synthesis daemon that actually renders the spoken audio). The chosen `ScreenAudioSelection` is passed into `ScreenAudioCapture`, which builds the matching `SCContentFilter`. iOS/ReplayKit has no equivalent control (see the table note above). +### Windows detail — per-app audio selection and self-echo + +`AppAudioPickerDialog` (`clients/windows/VoiceCat.App/Forms/`) offers the same shape as macOS: + +- **Entire desktop** — whole-device miniaudio loopback handled by the core (default path). +- **Only selected apps** — one `ProcessLoopbackCapture` in **INCLUDE** mode per ticked app, + mixed by `ProcessAudioMixer` and fed via `vc_stream_feed_pcm`. +- **All apps except selected** — a **single** `ProcessLoopbackCapture` in **EXCLUDE** mode of + the chosen process tree. WASAPI's `AUDIOCLIENT_PROCESS_LOOPBACK_MODE_EXCLUDE_TARGET_PROCESS_TREE` + captures the whole render mix minus that tree *dynamically* (apps launched after sharing + starts are included automatically). The activation params take a **single** target PID, so + exclude is restricted to **one** app — the picker enforces single-selection in this mode. + +An **"Exclude VoiceCat's own audio (prevents echo)"** checkbox (default on, enabled for +*entire desktop*) routes the desktop capture through the same EXCLUDE path targeting +VoiceCat's **own** process id (`Environment.ProcessId`) — i.e. "entire desktop except this +app" — which removes the self-echo loop the whole-device path otherwise has. The per-app +INCLUDE modes already never capture this app's tree, so they have no self-echo to remove. + ### iOS detail The extension **captures**, the host app **sends**. Unlike a self-connecting extension, this