From e806b698ecd247ea5f4d45aa61eef0c2f791ff1b Mon Sep 17 00:00:00 2001 From: Talon Date: Mon, 22 Jun 2026 00:19:38 +0200 Subject: [PATCH] feat(windows): per-app screen-audio sharing (include/exclude) Adds "only selected apps" and "all apps except selected" screen-audio modes to the Windows client, alongside the existing entire-desktop path. Per-app capture uses WASAPI process loopback (AUDCLNT_ACTIVATIONTYPE_ PROCESS_LOOPBACK) via ProcessLoopbackCapture, mixed by ProcessAudioMixer and fed to the core through vc_stream_feed_pcm (external_feed=1 so the core skips its own loopback device). Init must pass AUDCLNT_STREAMFLAGS_LOOPBACK | EVENTCALLBACK | AUTOCONVERTPCM; the LOOPBACK flag is what makes the virtual endpoint deliver rendered audio (without it every buffer is flagged SILENT) and AUTOCONVERTPCM resamples the app's native format to 48k s16. Co-Authored-By: Claude Opus 4.8 --- .../Audio/AudioSessionEnumerator.cs | 205 +++++++++ .../VoiceCat.App/Audio/ProcessAudioMixer.cs | 146 ++++++ .../Audio/ProcessLoopbackCapture.cs | 434 ++++++++++++++++++ .../Forms/AppAudioPickerDialog.cs | 166 +++++++ .../windows/VoiceCat.App/Forms/MainForm.cs | 83 +++- .../windows/VoiceCat.App/VoiceCat.App.csproj | 3 + clients/windows/VoiceCat.Interop/Structs.cs | 3 + .../VoiceCat.Interop/VoiceCatClient.cs | 20 + core/include/voicecat.h | 6 + core/src/core/client.cpp | 17 +- core/src/core/client.h | 5 + 11 files changed, 1064 insertions(+), 24 deletions(-) create mode 100644 clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs create mode 100644 clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs create mode 100644 clients/windows/VoiceCat.App/Audio/ProcessLoopbackCapture.cs create mode 100644 clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs diff --git a/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs b/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs new file mode 100644 index 0000000..4e146b3 --- /dev/null +++ b/clients/windows/VoiceCat.App/Audio/AudioSessionEnumerator.cs @@ -0,0 +1,205 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace VoiceCat.App.Audio; + +// ── Scope types (mirror macOS ScreenAudioScope / ScreenAudioSelection) ──────── + +public abstract record AppAudioScope; +public sealed record EntireDesktop : AppAudioScope; +public sealed record OnlyApps(IReadOnlyList Pids, IReadOnlyList Names) : AppAudioScope; +public sealed record AllExceptApps(IReadOnlyList Pids, IReadOnlyList Names) : AppAudioScope; + +public sealed record AudioAppInfo(int Pid, string DisplayName); + +// ── Enumerator ───────────────────────────────────────────────────────────────── + +public static class AudioSessionEnumerator +{ + // Returns all user-facing apps: visible-window processes (primary, like macOS + // SCShareableContent.current) plus any background audio-session-only processes + // (e.g. Spotify in mini-player). Excludes VoiceCat itself and system processes. + public static IReadOnlyList GetAudioApps() + { + var seen = new HashSet(); + var result = new List(); + int selfPid = Environment.ProcessId; + + // ── 1. Visible-window processes (EnumWindows) ───────────────────────── + // Same set macOS ScreenCaptureKit exposes: all apps with at least one + // visible top-level window. Shows apps even when not currently producing audio. + EnumWindows((hWnd, _) => + { + if (!IsWindowVisible(hWnd)) return true; + GetWindowThreadProcessId(hWnd, out uint pid); + if (pid == 0 || pid == (uint)selfPid || !seen.Add((int)pid)) return true; + + try + { + var proc = Process.GetProcessById((int)pid); + string name = proc.MainWindowTitle.Length > 0 + ? $"{proc.ProcessName} — {proc.MainWindowTitle}" + : proc.ProcessName; + if (!string.IsNullOrEmpty(proc.ProcessName)) + result.Add(new AudioAppInfo((int)pid, name)); + } + catch { /* process exited between EnumWindows and GetProcessById */ } + + return true; // continue enumeration + }, IntPtr.Zero); + + // ── 2. Background audio-session processes (WASAPI, supplement) ──────── + // Catches apps that produce audio but have no visible window (screen reader, + // background music player, etc.). Silently skipped if WASAPI is unavailable. + AppendAudioSessionApps(seen, selfPid, result); + + return result.OrderBy(a => a.DisplayName, StringComparer.OrdinalIgnoreCase).ToList(); + } + + // ── EnumWindows P/Invoke ────────────────────────────────────────────────── + + private delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam); + + [DllImport("user32.dll")] + private static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam); + + [DllImport("user32.dll")] + private static extern bool IsWindowVisible(IntPtr hWnd); + + [DllImport("user32.dll")] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); + + // ── WASAPI audio session supplement ────────────────────────────────────── + + private static void AppendAudioSessionApps(HashSet seen, int selfPid, + List result) + { + IMMDeviceEnumerator? enumerator = null; + IMMDevice? device = null; + IAudioSessionManager2? manager = null; + IAudioSessionEnumerator? sessions = null; + + try + { + enumerator = (IMMDeviceEnumerator)Activator.CreateInstance( + Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!; + + enumerator.GetDefaultAudioEndpoint(0 /*eRender*/, 1 /*eMultimedia*/, out device); + + var mgr2Iid = new Guid("77AA99A0-1BD6-484F-8BC7-2C654C9A9B6F"); + device.Activate(ref mgr2Iid, 0x17 /*CLSCTX_ALL*/, IntPtr.Zero, out object mgr); + manager = (IAudioSessionManager2)mgr; + + manager.GetSessionEnumerator(out sessions); + sessions.GetCount(out int count); + + for (int i = 0; i < count; i++) + { + IAudioSessionControl? ctrl = null; + try + { + sessions.GetSession(i, out ctrl); + var ctrl2 = (IAudioSessionControl2)ctrl; + ctrl2.GetProcessId(out uint pid); + + int ipid = (int)pid; + if (pid == 0 || ipid == selfPid || !seen.Add(ipid)) continue; + + try + { + var proc = Process.GetProcessById(ipid); + if (!string.IsNullOrEmpty(proc.ProcessName)) + result.Add(new AudioAppInfo(ipid, proc.ProcessName)); + } + catch { /* exited */ } + } + catch { /* stale session */ } + finally { if (ctrl != null) Marshal.ReleaseComObject(ctrl); } + } + } + catch { /* no audio device or WASAPI unavailable — ignore */ } + finally + { + if (sessions != null) Marshal.ReleaseComObject(sessions); + if (manager != null) Marshal.ReleaseComObject(manager); + if (device != null) Marshal.ReleaseComObject(device); + if (enumerator != null) Marshal.ReleaseComObject(enumerator); + } + } + + // ── COM interface declarations ───────────────────────────────────────────── + + [ComImport, Guid("A95664D2-9614-4F35-A746-DE8DB63617E6"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IMMDeviceEnumerator + { + [PreserveSig] int EnumAudioEndpoints(int dataFlow, int stateMask, out IntPtr devices); + [PreserveSig] int GetDefaultAudioEndpoint(int dataFlow, int role, out IMMDevice endpoint); + [PreserveSig] int GetDevice([MarshalAs(UnmanagedType.LPWStr)] string id, out IMMDevice device); + [PreserveSig] int RegisterEndpointNotificationCallback(IntPtr client); + [PreserveSig] int UnregisterEndpointNotificationCallback(IntPtr client); + } + + [ComImport, Guid("D666063F-1587-4E43-81F1-B948E807363F"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IMMDevice + { + [PreserveSig] int Activate(ref Guid iid, int clsCtx, IntPtr activationParams, + [MarshalAs(UnmanagedType.IUnknown)] out object ppInterface); + [PreserveSig] int OpenPropertyStore(int stgmAccess, out IntPtr propStore); + [PreserveSig] int GetId([MarshalAs(UnmanagedType.LPWStr)] out string id); + [PreserveSig] int GetState(out int state); + } + + [ComImport, Guid("77AA99A0-1BD6-484F-8BC7-2C654C9A9B6F"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioSessionManager2 + { + [PreserveSig] int GetAudioSessionControl(ref Guid audioSessionGuid, int streamFlags, + out IAudioSessionControl session); + [PreserveSig] int GetSimpleAudioVolume(ref Guid audioSessionGuid, int streamFlags, + out IntPtr audioVolume); + [PreserveSig] int GetSessionEnumerator(out IAudioSessionEnumerator sessionEnum); + [PreserveSig] int RegisterSessionNotification(IntPtr notification); + [PreserveSig] int UnregisterSessionNotification(IntPtr notification); + [PreserveSig] int RegisterDuckNotification([MarshalAs(UnmanagedType.LPWStr)] string sessionID, + IntPtr notification); + [PreserveSig] int UnregisterDuckNotification(IntPtr notification); + } + + [ComImport, Guid("E2F5BB11-0570-40CA-ACDD-3AA01277DEE8"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioSessionEnumerator + { + [PreserveSig] int GetCount(out int sessionCount); + [PreserveSig] int GetSession(int sessionIndex, out IAudioSessionControl session); + } + + [ComImport, Guid("F4B1A599-7266-4319-A8CA-E70ACB11E8CD"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioSessionControl + { + [PreserveSig] int GetState(out int state); + [PreserveSig] int GetDisplayName([MarshalAs(UnmanagedType.LPWStr)] out string name); + [PreserveSig] int SetDisplayName([MarshalAs(UnmanagedType.LPWStr)] string name, + ref Guid eventContext); + [PreserveSig] int GetIconPath([MarshalAs(UnmanagedType.LPWStr)] out string iconPath); + [PreserveSig] int SetIconPath([MarshalAs(UnmanagedType.LPWStr)] string iconPath, + ref Guid eventContext); + [PreserveSig] int GetGroupingParam(out Guid groupingParam); + [PreserveSig] int SetGroupingParam(ref Guid groupingParam, ref Guid eventContext); + [PreserveSig] int RegisterAudioSessionNotification(IntPtr notification); + [PreserveSig] int UnregisterAudioSessionNotification(IntPtr notification); + } + + [ComImport, Guid("BFB7FF88-7239-4FC9-8FA2-07C950BE9C6D"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioSessionControl2 : IAudioSessionControl + { + [PreserveSig] int GetSessionIdentifier([MarshalAs(UnmanagedType.LPWStr)] out string id); + [PreserveSig] int GetSessionInstanceIdentifier([MarshalAs(UnmanagedType.LPWStr)] out string id); + [PreserveSig] int GetProcessId(out uint pid); + [PreserveSig] int IsSystemSoundsSession(); + [PreserveSig] int SetDuckingPreference(bool optOut); + } +} diff --git a/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs b/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs new file mode 100644 index 0000000..04ed01c --- /dev/null +++ b/clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs @@ -0,0 +1,146 @@ +using VoiceCat.Interop; + +// Owns N ProcessLoopbackCapture instances, mixes their PCM every 20 ms, and feeds +// the result to the core via vc_stream_feed_pcm. Used for per-app audio sharing. +namespace VoiceCat.App.Audio; + +public sealed class ProcessAudioMixer : IDisposable +{ + private const int SampleRate = 48000; + private const int FrameSamples = 960; + private const int Channels = 2; // stereo; captures fall back to mono if needed + + private readonly List _captures = []; + + // Per-capture latest frame, protected by _frameLock. + private readonly object _frameLock = new(); + private List _latestFrames = []; + private int _activeChannels = Channels; + + private Thread? _mixThread; + private volatile bool _running; + private VoiceCatClient? _client; + private uint _streamId; + + public void Start(AppAudioScope scope, VoiceCatClient client, uint streamId, + IReadOnlyList allApps) + { + if (_running) return; + _client = client; + _streamId = streamId; + + var pids = ResolvePids(scope, allApps); + if (pids.Count == 0) + { + // nothing to capture — scope resolved to empty set + return; + } + + lock (_frameLock) + { + _latestFrames = new List(new short[pids.Count][]); + _activeChannels = Channels; + } + + for (int i = 0; i < pids.Count; i++) + { + int captureIndex = i; + var cap = new ProcessLoopbackCapture(pids[i], ProcessLoopbackCapture.Mode.Include); + cap.PcmFrameReady += (pcm, spc, ch) => OnCaptureFrame(captureIndex, pcm, ch); + _captures.Add(cap); + } + + foreach (var c in _captures) c.Start(); + + _running = true; + _mixThread = new Thread(MixLoop) { IsBackground = true, Name = "ProcessAudioMixer" }; + _mixThread.Start(); + } + + public void Stop() + { + _running = false; + _mixThread?.Join(500); + foreach (var c in _captures) { c.Stop(); c.Dispose(); } + _captures.Clear(); + } + + public void Dispose() => Stop(); + + // ── Capture callback ────────────────────────────────────────────────────── + + private void OnCaptureFrame(int index, short[] pcm, int channels) + { + lock (_frameLock) + { + // Upmix mono → stereo interleave if the capture fell back to mono. + if (channels == 1 && _activeChannels == 2) + pcm = MonoToStereo(pcm); + if (index < _latestFrames.Count) + _latestFrames[index] = pcm; + } + } + + // ── Mix loop (20 ms timer) ──────────────────────────────────────────────── + + private void MixLoop() + { + // Use a target period close to 20 ms; small under-shoot avoids accumulating drift. + const int periodMs = 19; + while (_running) + { + Thread.Sleep(periodMs); + if (!_running) break; + + short[] mix; + lock (_frameLock) + { + int len = FrameSamples * _activeChannels; + mix = new short[len]; + + foreach (var frame in _latestFrames) + { + if (frame == null) continue; + int frameLen = Math.Min(frame.Length, len); + for (int i = 0; i < frameLen; i++) + { + int sum = mix[i] + frame[i]; + // Saturating clamp + mix[i] = (short)Math.Clamp(sum, short.MinValue, short.MaxValue); + } + } + } + + _client?.StreamFeedPcm(_streamId, mix, FrameSamples, (uint)_activeChannels); + } + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + // For "AllExcept": enumerate all running audio apps and exclude the specified ones. + // For "OnlyApps": use their PIDs directly. + private static List ResolvePids(AppAudioScope scope, IReadOnlyList allApps) + { + return scope switch + { + OnlyApps o => [.. o.Pids], + AllExceptApps a => + allApps + .Where(app => !a.Pids.Contains(app.Pid)) + .Select(app => app.Pid) + .ToList(), + _ => [], + }; + } + + private static short[] MonoToStereo(short[] mono) + { + var stereo = new short[mono.Length * 2]; + for (int i = 0; i < mono.Length; i++) + { + stereo[i * 2] = mono[i]; + stereo[i * 2 + 1] = mono[i]; + } + return stereo; + } +} diff --git a/clients/windows/VoiceCat.App/Audio/ProcessLoopbackCapture.cs b/clients/windows/VoiceCat.App/Audio/ProcessLoopbackCapture.cs new file mode 100644 index 0000000..a6efd0e --- /dev/null +++ b/clients/windows/VoiceCat.App/Audio/ProcessLoopbackCapture.cs @@ -0,0 +1,434 @@ +using System.Runtime.InteropServices; + +// Single-process WASAPI loopback capture via AUDIOCLIENT_ACTIVATION_PARAMS +// (Windows 10 2004+ / Build 19041+). +// +// Threading: ALL WASAPI init runs on the capture thread (MTA). If called from the +// WinForms UI thread (STA), ActivateAudioInterfaceAsync fires ActivateCompleted on +// an MTA pool thread; COM marshals that back to the STA pump — but the STA thread is +// blocked on CompletionEvent.Wait → deadlock. MTA capture thread avoids this. +// +// COM QI policy: the COM objects returned by the process-loopback activation path +// reject QueryInterface for their own IIDs under .NET's RCW mechanism. Every call +// to IAudioClient and IAudioCaptureClient is therefore dispatched via raw vtable +// pointer arithmetic, bypassing .NET COM interop entirely. +namespace VoiceCat.App.Audio; + +public sealed class ProcessLoopbackCapture : IDisposable +{ + public enum Mode { Include, Exclude } + + // Fired on the capture thread every 20 ms (960 samples/channel @ 48 kHz, interleaved s16). + public event Action? PcmFrameReady; + + private const int SampleRate = 48000; + private const int FrameSamples = 960; // 20 ms + private const string LoopbackDevicePath = "VAD\\Process_Loopback"; + + private readonly int _pid; + private readonly Mode _mode; + + // Raw COM pointers — managed via explicit AddRef/Release, no RCW wrapping. + private IntPtr _audioClientPtr; // IAudioClient* + private IntPtr _captureClientPtr; // IAudioCaptureClient* + + private AutoResetEvent? _bufferEvent; + private Thread? _captureThread; + private volatile bool _running; + private int _channels; + + // Accumulator: assembles driver-callback-sized fragments into FrameSamples chunks. + private short[] _accumBuf = []; + private int _accumCount; + + // Init-done signal: Set() by the capture thread after ActivateClient() completes. + private readonly ManualResetEventSlim _initDone = new(false); + private bool _initOk; + + public ProcessLoopbackCapture(int pid, Mode mode) + { + _pid = pid; + _mode = mode; + } + + /// Starts capture. Blocks until WASAPI activation completes (typically <100 ms). + /// Returns false if the process cannot be captured. + public bool Start() + { + if (_running) return false; + _running = true; + _captureThread = new Thread(CaptureThreadProc) + { + IsBackground = true, + Name = $"ProcLoopback:{_pid}", + }; + _captureThread.Start(); + + bool ok = _initDone.Wait(5000) && _initOk; + if (!ok) _running = false; + return ok; + } + + public void Stop() + { + _running = false; + _bufferEvent?.Set(); + _captureThread?.Join(500); + if (_audioClientPtr != IntPtr.Zero) AC_Stop(_audioClientPtr); + } + + public void Dispose() + { + Stop(); + ComRelease(ref _captureClientPtr); + ComRelease(ref _audioClientPtr); + _bufferEvent?.Dispose(); + _initDone.Dispose(); + } + + // ── Capture thread (MTA) ────────────────────────────────────────────────── + + private void CaptureThreadProc() + { + _initOk = ActivateAndStart(); + _initDone.Set(); + if (!_initOk) return; + CaptureLoop(); + } + + private bool ActivateAndStart() + { + if (!ActivateClient()) return false; + + _bufferEvent = new AutoResetEvent(false); + if (AC_SetEventHandle(_audioClientPtr, _bufferEvent.SafeWaitHandle.DangerousGetHandle()) < 0) + return false; + + return AC_Start(_audioClientPtr) >= 0; + } + + // ── Activation ─────────────────────────────────────────────────────────── + + private unsafe bool ActivateClient() + { + var activationParams = new AudioClientActivationParams + { + ActivationType = 1, // AUDCLNT_ACTIVATIONTYPE_PROCESS_LOOPBACK + TargetProcessId = (uint)_pid, + ProcessLoopbackMode = _mode == Mode.Include ? 0u : 1u, + }; + + var handler = new ActivationCompletionHandler(); + var audioClientIid = new Guid("1CB9AD4C-DBFA-4c32-B178-C2F568A703B2"); + + IntPtr opPtr; + { + AudioClientActivationParams* pParams = &activationParams; + // PROPVARIANT (VT_BLOB) x64: vt(2)+res(6)+cbSize(4)+pad(4)+pBlobData(8) = 24 B. + var pv = stackalloc byte[24]; + *(ushort*)(pv + 0) = 65; + *(uint*) (pv + 8) = (uint)sizeof(AudioClientActivationParams); + *(nint*) (pv + 16) = (nint)pParams; + + int hr = ActivateAudioInterfaceAsync( + LoopbackDevicePath, ref audioClientIid, + (IntPtr)pv, handler, out opPtr); + if (hr < 0) return false; + } + + if (!handler.CompletionEvent.Wait(3000)) + { + ComRelease(ref opPtr); + return false; + } + + // Call GetActivateResult via vtable (slot 3) — avoids QI on the async-op object. + if (!Vtable_GetActivateResult(handler.OperationPtr, out int activateHr, out IntPtr activatedPtr)) + { + handler.ReleaseOp(); + ComRelease(ref opPtr); + return false; + } + handler.ReleaseOp(); + ComRelease(ref opPtr); + + if (activateHr < 0 || activatedPtr == IntPtr.Zero) return false; + + // Store the raw IAudioClient* — do NOT create an RCW; use vtable dispatch instead. + _audioClientPtr = activatedPtr; + // activatedPtr already has ref count from GetActivateResult; don't double-release. + + return InitializeStream(); + } + + // IActivateAudioInterfaceAsyncOperation vtable slot 3: GetActivateResult(HRESULT*, IUnknown**) + private static unsafe bool Vtable_GetActivateResult(IntPtr op, + out int activateHr, out IntPtr activatedPtr) + { + activateHr = unchecked((int)0x80004005); + activatedPtr = IntPtr.Zero; + if (op == IntPtr.Zero) return false; + void** vtable = *(void***)op.ToPointer(); + var fn = (delegate* unmanaged[Stdcall])vtable[3]; + fixed (int* pHr = &activateHr) + fixed (IntPtr* pPtr = &activatedPtr) + return fn(op, pHr, pPtr) >= 0; + } + + private unsafe bool InitializeStream() + { + // Try s16 stereo first; fall back to s16 mono. + foreach (int ch in new[] { 2, 1 }) + { + var fmt = new WaveFormatEx + { + wFormatTag = 1, // WAVE_FORMAT_PCM + nChannels = (ushort)ch, + nSamplesPerSec = SampleRate, + wBitsPerSample = 16, + nBlockAlign = (ushort)(ch * 2), + nAvgBytesPerSec = (uint)(SampleRate * ch * 2), + cbSize = 0, + }; + + // Process-loopback requires LOOPBACK (deliver rendered audio) + EVENTCALLBACK + + // AUTOCONVERTPCM (resample the app's native format to our requested s16/48k). Without + // LOOPBACK every buffer comes back AUDCLNT_BUFFERFLAGS_SILENT; without AUTOCONVERTPCM + // the requested format is rejected. Matches the MS ApplicationLoopback sample. + // AUDCLNT_SHAREMODE_SHARED = 0 + // AUDCLNT_STREAMFLAGS_LOOPBACK = 0x00020000 + // AUDCLNT_STREAMFLAGS_EVENTCALLBACK = 0x00040000 + // AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM = 0x80000000 + const uint streamFlags = 0x00020000u | 0x00040000u | 0x80000000u; + int hr = AC_Initialize(_audioClientPtr, 0, streamFlags, + 2_000_000 /*200ms hns*/, 0, &fmt, null); + if (hr >= 0) + { + _channels = ch; + _accumBuf = new short[FrameSamples * ch]; + _accumCount = 0; + break; + } + if (ch == 1) return false; + } + + var captureIid = new Guid("C8ADBD64-E71E-48a0-A4DE-185C395CD317"); + int getHr = AC_GetService(_audioClientPtr, ref captureIid, out _captureClientPtr); + return getHr >= 0 && _captureClientPtr != IntPtr.Zero; + } + + // ── Capture loop ───────────────────────────────────────────────────────── + + private void CaptureLoop() + { + while (_running) + { + _bufferEvent!.WaitOne(100); + if (!_running) break; + + while (_running) + { + int hr = CC_GetNextPacketSize(_captureClientPtr, out uint packetSize); + if (hr < 0 || packetSize == 0) break; + + hr = CC_GetBuffer(_captureClientPtr, out IntPtr dataPtr, out uint framesAvailable, + out uint flags); + if (hr < 0) break; + + bool silent = (flags & 2) != 0; // AUDCLNT_BUFFERFLAGS_SILENT + if (framesAvailable > 0) + { + if (silent) AccumulateSilence((int)framesAvailable); + else AccumulatePcm(dataPtr, (int)framesAvailable); + } + + CC_ReleaseBuffer(_captureClientPtr, framesAvailable); + } + } + } + + private unsafe void AccumulatePcm(IntPtr data, int frames) + { + var src = (short*)data.ToPointer(); + int total = frames * _channels; + int idx = 0; + while (idx < total) + { + int space = _accumBuf.Length - _accumCount; + int copy = Math.Min(total - idx, space); + fixed (short* dst = _accumBuf) + Buffer.MemoryCopy(src + idx, dst + _accumCount, copy * 2L, copy * 2L); + _accumCount += copy; + idx += copy; + if (_accumCount == _accumBuf.Length) + FlushFrame(); + } + } + + private void AccumulateSilence(int frames) + { + int total = frames * _channels; + int idx = 0; + while (idx < total) + { + int space = _accumBuf.Length - _accumCount; + int fill = Math.Min(total - idx, space); + Array.Clear(_accumBuf, _accumCount, fill); + _accumCount += fill; + idx += fill; + if (_accumCount == _accumBuf.Length) + FlushFrame(); + } + } + + private void FlushFrame() + { + var copy = new short[_accumBuf.Length]; + _accumBuf.AsSpan().CopyTo(copy); + PcmFrameReady?.Invoke(copy, FrameSamples, _channels); + _accumCount = 0; + } + + // ── IAudioClient vtable helpers (raw dispatch, no RCW / no QI) ─────────── + // + // Vtable layout (IUnknown base: 0=QI 1=AddRef 2=Release; then IAudioClient methods): + // 3=Initialize 4=GetBufferSize 5=GetStreamLatency 6=GetCurrentPadding + // 7=IsFormatSupported 8=GetMixFormat 9=GetDevicePeriod + // 10=Start 11=Stop 12=Reset 13=SetEventHandle 14=GetService + + private static unsafe int AC_Initialize(IntPtr ac, int shareMode, uint streamFlags, + long hnsBufferDuration, long hnsPeriodicity, WaveFormatEx* pFormat, Guid* pSession) + { + var fn = (delegate* unmanaged[Stdcall]) + (*(void***)ac)[3]; + return fn(ac, shareMode, streamFlags, hnsBufferDuration, hnsPeriodicity, pFormat, pSession); + } + + private static unsafe int AC_Start(IntPtr ac) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)ac)[10]; + return fn(ac); + } + + private static unsafe int AC_Stop(IntPtr ac) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)ac)[11]; + return fn(ac); + } + + private static unsafe int AC_SetEventHandle(IntPtr ac, IntPtr eventHandle) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)ac)[13]; + return fn(ac, eventHandle); + } + + private static unsafe int AC_GetService(IntPtr ac, ref Guid riid, out IntPtr ppv) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)ac)[14]; + fixed (Guid* pIid = &riid) + fixed (IntPtr* pPpv = &ppv) + return fn(ac, pIid, pPpv); + } + + // ── IAudioCaptureClient vtable helpers ──────────────────────────────────── + // + // Vtable (IUnknown: 0-2; then): 3=GetBuffer 4=ReleaseBuffer 5=GetNextPacketSize + + private static unsafe int CC_GetBuffer(IntPtr cc, out IntPtr ppData, + out uint pNumFrames, out uint pdwFlags) + { + var fn = (delegate* unmanaged[Stdcall]) + (*(void***)cc)[3]; + ulong devPos = 0, qpcPos = 0; + fixed (IntPtr* p0 = &ppData) + fixed (uint* p1 = &pNumFrames) + fixed (uint* p2 = &pdwFlags) + return fn(cc, p0, p1, p2, &devPos, &qpcPos); + } + + private static unsafe int CC_ReleaseBuffer(IntPtr cc, uint numFrames) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)cc)[4]; + return fn(cc, numFrames); + } + + private static unsafe int CC_GetNextPacketSize(IntPtr cc, out uint pNumFrames) + { + var fn = (delegate* unmanaged[Stdcall])(*(void***)cc)[5]; + fixed (uint* p = &pNumFrames) + return fn(cc, p); + } + + // ── COM utilities ───────────────────────────────────────────────────────── + + private static unsafe void ComRelease(ref IntPtr ptr) + { + if (ptr == IntPtr.Zero) return; + var fn = (delegate* unmanaged[Stdcall])(*(void***)ptr)[2]; // IUnknown::Release + fn(ptr); + ptr = IntPtr.Zero; + } + + // ── P/Invoke & structs ──────────────────────────────────────────────────── + + [DllImport("Mmdevapi.dll", CharSet = CharSet.Unicode)] + private static extern int ActivateAudioInterfaceAsync( + string deviceInterfacePath, + ref Guid riid, + IntPtr activationParams, + [MarshalAs(UnmanagedType.Interface)] IActivateAudioInterfaceCompletionHandler completionHandler, + out IntPtr activationOperation); + + [StructLayout(LayoutKind.Sequential)] + private struct AudioClientActivationParams + { + public int ActivationType; // AUDCLNT_ACTIVATIONTYPE_PROCESS_LOOPBACK = 1 + public uint TargetProcessId; + public uint ProcessLoopbackMode; // INCLUDE=0, EXCLUDE=1 + } + + [StructLayout(LayoutKind.Sequential, Pack = 2)] + private struct WaveFormatEx + { + public ushort wFormatTag; + public ushort nChannels; + public uint nSamplesPerSec; + public uint nAvgBytesPerSec; + public ushort nBlockAlign; + public ushort wBitsPerSample; + public ushort cbSize; + } + + // ── Completion handler CCW ──────────────────────────────────────────────── + // + // Only this object still uses .NET COM interop (as a CCW). The activateOperation + // parameter is IntPtr to avoid QI on the incoming async-op pointer. + + [ComVisible(true), ClassInterface(ClassInterfaceType.None)] + private sealed class ActivationCompletionHandler : IActivateAudioInterfaceCompletionHandler + { + public readonly ManualResetEventSlim CompletionEvent = new(false); + public IntPtr OperationPtr { get; private set; } + + public void ActivateCompleted(IntPtr activateOperation) + { + OperationPtr = activateOperation; + if (OperationPtr != IntPtr.Zero) Marshal.AddRef(OperationPtr); + CompletionEvent.Set(); + } + + public void ReleaseOp() + { + if (OperationPtr == IntPtr.Zero) return; + Marshal.Release(OperationPtr); + OperationPtr = IntPtr.Zero; + } + } + + [ComImport, Guid("41D949AB-9862-444A-80F6-C261334DA5EB"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface IActivateAudioInterfaceCompletionHandler + { + void ActivateCompleted(IntPtr activateOperation); + } +} diff --git a/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs b/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs new file mode 100644 index 0000000..ac06e38 --- /dev/null +++ b/clients/windows/VoiceCat.App/Forms/AppAudioPickerDialog.cs @@ -0,0 +1,166 @@ +using VoiceCat.App.Audio; + +namespace VoiceCat.App.Forms; + +/// +/// Modal dialog for selecting which apps' audio to share. +/// Returns the chosen via , +/// or if the user dismissed without sharing. +/// +public sealed class AppAudioPickerDialog : Form +{ + private readonly RadioButton _rdoAll; + private readonly RadioButton _rdoOnly; + private readonly RadioButton _rdoExcept; + private readonly ListView _appList; + private readonly Label _lblApps; + + // Snapshot taken when the dialog opens (refresh on open, not on every check change). + private IReadOnlyList _apps = []; + + public AppAudioScope? ChosenScope { get; private set; } + + public AppAudioPickerDialog() + { + // ── Radio buttons ─────────────────────────────────────────────────── + _rdoAll = new RadioButton + { + Text = "&Entire desktop", + Checked = true, + Location = new Point(12, 12), + Size = new Size(360, 20), + TabIndex = 0, + }; + _rdoOnly = new RadioButton + { + Text = "&Only selected apps", + Location = new Point(12, 36), + Size = new Size(360, 20), + TabIndex = 1, + }; + _rdoExcept = new RadioButton + { + Text = "All apps e&xcept selected", + Location = new Point(12, 60), + Size = new Size(360, 20), + TabIndex = 2, + }; + + _rdoAll.CheckedChanged += OnModeChanged; + _rdoOnly.CheckedChanged += OnModeChanged; + _rdoExcept.CheckedChanged += OnModeChanged; + + // ── App list ──────────────────────────────────────────────────────── + _lblApps = new Label + { + Text = "Apps with active audio sessions:", + AutoSize = true, + Location = new Point(12, 90), + Visible = false, + TabIndex = 3, + }; + + _appList = new ListView + { + Location = new Point(12, 112), + Size = new Size(360, 180), + CheckBoxes = true, + View = View.List, + Visible = false, + TabIndex = 4, + FullRowSelect = true, + }; + + // ── Buttons ───────────────────────────────────────────────────────── + var btnShare = new Button + { + Text = "&Share", + DialogResult = DialogResult.OK, + Location = new Point(216, 308), + Size = new Size(75, 27), + TabIndex = 5, + }; + var btnCancel = new Button + { + Text = "&Cancel", + DialogResult = DialogResult.Cancel, + Location = new Point(297, 308), + Size = new Size(75, 27), + TabIndex = 6, + }; + btnShare.Click += OnShareClick; + + AcceptButton = btnShare; + CancelButton = btnCancel; + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(384, 348); + Controls.AddRange([_rdoAll, _rdoOnly, _rdoExcept, _lblApps, _appList, btnShare, btnCancel]); + FormBorderStyle = FormBorderStyle.FixedDialog; + MaximizeBox = false; + MinimizeBox = false; + StartPosition = FormStartPosition.CenterParent; + Text = "Share App Audio"; + } + + protected override void OnLoad(EventArgs e) + { + base.OnLoad(e); + RefreshAppList(); + } + + private void RefreshAppList() + { + _apps = AudioSessionEnumerator.GetAudioApps(); + _appList.Items.Clear(); + foreach (var app in _apps) + _appList.Items.Add(new ListViewItem($"{app.DisplayName} (PID {app.Pid})") { Tag = app.Pid }); + } + + private void OnModeChanged(object? sender, EventArgs e) + { + bool showList = _rdoOnly.Checked || _rdoExcept.Checked; + _lblApps.Visible = showList; + _appList.Visible = showList; + + if (showList && _appList.Items.Count == 0) + RefreshAppList(); + } + + private void OnShareClick(object? sender, EventArgs e) + { + if (_rdoAll.Checked) + { + ChosenScope = new EntireDesktop(); + return; + } + + var checkedPids = new List(); + var checkedNames = new List(); + foreach (ListViewItem item in _appList.CheckedItems) + { + if (item.Tag is int pid) + { + checkedPids.Add(pid); + checkedNames.Add(item.Text); + } + } + + if (checkedPids.Count == 0) + { + MessageBox.Show( + "Select at least one app, or choose 'Entire desktop'.", + "No apps selected", + MessageBoxButtons.OK, + MessageBoxIcon.Warning); + DialogResult = DialogResult.None; // prevent close + return; + } + + ChosenScope = _rdoOnly.Checked + ? new OnlyApps(checkedPids, checkedNames) + : new AllExceptApps(checkedPids, checkedNames); + } + + /// All apps that were visible in the list when the user clicked Share. + public IReadOnlyList VisibleApps => _apps; +} diff --git a/clients/windows/VoiceCat.App/Forms/MainForm.cs b/clients/windows/VoiceCat.App/Forms/MainForm.cs index a3277f4..f4bfe22 100644 --- a/clients/windows/VoiceCat.App/Forms/MainForm.cs +++ b/clients/windows/VoiceCat.App/Forms/MainForm.cs @@ -1,3 +1,4 @@ +using VoiceCat.App.Audio; using VoiceCat.Interop; namespace VoiceCat.App.Forms; @@ -22,6 +23,7 @@ public partial class MainForm : Form // Voice state private uint _micStreamId; // 0 = not started private uint _screenStreamId; // 0 = not sharing screen audio + private ProcessAudioMixer? _screenMixer; // non-null only in per-app capture mode private Keys _pttKey = Keys.F8; private bool _serverMuted; private bool _serverDeafened; @@ -442,6 +444,7 @@ public partial class MainForm : Form _currentChannelId = 0; _micStreamId = 0; _screenStreamId = 0; + _screenMixer?.Stop(); _screenMixer?.Dispose(); _screenMixer = null; txtCompose.Enabled = false; btnSend.Enabled = false; tsbJoinVoice.Enabled = false; @@ -622,29 +625,73 @@ public partial class MainForm : Form { if (_screenStreamId == 0) { - var (result, streamId) = _client.StartStream(VcStreamKind.ScreenAudio, "Desktop audio"); - if (result == VcResult.Ok) - { - _screenStreamId = streamId; - tsbScreenShare.Text = "Stop Screen Audio"; - _miScreenShare.Text = "Stop Screen &Audio"; - AddActivity("Started sharing screen audio"); - } - else - { - AddActivity($"Failed to start screen audio: {result}"); - } + StartScreenAudio(); } else { - _client.StopStream(_screenStreamId); - _screenStreamId = 0; - tsbScreenShare.Text = "Share Screen Audio"; - _miScreenShare.Text = "Share Screen &Audio"; - AddActivity("Stopped sharing screen audio"); + StopScreenAudio(); } } + private void StartScreenAudio() + { + using var picker = new AppAudioPickerDialog(); + if (picker.ShowDialog(this) != DialogResult.OK || picker.ChosenScope == null) return; + + var scope = picker.ChosenScope; + + if (scope is EntireDesktop) + { + // Existing whole-device WASAPI loopback path — core handles it. + var (result, streamId) = _client.StartStream(VcStreamKind.ScreenAudio, "Desktop audio"); + if (result != VcResult.Ok) + { + AddActivity($"Failed to start screen audio: {result}"); + return; + } + _screenStreamId = streamId; + AddActivity("Sharing screen audio: entire desktop"); + } + else + { + // Per-app path: suppress core loopback, C# mixer feeds PCM. + var (result, streamId) = _client.StartStreamExternalFeed(VcStreamKind.ScreenAudio, "App audio"); + if (result != VcResult.Ok) + { + AddActivity($"Failed to start screen audio: {result}"); + return; + } + _screenStreamId = streamId; + + _screenMixer = new ProcessAudioMixer(); + _screenMixer.Start(scope, _client, streamId, picker.VisibleApps); + + string desc = scope switch + { + OnlyApps o => $"only {o.Pids.Count} app(s)", + AllExceptApps a => $"all except {a.Pids.Count} app(s)", + _ => "apps", + }; + AddActivity($"Sharing screen audio: {desc}"); + } + + tsbScreenShare.Text = "Stop Screen Audio"; + _miScreenShare.Text = "Stop Screen &Audio"; + } + + private void StopScreenAudio() + { + _screenMixer?.Stop(); + _screenMixer?.Dispose(); + _screenMixer = null; + + _client.StopStream(_screenStreamId); + _screenStreamId = 0; + tsbScreenShare.Text = "Share Screen Audio"; + _miScreenShare.Text = "Share Screen &Audio"; + AddActivity("Stopped sharing screen audio"); + } + private void TrkOutputVolume_Scroll(object? sender, EventArgs e) => _client.SetOutputVolume(trkOutputVolume.Value / 100f); @@ -1019,7 +1066,7 @@ public partial class MainForm : Form _client.EventReceived -= OnEvent; foreach (var win in _pmWindows.Values.ToList()) win.Close(); _pmWindows.Clear(); - if (_screenStreamId != 0) _client.StopStream(_screenStreamId); + if (_screenStreamId != 0) { _screenMixer?.Stop(); _screenMixer?.Dispose(); _screenMixer = null; _client.StopStream(_screenStreamId); } if (_micStreamId != 0) _client.StopStream(_micStreamId); _client.Disconnect(); _client.Dispose(); diff --git a/clients/windows/VoiceCat.App/VoiceCat.App.csproj b/clients/windows/VoiceCat.App/VoiceCat.App.csproj index c686b8f..b19953a 100644 --- a/clients/windows/VoiceCat.App/VoiceCat.App.csproj +++ b/clients/windows/VoiceCat.App/VoiceCat.App.csproj @@ -18,6 +18,9 @@ WinForms' own analyzer (WFO0003) flags manifest-based DPI settings as superseded by this property in modern .NET. --> PerMonitorV2 + + true