Port managed client audio and Windows application

clients/windows/VoiceCat.App/Audio/InputDeviceCapture.cs

@@ -16,7 +16,7 @@ public sealed record InputDeviceInfo(string Id, string Name, bool IsDefault)
 /// the aux device is opened client-side and needs a WASAPI id, not a miniaudio one.</summary>
 public static class InputDeviceEnumerator
 {
-    public static IReadOnlyList<InputDeviceInfo> List()
+    public static IReadOnlyList<InputDeviceInfo> List(bool input = true)
     {
         var result = new List<InputDeviceInfo>();
         InputDeviceCapture.IMMDeviceEnumerator? enumerator = null;
@@ -29,7 +29,7 @@ public static class InputDeviceEnumerator
                 Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!;
 
             // Resolve the default capture endpoint id so the picker can flag it.
-            if (enumerator.GetDefaultAudioEndpoint(1 /*eCapture*/, 0 /*eConsole*/,
+            if (enumerator.GetDefaultAudioEndpoint(input ? 1 : 0, 0 /*eConsole*/,
                     out var defDev) == 0 && defDev != null)
             {
                 try { if (defDev.GetId(out string id) == 0) defaultId = id; }
@@ -37,7 +37,7 @@ public static class InputDeviceEnumerator
             }
 
             // DEVICE_STATE_ACTIVE = 0x1 — only currently-usable endpoints.
-            if (enumerator.EnumAudioEndpoints(1 /*eCapture*/, 0x1, out collectionPtr) != 0
+            if (enumerator.EnumAudioEndpoints(input ? 1 : 0, 0x1, out collectionPtr) != 0
                 || collectionPtr == IntPtr.Zero)
                 return result;
 
@@ -107,6 +107,7 @@ public sealed class InputDeviceCapture : IDisposable
     private const int FrameSamples = 960;  // 20 ms
 
     private readonly string? _deviceId;  // null = system default capture endpoint
+    private readonly bool _loopback;
 
     private IAudioClient?        _audioClient;
     private IAudioCaptureClient? _captureClient;
@@ -123,8 +124,9 @@ public sealed class InputDeviceCapture : IDisposable
     // Init-done signal: Set() by the capture thread after activation completes.
     private readonly ManualResetEventSlim _initDone = new(false);
     private bool _initOk;
+    private int _disposed;
 
-    public InputDeviceCapture(string? deviceId) => _deviceId = deviceId;
+    public InputDeviceCapture(string? deviceId, bool loopback = false) { _deviceId = deviceId; _loopback = loopback; }
 
     /// <summary>Starts capture. Blocks until WASAPI activation completes (typically &lt;100 ms).
     /// Returns false if the device cannot be opened.</summary>
@@ -148,15 +150,13 @@ public sealed class InputDeviceCapture : IDisposable
     {
         _running = false;
         _bufferEvent?.Set();
-        _captureThread?.Join(500);
-        try { _audioClient?.Stop(); } catch { /* device already gone */ }
+        if (_captureThread is not null && !_captureThread.Join(5000)) throw new TimeoutException("Capture did not stop.");
     }
 
     public void Dispose()
     {
+        if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
         Stop();
-        if (_captureClient != null) { Marshal.ReleaseComObject(_captureClient); _captureClient = null; }
-        if (_audioClient   != null) { Marshal.ReleaseComObject(_audioClient);   _audioClient   = null; }
         _bufferEvent?.Dispose();
         _initDone.Dispose();
     }
@@ -165,10 +165,19 @@ public sealed class InputDeviceCapture : IDisposable
 
     private void CaptureThreadProc()
     {
-        _initOk = ActivateAndStart();
-        _initDone.Set();
-        if (!_initOk) return;
-        CaptureLoop();
+        try
+        {
+            _initOk = ActivateAndStart();
+            _initDone.Set();
+            if (_initOk && _running) CaptureLoop();
+        }
+        catch { _initOk = false; _initDone.Set(); }
+        finally
+        {
+            try { _audioClient?.Stop(); } catch { }
+            if (_captureClient != null) { Marshal.ReleaseComObject(_captureClient); _captureClient = null; }
+            if (_audioClient != null) { Marshal.ReleaseComObject(_audioClient); _audioClient = null; }
+        }
     }
 
     private bool ActivateAndStart()
@@ -192,7 +201,7 @@ public sealed class InputDeviceCapture : IDisposable
                 Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!;
 
             int hr = _deviceId is null
-                ? enumerator.GetDefaultAudioEndpoint(1 /*eCapture*/, 0 /*eConsole*/, out device)
+                ? enumerator.GetDefaultAudioEndpoint(_loopback ? 0 : 1, 0 /*eConsole*/, out device)
                 : enumerator.GetDevice(_deviceId, out device);
             if (hr != 0 || device == null) return false;
 
@@ -220,7 +229,7 @@ public sealed class InputDeviceCapture : IDisposable
         //   AUDCLNT_STREAMFLAGS_EVENTCALLBACK    = 0x00040000
         //   AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM   = 0x80000000
         //   AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY = 0x08000000
-        const uint streamFlags = 0x00040000u | 0x80000000u | 0x08000000u;
+        uint streamFlags = 0x00040000u | 0x80000000u | 0x08000000u | (_loopback ? 0x00020000u : 0u);
 
         foreach (int ch in new[] { 2, 1 })
         {
@@ -327,9 +336,7 @@ public sealed class InputDeviceCapture : IDisposable
 
     private void FlushFrame()
     {
-        var copy = new short[_accumBuf.Length];
-        _accumBuf.AsSpan().CopyTo(copy);
-        PcmFrameReady?.Invoke(copy, FrameSamples, _channels);
+        PcmFrameReady?.Invoke(_accumBuf, FrameSamples, _channels); // Borrowed until callback returns.
         _accumCount = 0;
     }
 

clients/windows/VoiceCat.App/Audio/ProcessAudioMixer.cs

@@ -1,148 +1,85 @@
+using VoiceCat.Audio;
 using VoiceCat.Interop;
 
-// Owns N ProcessLoopbackCapture instances, mixes their PCM every 20 ms, and feeds
-// the result to the core via vc_stream_feed_pcm. Used for per-app audio sharing.
 namespace VoiceCat.App.Audio;
 
+// Capture threads own their ring producers; the mix thread alone consumes them.
 public sealed class ProcessAudioMixer : IDisposable
 {
-    private const int SampleRate   = 48000;
-    private const int FrameSamples = 960;
-    private const int Channels     = 2; // stereo; captures fall back to mono if needed
-
-    private readonly List<ProcessLoopbackCapture> _captures = [];
-
-    // Per-capture latest frame, protected by _frameLock.
-    private readonly object _frameLock = new();
-    private List<short[]>   _latestFrames = [];
-    private int             _activeChannels = Channels;
-
-    private Thread?          _mixThread;
-    private volatile bool    _running;
-    private VoiceCatClient?  _client;
-    private uint             _streamId;
-
+    private sealed class Input
+    {
+        internal readonly PcmRing Ring = new(16384);
+        private readonly short[] stereo = new short[1920];
+        internal void Feed(short[] pcm, int channels)
+        {
+            if (channels == 2) { Ring.TryWrite(pcm); return; }
+            if (channels != 1 || pcm.Length > 960) return;
+            for (int i = 0; i < pcm.Length; i++) stereo[2 * i] = stereo[2 * i + 1] = pcm[i];
+            Ring.TryWrite(stereo.AsSpan(0, pcm.Length * 2));
+        }
+    }
+    private readonly List<ProcessLoopbackCapture> captures = [];
+    private Input[] inputs = [];
+    private Thread? thread;
+    private volatile bool running;
+    private VoiceCatClient? client;
+    private uint streamId;
     public void Start(AppAudioScope scope, VoiceCatClient client, uint streamId)
     {
-        if (_running) return;
-        _client   = client;
-        _streamId = streamId;
-
+        if (running) return;
+        this.client = client; this.streamId = streamId;
         var specs = ResolveCaptures(scope);
-        if (specs.Count == 0)
+        inputs = specs.Select(_ => new Input()).ToArray();
+        try
         {
-            // nothing to capture — scope resolved to empty set
-            return;
+            for (int i = 0; i < specs.Count; i++)
+            {
+                Input input = inputs[i]; var (pid, mode) = specs[i];
+                var capture = new ProcessLoopbackCapture(pid, mode);
+                capture.PcmFrameReady += (pcm, _, channels) => input.Feed(pcm, channels);
+                captures.Add(capture);
+                if (!capture.Start()) throw new InvalidOperationException("Process audio capture could not start.");
+            }
+            if (inputs.Length == 0) return;
+            running = true;
+            thread = new Thread(MixLoop) { IsBackground = true, Name = "ProcessAudioMixer" }; thread.Start();
         }
-
-        lock (_frameLock)
-        {
-            _latestFrames   = new List<short[]>(new short[specs.Count][]);
-            _activeChannels = Channels;
-        }
-
-        for (int i = 0; i < specs.Count; i++)
-        {
-            int captureIndex = i;
-            var (pid, mode) = specs[i];
-            var cap = new ProcessLoopbackCapture(pid, mode);
-            cap.PcmFrameReady += (pcm, spc, ch) => OnCaptureFrame(captureIndex, pcm, ch);
-            _captures.Add(cap);
-        }
-
-        foreach (var c in _captures) c.Start();
-
-        _running   = true;
-        _mixThread = new Thread(MixLoop) { IsBackground = true, Name = "ProcessAudioMixer" };
-        _mixThread.Start();
+        catch { Stop(); throw; }
     }
-
     public void Stop()
     {
-        _running = false;
-        _mixThread?.Join(500);
-        foreach (var c in _captures) { c.Stop(); c.Dispose(); }
-        _captures.Clear();
+        running = false;
+        if (thread is not null && !thread.Join(5000)) throw new TimeoutException("Process audio mixer did not stop.");
+        foreach (var capture in captures) capture.Dispose();
+        captures.Clear(); inputs = []; thread = null;
     }
-
     public void Dispose() => Stop();
-
-    // ── Capture callback ──────────────────────────────────────────────────────
-
-    private void OnCaptureFrame(int index, short[] pcm, int channels)
-    {
-        lock (_frameLock)
-        {
-            // Upmix mono → stereo interleave if the capture fell back to mono.
-            if (channels == 1 && _activeChannels == 2)
-                pcm = MonoToStereo(pcm);
-            if (index < _latestFrames.Count)
-                _latestFrames[index] = pcm;
-        }
-    }
-
-    // ── Mix loop (20 ms timer) ────────────────────────────────────────────────
-
     private void MixLoop()
     {
-        // Use a target period close to 20 ms; small under-shoot avoids accumulating drift.
-        const int periodMs = 19;
-        while (_running)
+        var frame = new short[1920]; var mix = new short[1920]; var sums = new int[1920];
+        long deadline = System.Diagnostics.Stopwatch.GetTimestamp();
+        while (running)
         {
-            Thread.Sleep(periodMs);
-            if (!_running) break;
-
-            short[] mix;
-            lock (_frameLock)
+            sums.AsSpan().Clear();
+            foreach (Input input in inputs)
             {
-                int len = FrameSamples * _activeChannels;
-                mix = new short[len];
-
-                foreach (var frame in _latestFrames)
-                {
-                    if (frame == null) continue;
-                    int frameLen = Math.Min(frame.Length, len);
-                    for (int i = 0; i < frameLen; i++)
-                    {
-                        int sum = mix[i] + frame[i];
-                        mix[i] = (short)Math.Clamp(sum, short.MinValue, short.MaxValue);
-                    }
-                }
+                while (input.Ring.Count > 11520) input.Ring.Read(frame);
+                frame.AsSpan().Clear(); input.Ring.Read(frame);
+                for (int i = 0; i < frame.Length; i++) sums[i] += frame[i];
             }
-
-            _client?.StreamFeedPcm(_streamId, mix, FrameSamples, (uint)_activeChannels);
+            for (int i = 0; i < mix.Length; i++) mix[i] = (short)Math.Clamp(sums[i], short.MinValue, short.MaxValue);
+            client?.StreamFeedPcm(streamId, mix, 960, 2);
+            deadline += System.Diagnostics.Stopwatch.Frequency / 50;
+            double wait = (deadline - System.Diagnostics.Stopwatch.GetTimestamp()) * 1000.0 / System.Diagnostics.Stopwatch.Frequency;
+            if (wait > 0) Thread.Sleep((int)Math.Ceiling(wait));
+            else if (wait < -100) deadline = System.Diagnostics.Stopwatch.GetTimestamp();
         }
     }
-
-    // ── Helpers ───────────────────────────────────────────────────────────────
-
-    // Resolve the scope to the WASAPI captures to open:
-    //  OnlyApps      → one INCLUDE capture per selected process tree.
-    //  AllExceptApps → one EXCLUDE capture of the single selected process tree, which
-    //                  natively captures the whole system render mix minus that tree
-    //                  (dynamic — apps launched later are included automatically).
-    private static List<(int pid, ProcessLoopbackCapture.Mode mode)> ResolveCaptures(AppAudioScope scope)
+    private static List<(int pid, ProcessLoopbackCapture.Mode mode)> ResolveCaptures(AppAudioScope scope) => scope switch
     {
-        return scope switch
-        {
-            OnlyApps      o => o.Pids.Select(p => (p, ProcessLoopbackCapture.Mode.Include)).ToList(),
-            AllExceptApps a when a.Pids.Count > 0 =>
-                [(a.Pids[0], ProcessLoopbackCapture.Mode.Exclude)],
-            // Entire desktop minus VoiceCat itself: EXCLUDE our own process tree.
-            EntireDesktop { ExcludeSelf: true } =>
-                [(Environment.ProcessId, ProcessLoopbackCapture.Mode.Exclude)],
-            _ => [],
-        };
-    }
-
-    private static short[] MonoToStereo(short[] mono)
-    {
-        var stereo = new short[mono.Length * 2];
-        for (int i = 0; i < mono.Length; i++)
-        {
-            stereo[i * 2]     = mono[i];
-            stereo[i * 2 + 1] = mono[i];
-        }
-        return stereo;
-    }
+        OnlyApps o => o.Pids.Select(p => (p, ProcessLoopbackCapture.Mode.Include)).ToList(),
+        AllExceptApps a when a.Pids.Count > 0 => [(a.Pids[0], ProcessLoopbackCapture.Mode.Exclude)],
+        EntireDesktop { ExcludeSelf: true } => [(Environment.ProcessId, ProcessLoopbackCapture.Mode.Exclude)],
+        _ => [],
+    };
 }

clients/windows/VoiceCat.App/Audio/ProcessLoopbackCapture.cs

@@ -64,15 +64,15 @@ public sealed class ProcessLoopbackCapture : IDisposable
     {
         _running = false;
         _bufferEvent?.Set();
-        _captureThread?.Join(500);
-        if (_audioClientPtr != IntPtr.Zero) AC_Stop(_audioClientPtr);
+        if (_captureThread is not null && !_captureThread.Join(5000))
+            throw new TimeoutException("Process capture did not stop.");
     }
 
+    private int _disposed;
     public void Dispose()
     {
+        if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
         Stop();
-        ComRelease(ref _captureClientPtr);
-        ComRelease(ref _audioClientPtr);
         _bufferEvent?.Dispose();
         _initDone.Dispose();
     }
@@ -81,10 +81,23 @@ public sealed class ProcessLoopbackCapture : IDisposable
 
     private void CaptureThreadProc()
     {
-        _initOk = ActivateAndStart();
-        _initDone.Set();
-        if (!_initOk) return;
-        CaptureLoop();
+        try
+        {
+            _initOk = ActivateAndStart();
+            _initDone.Set();
+            if (_initOk && _running) CaptureLoop();
+        }
+        catch
+        {
+            _initOk = false;
+            _initDone.Set();
+        }
+        finally
+        {
+            if (_audioClientPtr != IntPtr.Zero) AC_Stop(_audioClientPtr);
+            ComRelease(ref _captureClientPtr);
+            ComRelease(ref _audioClientPtr);
+        }
     }
 
     private bool ActivateAndStart()
@@ -274,9 +287,9 @@ public sealed class ProcessLoopbackCapture : IDisposable
 
     private void FlushFrame()
     {
-        var copy = new short[_accumBuf.Length];
-        _accumBuf.AsSpan().CopyTo(copy);
-        PcmFrameReady?.Invoke(copy, FrameSamples, _channels);
+        // The callback borrows this buffer until it returns. This capture owner cannot
+        // overwrite it concurrently, which avoids one allocation every 20 ms.
+        PcmFrameReady?.Invoke(_accumBuf, FrameSamples, _channels);
         _accumCount = 0;
     }
 

clients/windows/VoiceCat.App/Audio/WasapiAudioBackend.cs

@@ -0,0 +1,113 @@
+using System.Runtime.InteropServices;
+using VoiceCat.Audio;
+using static VoiceCat.App.Audio.InputDeviceCapture;
+
+namespace VoiceCat.App.Audio;
+
+public sealed class WasapiAudioBackend : IAudioDeviceBackend
+{
+    public IReadOnlyList<AudioDeviceInfo> Enumerate(bool input) => InputDeviceEnumerator.List(input).Select(d => new AudioDeviceInfo(d.Id, d.Name, d.IsDefault)).ToArray();
+    public IAudioCapture OpenCapture(string? deviceId, bool loopback, CapturePcmHandler handler)
+    {
+        var capture = new InputDeviceCapture(NormalizeDeviceId(deviceId), loopback);
+        capture.PcmFrameReady += (pcm, _, channels) => handler(pcm, channels);
+        if (!capture.Start()) { capture.Dispose(); throw new InvalidOperationException("WASAPI capture could not start."); }
+        return new Capture(capture);
+    }
+    public IAudioPlayback OpenPlayback(string? deviceId = null) => new Playback(deviceId);
+    // The old miniaudio ABI persisted its raw device union as hex. Its Windows member
+    // is a null-terminated UTF-16 WASAPI endpoint id; preserve saved selections.
+    internal static string? NormalizeDeviceId(string? id)
+    {
+        if (string.IsNullOrEmpty(id)) return null;
+        if (id.StartsWith('{')) return id;
+        try
+        {
+            byte[] bytes = Convert.FromHexString(id);
+            string decoded = System.Text.Encoding.Unicode.GetString(bytes).Split('\0')[0];
+            return decoded.StartsWith('{') ? decoded : null;
+        }
+        catch (FormatException) { return id; }
+    }
+    private sealed class Capture(InputDeviceCapture capture) : IAudioCapture { public void Dispose() => capture.Dispose(); }
+
+    private sealed class Playback : IAudioPlayback
+    {
+        private readonly string? deviceId;
+        private readonly PcmRing pcm = new(32768);
+        private readonly ManualResetEventSlim initialized = new(false);
+        private readonly Thread thread;
+        private volatile bool running = true;
+        private bool ready;
+        private int disposed;
+        internal Playback(string? deviceId)
+        {
+            this.deviceId = deviceId;
+            thread = new Thread(Work) { IsBackground = true, Name = "VoiceCat WASAPI playback" };
+            thread.Start();
+            if (!initialized.Wait(5000) || !ready) { Dispose(); throw new InvalidOperationException("WASAPI playback could not start."); }
+        }
+        public void Write(ReadOnlySpan<short> stereoPcm) => pcm.TryWrite(stereoPcm);
+        private unsafe void Work()
+        {
+            IMMDeviceEnumerator? enumerator = null; IMMDevice? device = null; IAudioClient? client = null; IAudioRenderClient? render = null;
+            using var bufferReady = new AutoResetEvent(false);
+            try
+            {
+                enumerator = (IMMDeviceEnumerator)Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!;
+                int result = deviceId is null ? enumerator.GetDefaultAudioEndpoint(0, 0, out device) : enumerator.GetDevice(deviceId, out device);
+                if (result < 0 || device is null) return;
+                Guid iid = typeof(IAudioClient).GUID;
+                if (device.Activate(ref iid, 0x17, 0, out object audio) < 0) return;
+                client = (IAudioClient)audio;
+                WaveFormat format = new() { Format = 1, Channels = 2, Samples = 48000, Bytes = 192000, Align = 4, Bits = 16 };
+                if (client.Initialize(0, 0x00040000u | 0x80000000u | 0x08000000u, 600000, 0, (nint)(&format), 0) < 0) return;
+                if (client.GetBufferSize(out uint capacity) < 0 || client.SetEventHandle(bufferReady.SafeWaitHandle.DangerousGetHandle()) < 0) return;
+                iid = typeof(IAudioRenderClient).GUID;
+                if (client.GetService(ref iid, out object output) < 0) return;
+                render = (IAudioRenderClient)output;
+                if (client.Start() < 0) return;
+                ready = true; initialized.Set();
+                Span<short> discard = stackalloc short[1920];
+                while (running)
+                {
+                    bufferReady.WaitOne(20); // Scheduling wait is outside the buffer-fill cycle.
+                    if (!running || client.GetCurrentPadding(out uint padding) < 0) break;
+                    uint frames = capacity - Math.Min(capacity, padding);
+                    if (frames == 0) continue;
+                    if (render.GetBuffer(frames, out nint buffer) < 0) break;
+                    var destination = new Span<short>((void*)buffer, checked((int)frames * 2));
+                    // Keep queued playback bounded to ~120 ms, then fill underflow with silence.
+                    while (pcm.Count > 11520) pcm.Read(discard);
+                    int count = pcm.Read(destination); destination[count..].Clear();
+                    if (render.ReleaseBuffer(frames, 0) < 0) break;
+                }
+            }
+            catch { ready = false; }
+            finally
+            {
+                initialized.Set();
+                try { client?.Stop(); } catch { }
+                if (render is not null) Marshal.ReleaseComObject(render);
+                if (client is not null) Marshal.ReleaseComObject(client);
+                if (device is not null) Marshal.ReleaseComObject(device);
+                if (enumerator is not null) Marshal.ReleaseComObject(enumerator);
+            }
+        }
+        public void Dispose()
+        {
+            if (Interlocked.Exchange(ref disposed, 1) != 0) return;
+            running = false;
+            if (!thread.Join(5000)) throw new TimeoutException("WASAPI playback did not stop.");
+            initialized.Dispose();
+        }
+    }
+    [StructLayout(LayoutKind.Sequential, Pack = 2)]
+    private struct WaveFormat { internal ushort Format, Channels; internal uint Samples, Bytes; internal ushort Align, Bits, Extra; }
+    [ComImport, Guid("F294ACFC-3146-4483-A7BF-ADDCA7C260E2"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
+    private interface IAudioRenderClient
+    {
+        [PreserveSig] int GetBuffer(uint frames, out nint data);
+        [PreserveSig] int ReleaseBuffer(uint frames, uint flags);
+    }
+}