Extend managed macOS client toward feature parity

clients/apple/dotnet/VoiceCat.Mac/MainWindowController.cs

@@ -22,20 +22,44 @@ internal sealed class MainWindowController : NSWindowController
     private readonly NSTextField compose = new(new CGRect(400, 55, 230, 28)) { PlaceholderString = "Message" };
     private readonly NSButton send = new(new CGRect(640, 53, 90, 32)) { Title = "Send" };
     private readonly NSButton voice = new(new CGRect(630, 510, 100, 32)) { Title = "Join voice" };
+    private readonly NSButton settingsButton = new(new CGRect(740, 510, 90, 32)) { Title = "Settings" };
+    private readonly NSButton screenAudioButton = new(new CGRect(840, 510, 130, 32)) { Title = "Share audio" };
+    private readonly NSButton administrationButton = new(new CGRect(840, 20, 130, 28)) { Title = "Administration" };
+    private readonly NSButton privateMessageButton = new(new CGRect(700, 20, 130, 28)) { Title = "Private chat" };
+    private readonly NSButton muteButton = NSButton.CreateCheckbox("Mute", () => { });
+    private readonly NSButton deafenButton = NSButton.CreateCheckbox("Deafen", () => { });
     private readonly NSTextField status = NSTextField.CreateLabel("Connected");
+    private readonly MacSettings settings = new();
+    private readonly EventFeedback feedback;
     private readonly NSTimer timer;
     private uint currentChannel = 1;
     private uint microphoneStreamId;
+    private uint auxiliaryStreamId;
+    private uint screenAudioStreamId;
     private bool joinedVoice;
     private bool changingChannel;
     private IAudioCapture? microphone;
+    private IAudioCapture? auxiliary;
     private IAudioPlayback? playback;
+    private ScreenAudioCapture? screenAudio;
+    private ScreenAudioSelection screenAudioSelection = ScreenAudioSelection.Default;
+    private SettingsWindowController? settingsWindow;
+    private AdministrationWindowController? administrationWindow;
+    private readonly Dictionary<uint, PrivateMessageWindowController> privateWindows = [];
+    private readonly HashSet<uint> talkingUsers = [];
+    private bool lastTalking;
+    private string? activeInputDevice;
+    private string? activeOutputDevice;
+    private string? activeAuxiliaryDevice;
+    private bool activeStereo;
+    private NSObject? pushToTalkMonitor;
+    private bool pushToTalkEngaged;
     private long lastRemoteAudioTick;
 
-    internal MainWindowController(VoiceCatClient client, uint selfId, string nickname) : base(new NSWindow(new CGRect(0, 0, 760, 570),
+    internal MainWindowController(VoiceCatClient client, uint selfId, string nickname) : base(new NSWindow(new CGRect(0, 0, 1000, 570),
         NSWindowStyle.Titled | NSWindowStyle.Closable | NSWindowStyle.Resizable | NSWindowStyle.Miniaturizable, NSBackingStore.Buffered, false))
     {
-        this.client = client; this.selfId = selfId;
+        this.client = client; this.selfId = selfId; settings.Load(); feedback = new(settings);
         Window!.Title = $"VoiceCat — {nickname}"; Window.Center(); Window.MinSize = new CGSize(680, 480);
         var content = Window.ContentView!;
         ((INSAccessibility)channels).AccessibilityLabel = "Channel"; channels.Activated += ChangeChannel; content.AddSubview(channels);
@@ -43,26 +67,67 @@ internal sealed class MainWindowController : NSWindowController
         ((INSAccessibility)inputDevice).AccessibilityLabel = "Microphone input device"; inputDevice.Activated += ChangeAudioDevice; content.AddSubview(inputDevice);
         ((INSAccessibility)outputDevice).AccessibilityLabel = "Audio output device"; outputDevice.Activated += ChangeAudioDevice; content.AddSubview(outputDevice);
         ((INSAccessibility)voice).AccessibilityLabel = "Join or leave voice"; voice.Activated += ToggleVoice; content.AddSubview(voice);
+        ((INSAccessibility)settingsButton).AccessibilityLabel = "Open audio and notification settings"; settingsButton.Activated += OpenSettings; content.AddSubview(settingsButton);
+        ((INSAccessibility)screenAudioButton).AccessibilityLabel = "Start or stop sharing screen audio"; screenAudioButton.Activated += ToggleScreenAudio; content.AddSubview(screenAudioButton);
+        ((INSAccessibility)administrationButton).AccessibilityLabel = "Open server moderation and administration"; administrationButton.Activated += OpenAdministration; content.AddSubview(administrationButton);
+        ((INSAccessibility)privateMessageButton).AccessibilityLabel = "Open a private conversation with the selected message recipient"; privateMessageButton.Activated += OpenPrivateMessage; content.AddSubview(privateMessageButton);
+        muteButton.Frame = new CGRect(275, 490, 75, 22); deafenButton.Frame = new CGRect(355, 490, 85, 22);
+        ((INSAccessibility)muteButton).AccessibilityLabel = "Mute microphone"; ((INSAccessibility)deafenButton).AccessibilityLabel = "Deafen playback";
+        muteButton.Activated += SelfAudioChanged; deafenButton.Activated += SelfAudioChanged; content.AddSubview(muteButton); content.AddSubview(deafenButton);
         var userScroll = new NSScrollView(new CGRect(20, 90, 190, 410)) { HasVerticalScroller = true, DocumentView = users }; userScroll.AccessibilityLabel = "Users in channel"; content.AddSubview(userScroll);
         var chatScroll = new NSScrollView(new CGRect(230, 90, 500, 410)) { HasVerticalScroller = true, DocumentView = chat }; chatScroll.AccessibilityLabel = "Channel messages"; content.AddSubview(chatScroll);
         ((INSAccessibility)messageTarget).AccessibilityLabel = "Message recipient"; content.AddSubview(messageTarget);
         ((INSAccessibility)compose).AccessibilityLabel = "Message"; compose.Activated += Send; content.AddSubview(compose);
         ((INSAccessibility)send).AccessibilityLabel = "Send message"; send.Activated += Send; content.AddSubview(send);
-        status.Frame = new CGRect(20, 22, 700, 22); status.AccessibilityLabel = "Connection status"; content.AddSubview(status);
+        status.Frame = new CGRect(20, 22, 660, 22); status.AccessibilityLabel = "Connection status"; content.AddSubview(status);
         timer = NSTimer.CreateRepeatingScheduledTimer(TimeSpan.FromMilliseconds(50), _ => Pump());
+        ApplyEngineSettings(); feedback.Play(SoundEvent.Login); feedback.Speak("Connected");
+        pushToTalkMonitor = NSEvent.AddLocalMonitorForEventsMatchingMask(NSEventMask.KeyDown | NSEventMask.KeyUp, value => HandlePushToTalk(value)!);
         RefreshState();
     }
+    private NSEvent? HandlePushToTalk(NSEvent value)
+    {
+        if (settings.InputMode != AudioInputMode.PushToTalk || value.KeyCode != settings.PushToTalkKeyCode) return value;
+        bool engaged = value.Type == NSEventType.KeyDown;
+        client.Audio.PushToTalk = engaged;
+        if (engaged && !pushToTalkEngaged) feedback.Play(SoundEvent.PushToTalk);
+        pushToTalkEngaged = engaged; return null;
+    }
 
     private void Pump()
     {
         while (client.TryReadEvent(out Envelope? envelope))
         {
-            if (envelope!.TextMessage is { } text) Append(FormatMessage(text));
+            if (envelope!.TextMessage is { } text)
+            {
+                if (text.Scope == TextScope.TextPrivate)
+                {
+                    uint other = text.SenderId == selfId ? text.TargetId : text.SenderId;
+                    GetPrivateWindow(other).Append(text, Name(text.SenderId));
+                    feedback.Play(text.SenderId == selfId ? SoundEvent.PrivateSent : SoundEvent.PrivateReceived);
+                    if (text.SenderId != selfId) feedback.Speak($"Private message from {Name(text.SenderId)}: {text.Body}");
+                }
+                else { Append(FormatMessage(text)); feedback.Play(text.SenderId == selfId ? SoundEvent.ChannelSent : SoundEvent.ChannelReceived); }
+            }
+            if (envelope.StreamState is { } streamState)
+            {
+                if (streamState.Talking) talkingUsers.Add(streamState.UserId); else talkingUsers.Remove(streamState.UserId);
+                RefreshState();
+            }
+            if (envelope.UserEvent is { } userEvent)
+            {
+                string name = userEvent.User?.Nickname ?? Name(userEvent.LeftId);
+                if (userEvent.Kind == UserEvent.Types.Kind.Joined && userEvent.User?.ChannelId == currentChannel && userEvent.User.Id != selfId)
+                { AppendActivity($"{name} joined the channel"); feedback.Play(SoundEvent.ChannelJoin); feedback.Speak($"{name} joined"); }
+                else if (userEvent.Kind == UserEvent.Types.Kind.Left)
+                { talkingUsers.Remove(userEvent.LeftId); AppendActivity($"{name} left the server"); feedback.Play(SoundEvent.ChannelLeave); }
+            }
             if (envelope.ServerState is not null || envelope.ChannelEvent is not null || envelope.UserEvent is not null) RefreshState();
             if (envelope.Disconnect is { } disconnected)
             {
                 StopAudioDevices(); joinedVoice = false; voice.Title = "Join voice";
                 status.StringValue = "Disconnected: " + disconnected.Reason; voice.Enabled = send.Enabled = messageTarget.Enabled = false;
+                feedback.Play(SoundEvent.ConnectionLost); feedback.Speak("Connection lost"); Announce(status.StringValue);
             }
         }
         if (client.Audio.Failure is { } failure) { status.StringValue = "Audio stopped: " + failure.Message; return; }
@@ -70,22 +135,111 @@ internal sealed class MainWindowController : NSWindowController
         if (joinedVoice)
         {
             (float level, bool talking) = client.Audio.GetLocalLevel(microphoneStreamId);
+            if (talking != lastTalking)
+            {
+                lastTalking = talking; client.PublishStreamState(microphoneStreamId, talking);
+                feedback.Play(talking ? SoundEvent.VoiceStart : SoundEvent.VoiceStop);
+            }
             bool receiving = Environment.TickCount64 - Volatile.Read(ref lastRemoteAudioTick) < 1_000;
             status.StringValue = $"Voice connected · Mic {(talking ? "sending" : "idle")} {level:P0} · Remote audio {(receiving ? "active" : "idle")}";
         }
     }
+    private void OpenSettings(object? sender, EventArgs args)
+    {
+        settingsWindow ??= new(this, settings, audioBackend);
+        settingsWindow.ShowWindow(this); settingsWindow.Window?.MakeKeyAndOrderFront(this);
+    }
+    private void OpenAdministration(object? sender, EventArgs args)
+    {
+        administrationWindow ??= new(client, selfId); administrationWindow.ShowWindow(this); administrationWindow.Window?.MakeKeyAndOrderFront(this);
+    }
+    private void OpenPrivateMessage(object? sender, EventArgs args)
+    {
+        uint target = uint.TryParse(messageTarget.SelectedItem?.RepresentedObject?.ToString(), out uint selected) ? selected : 0;
+        if (target == 0) { status.StringValue = "Choose a private message recipient first."; return; }
+        PrivateMessageWindowController window = GetPrivateWindow(target); window.ShowWindow(this); window.Window?.MakeKeyAndOrderFront(this);
+    }
+    private PrivateMessageWindowController GetPrivateWindow(uint userId)
+    {
+        if (privateWindows.TryGetValue(userId, out PrivateMessageWindowController? existing)) return existing;
+        var created = new PrivateMessageWindowController(client, userId, selfId, Name(userId)); privateWindows[userId] = created; return created;
+    }
+
+    private void SelfAudioChanged(object? sender, EventArgs args)
+    {
+        bool muted = muteButton.State == NSCellStateValue.On;
+        bool deafened = deafenButton.State == NSCellStateValue.On;
+        client.SetSelfAudioState(muted, deafened);
+        status.StringValue = deafened ? "Deafened" : muted ? "Microphone muted" : "Voice active";
+    }
+
+    private async void ToggleScreenAudio(object? sender, EventArgs args)
+    {
+        try
+        {
+            screenAudioButton.Enabled = false;
+            if (screenAudioStreamId != 0) { StopScreenAudio(); return; }
+            if (!joinedVoice) throw new InvalidOperationException("Join voice before sharing screen audio.");
+            ScreenAudioSelection? selection = await ScreenAudioPicker.ChooseAsync(screenAudioSelection);
+            if (selection is null) return;
+            screenAudioSelection = selection;
+            StreamInfo stream = await client.StartStreamAsync(StreamKind.StreamScreenAudio, "Desktop audio", 2);
+            screenAudioStreamId = stream.StreamId;
+            int channelCount = stream.Audio.Mode == ChannelMode.ModeStereo ? 2 : 1;
+            var capture = new ScreenAudioCapture(channelCount, screenAudioSelection,
+                (pcm, channels) => { uint id = screenAudioStreamId; if (id != 0) client.Audio.FeedPcm(id, pcm, channels); });
+            capture.Failed += exception => NSApplication.SharedApplication.InvokeOnMainThread(() => { status.StringValue = exception.Message; StopScreenAudio(); });
+            screenAudio = capture; await capture.StartAsync(); screenAudioButton.Title = "Stop sharing";
+            status.StringValue = "Sharing desktop audio";
+        }
+        catch (Exception exception) { StopScreenAudio(); status.StringValue = exception.Message; }
+        finally { screenAudioButton.Enabled = true; }
+    }
+
+    private void StopScreenAudio()
+    {
+        Interlocked.Exchange(ref screenAudio, null)?.Dispose();
+        uint id = screenAudioStreamId; screenAudioStreamId = 0;
+        TryStopStream(id);
+        screenAudioButton.Title = "Share audio";
+    }
+
+    private void ApplyEngineSettings()
+    {
+        client.Audio.InputMode = settings.InputMode; client.Audio.VadThreshold = settings.VadThreshold;
+        if (settings.InputMode != AudioInputMode.PushToTalk) { client.Audio.PushToTalk = false; pushToTalkEngaged = false; }
+        client.Audio.InputGain = settings.InputGain; client.Audio.OutputGain = settings.OutputGain;
+        client.Audio.InputNoiseReduction = settings.InputNoiseReduction;
+        if (auxiliaryStreamId != 0) client.Audio.SetLocalGain(auxiliaryStreamId, settings.AuxiliaryGain);
+    }
+
+    internal async Task ApplySettingsAsync()
+    {
+        ApplyEngineSettings();
+        if (!joinedVoice) return;
+        bool restart = activeInputDevice != settings.InputDeviceId || activeOutputDevice != settings.OutputDeviceId || activeStereo != settings.StereoMicrophone;
+        if (restart) { await LeaveVoice(); await JoinVoice(); return; }
+        if (settings.AuxiliaryEnabled && auxiliaryStreamId == 0) await StartAuxiliaryAsync();
+        else if (!settings.AuxiliaryEnabled && auxiliaryStreamId != 0) StopAuxiliary();
+        else if (settings.AuxiliaryEnabled && activeAuxiliaryDevice != settings.AuxiliaryDeviceId) { StopAuxiliary(); await StartAuxiliaryAsync(); }
+    }
     private void RefreshState()
     {
         uint previousTarget = uint.TryParse(messageTarget.SelectedItem?.RepresentedObject?.ToString(), out uint selectedTarget) ? selectedTarget : 0;
         channels.RemoveAllItems();
-        foreach (var channel in client.Channels.OrderBy(c => c.Order).ThenBy(c => c.Name))
+        IReadOnlyList<(Channel Channel, int Depth)> hierarchy = FlattenChannels(client.Channels);
+        foreach ((Channel channel, int depth) in hierarchy)
         {
-            channels.AddItem(channel.Name + (channel.PasswordProtected ? " [password]" : ""));
+            int members = client.Users.Count(user => user.ChannelId == channel.Id);
+            channels.AddItem(new string(' ', depth * 3) + channel.Name + $" ({members})" + (channel.PasswordProtected ? " [password]" : ""));
             channels.LastItem!.RepresentedObject = new NSString(channel.Id.ToString());
+            channels.LastItem!.ToolTip = channel.Topic;
         }
         currentChannel = client.Users.FirstOrDefault(u => u.Id == selfId)?.ChannelId ?? currentChannel;
-        int selectedIndex = client.Channels.ToList().FindIndex(c => c.Id == currentChannel); if (selectedIndex >= 0) channels.SelectItem(selectedIndex);
-        users.Value = string.Join("\n", client.Users.Where(u => u.ChannelId == currentChannel).OrderBy(u => u.Nickname).Select(u => (u.Id == selfId ? "You — " : "") + u.Nickname));
+        int selectedIndex = hierarchy.ToList().FindIndex(item => item.Channel.Id == currentChannel); if (selectedIndex >= 0) channels.SelectItem(selectedIndex);
+        users.Value = string.Join("\n", client.Users.Where(u => u.ChannelId == currentChannel).OrderBy(u => u.Nickname).Select(u =>
+            (talkingUsers.Contains(u.Id) ? "Speaking — " : "") + (u.Id == selfId ? "You — " : "") + u.Nickname +
+            (u.ServerDeafened ? " [server deafened]" : u.ServerMuted ? " [server muted]" : u.SelfDeafened ? " [deafened]" : u.SelfMicMuted ? " [muted]" : "")));
         messageTarget.RemoveAllItems(); messageTarget.AddItem("Current channel"); messageTarget.LastItem!.RepresentedObject = new NSString("0");
         foreach (var user in client.Users.Where(user => user.Id != selfId).OrderBy(user => user.Nickname))
         {
@@ -96,6 +250,18 @@ internal sealed class MainWindowController : NSWindowController
         messageTarget.SelectItem(Math.Max(0, targetIndex));
         status.StringValue = $"Connected · {client.Users.Count} users";
     }
+    private static IReadOnlyList<(Channel Channel, int Depth)> FlattenChannels(IReadOnlyList<Channel> values)
+    {
+        var result = new List<(Channel, int)>(); var visited = new HashSet<uint>();
+        void AddChildren(uint parent, int depth)
+        {
+            foreach (Channel channel in values.Where(channel => channel.ParentId == parent).OrderBy(channel => channel.Order).ThenBy(channel => channel.Name))
+                if (visited.Add(channel.Id)) { result.Add((channel, depth)); AddChildren(channel.Id, depth + 1); }
+        }
+        AddChildren(0, 0);
+        foreach (Channel channel in values.Where(channel => !visited.Contains(channel.Id)).OrderBy(channel => channel.Order).ThenBy(channel => channel.Name)) result.Add((channel, 0));
+        return result;
+    }
     private async void ChangeChannel(object? sender, EventArgs args)
     {
         if (changingChannel || !uint.TryParse(channels.SelectedItem?.RepresentedObject?.ToString(), out uint id) || id == currentChannel) return;
@@ -196,12 +362,18 @@ internal sealed class MainWindowController : NSWindowController
         if (!result.Ok) throw new InvalidOperationException(result.Error);
         try
         {
-            playback = audioBackend.OpenPlayback(SelectedDevice(outputDevice));
+            ApplyEngineSettings();
+            activeInputDevice = settings.InputDeviceId ?? SelectedDevice(inputDevice);
+            activeOutputDevice = settings.OutputDeviceId ?? SelectedDevice(outputDevice);
+            activeStereo = settings.StereoMicrophone;
+            playback = audioBackend.OpenPlayback(activeOutputDevice);
             client.Audio.MixedPcm += PlayMixedPcm;
-            StreamInfo stream = await client.StartStreamAsync(StreamKind.StreamMic, "Microphone");
+            StreamInfo stream = await client.StartStreamAsync(StreamKind.StreamMic, "Microphone", settings.StereoMicrophone ? 2 : 1);
             microphoneStreamId = stream.StreamId;
-            microphone = audioBackend.OpenCapture(SelectedDevice(inputDevice), false, FeedMicrophone);
+            microphone = audioBackend.OpenCapture(activeInputDevice, false, FeedMicrophone);
+            if (settings.AuxiliaryEnabled) await StartAuxiliaryAsync();
             joinedVoice = true; voice.Title = "Leave voice"; status.StringValue = "Voice connected";
+            feedback.Play(SoundEvent.VoiceOn);
         }
         catch
         {
@@ -220,6 +392,29 @@ internal sealed class MainWindowController : NSWindowController
         if (streamId != 0) client.StopStream(streamId);
         await client.SubscribeVoiceAsync(false);
         status.StringValue = "Voice disconnected";
+        feedback.Play(SoundEvent.VoiceOff);
+    }
+
+    private async Task StartAuxiliaryAsync()
+    {
+        if (auxiliaryStreamId != 0) return;
+        StreamInfo stream = await client.StartStreamAsync(StreamKind.StreamAuxDevice, "Aux device", 2);
+        auxiliaryStreamId = stream.StreamId; client.Audio.SetLocalGain(stream.StreamId, settings.AuxiliaryGain);
+        try { activeAuxiliaryDevice = settings.AuxiliaryDeviceId; auxiliary = audioBackend.OpenCapture(activeAuxiliaryDevice, false, FeedAuxiliary); }
+        catch { client.StopStream(auxiliaryStreamId); auxiliaryStreamId = 0; throw; }
+    }
+
+    private void StopAuxiliary()
+    {
+        Interlocked.Exchange(ref auxiliary, null)?.Dispose();
+        activeAuxiliaryDevice = null;
+        uint id = auxiliaryStreamId; auxiliaryStreamId = 0;
+        TryStopStream(id);
+    }
+
+    private void FeedAuxiliary(ReadOnlySpan<short> pcm, int channels)
+    {
+        uint id = auxiliaryStreamId; if (id != 0) client.Audio.FeedPcm(id, pcm, channels);
     }
 
     private void FeedMicrophone(ReadOnlySpan<short> pcm, int channels)
@@ -241,11 +436,21 @@ internal sealed class MainWindowController : NSWindowController
 
     private void StopAudioDevices()
     {
+        StopScreenAudio();
+        StopAuxiliary();
         microphoneStreamId = 0;
         Volatile.Write(ref lastRemoteAudioTick, 0);
         Interlocked.Exchange(ref microphone, null)?.Dispose();
         client.Audio.MixedPcm -= PlayMixedPcm;
         Interlocked.Exchange(ref playback, null)?.Dispose();
+        activeInputDevice = activeOutputDevice = null; lastTalking = false;
+    }
+    private void TryStopStream(uint id)
+    {
+        if (id == 0 || client.State != ClientConnectionState.Connected) return;
+        try { client.StopStream(id); }
+        catch (InvalidOperationException) { }
+        catch (IOException) { }
     }
     private string Name(uint id) => client.Users.FirstOrDefault(u => u.Id == id)?.Nickname ?? $"User {id}";
     private string FormatMessage(TextMessage text)
@@ -263,9 +468,16 @@ internal sealed class MainWindowController : NSWindowController
         chat.Value = chat.Value.Length == 0 ? line : chat.Value + "\n" + line;
         chat.ScrollRangeToVisible(new NSRange(chat.Value.Length, 0));
     }
+    private void AppendActivity(string line) => Append($"[{DateTime.Now:t}] — {line}");
+    private void Announce(string value)
+    {
+        NSString[] keys = [NSAccessibilityNotificationUserInfoKeys.AnnouncementKey, NSAccessibilityNotificationUserInfoKeys.PriorityKey];
+        NSObject[] values = [(NSString)value, NSNumber.FromInt32(1)];
+        NSAccessibility.PostNotification(status, NSView.AnnouncementRequestedNotification, new NSDictionary<NSString, NSObject>(keys, values));
+    }
     protected override void Dispose(bool disposing)
     {
-        if (disposing) { timer.Invalidate(); StopAudioDevices(); client.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
+        if (disposing) { timer.Invalidate(); if (pushToTalkMonitor is not null) NSEvent.RemoveMonitor(pushToTalkMonitor); client.Audio.PushToTalk = false; settingsWindow?.Close(); administrationWindow?.Close(); foreach (PrivateMessageWindowController window in privateWindows.Values) window.Close(); privateWindows.Clear(); StopAudioDevices(); feedback.Dispose(); client.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
         base.Dispose(disposing);
     }
 }