Port managed client audio and Windows application

dotnet/tests/VoiceCat.Tests/AudioEngineTests.cs

@@ -0,0 +1,117 @@
+using VoiceCat.Audio;
+using VoiceCat.Codec;
+using VoiceCat.Protocol;
+using Voicecat.V1;
+
+namespace VoiceCat.Tests;
+
+public class AudioEngineTests
+{
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public void LostFramesUseDredThenFecBeforeBoundedPlc(bool useDred)
+    {
+        using var receive = new ReceiveStream(2, Stream(dred: useDred));
+        using var encoder = new OpusEncoder(new() { DeepRedundancy = useDred, ForwardErrorCorrection = true, ExpectedPacketLossPercent = 30, Complexity = 10, Bitrate = 64000 });
+        byte[] packet = new byte[1275]; short[] tone = Tone(); int[] output = new int[1920];
+        for (uint i = 0; i < 40; i++)
+        {
+            CodecTests.FillTone(tone, 960, 1, 48000, (int)i);
+            int size = encoder.Encode(tone, packet);
+            if (i != 25 && i != 30 && i != 35) receive.Enqueue(new(MediaFrameType.Voice, 0, 0, 42, i, i * 960), packet.AsSpan(0, size));
+            output.AsSpan().Clear(); receive.Mix(output, false, null);
+        }
+        if (useDred) Assert.True(receive.DredFrames > 0); else Assert.True(receive.FecFrames > 0);
+        for (int i = 0; i < 20; i++) { output.AsSpan().Clear(); receive.Mix(output, false, null); }
+        Assert.True(receive.ConcealedFrames > 0); Assert.All(output, sample => Assert.Equal(0, sample));
+    }
+
+    [Fact]
+    public void PcmRingDropsWholeFramesWhenFullAndPreservesOrderAcrossWraps()
+    {
+        var ring = new PcmRing(8); short[] output = new short[8];
+        for (int i = 0; i < 100; i++)
+        {
+            Assert.True(ring.TryWrite([1, 2, 3, 4, 5, 6])); Assert.False(ring.TryWrite([7, 8, 9]));
+            Assert.Equal(4, ring.Read(output.AsSpan(0, 4))); Assert.Equal(new short[] { 1, 2, 3, 4 }, output[..4]);
+            Assert.True(ring.TryWrite([7, 8])); Assert.Equal(4, ring.Read(output)); Assert.Equal(new short[] { 5, 6, 7, 8 }, output[..4]); Assert.Equal(0, ring.Count);
+        }
+    }
+    internal static StreamInfo Stream(int frame = 20, bool stereo = false, bool dred = false) => new()
+    {
+        StreamId = 1, Ssrc = 42, Kind = StreamKind.StreamMic,
+        Audio = new() { SampleRate = 48000, BitrateBps = 32000, FrameMs = (uint)frame, Complexity = 5,
+            Mode = stereo ? ChannelMode.ModeStereo : ChannelMode.ModeMono, Fec = true, ExpectedPacketLoss = 20, Dred = dred }
+    };
+    private static short[] Tone(int channels = 1)
+    {
+        var pcm = new short[960 * channels];
+        for (int i = 0; i < 960; i++) for (int c = 0; c < channels; c++) pcm[i * channels + c] = (short)(Math.Sin(i * 2 * Math.PI * (c == 0 ? 440 : 660) / 48000) * 8000);
+        return pcm;
+    }
+
+    [Theory]
+    [InlineData(5, false)] [InlineData(10, false)] [InlineData(20, false)] [InlineData(40, false)] [InlineData(60, false)] [InlineData(20, true)]
+    public void ReframedEncodedPcmIsDecodedAndMixedForMonoAndStereo(int frame, bool stereo)
+    {
+        StreamInfo stream = Stream(frame, stereo);
+        using var receive = new AudioEngine((_, _, _, _) => true, false);
+        receive.SetRemoteStreams([new() { Id = 2, ChannelId = 1, Streams = { stream } }], 1, 1);
+        using var send = new AudioEngine((ssrc, timestamp, payload, flags) => { receive.Receive(new(MediaFrameType.Voice, flags, 0, ssrc, 0, timestamp), payload); return true; }, false);
+        send.InputMode = AudioInputMode.AlwaysOn; send.AddLocalStream(stream, stereo ? 2 : 1);
+        long energy = 0; int sinkChannels = 0;
+        receive.MixedPcm += pcm => { foreach (short sample in pcm) energy += Math.Abs((int)sample); };
+        receive.StreamPcm += (_, _, _, channels) => sinkChannels = channels;
+        short[] tone = Tone(stereo ? 2 : 1);
+        for (int i = 0; i < 30; i++) { Assert.True(send.FeedPcm(1, tone, stereo ? 2 : 1)); send.ProcessCycle(); receive.ProcessCycle(); }
+        Assert.True(energy > 100000); Assert.Equal(stereo ? 2 : 1, sinkChannels);
+        receive.SetRemotePlayback(2, 1, 1, true, false); energy = 0;
+        for (int i = 0; i < 5; i++) { send.FeedPcm(1, tone, stereo ? 2 : 1); send.ProcessCycle(); receive.ProcessCycle(); }
+        Assert.Equal(0, energy);
+    }
+
+    [Fact]
+    public void AudioCyclesAllocateZeroBytesWithEncodeDecodeStereoNoiseReductionAndMixing()
+    {
+        StreamInfo stream = Stream(20, true);
+        using var receive = new AudioEngine((_, _, _, _) => true, false);
+        receive.SetRemoteStreams([new() { Id = 2, ChannelId = 1, Streams = { stream } }], 1, 1);
+        receive.SetRemotePlayback(2, 1, 0.8f, false, true);
+        using var send = new AudioEngine((ssrc, timestamp, payload, flags) => { receive.Receive(new(MediaFrameType.Voice, flags, 0, ssrc, 0, timestamp), payload); return true; }, false);
+        send.InputMode = AudioInputMode.AlwaysOn; send.InputNoiseReduction = true; send.AddLocalStream(stream, 2);
+        short[] tone = Tone(2);
+        for (int i = 0; i < 30; i++) { send.FeedPcm(1, tone, 2); send.ProcessCycle(); receive.ProcessCycle(); }
+        long before = GC.GetAllocatedBytesForCurrentThread();
+        for (int i = 0; i < 100; i++) { send.FeedPcm(1, tone, 2); send.ProcessCycle(); receive.ProcessCycle(); }
+        Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before);
+    }
+
+    [Fact]
+    public void JitterBacklogIsBoundedAndPlcEventuallyBecomesSilence()
+    {
+        var info = Stream(); using var stream = new ReceiveStream(2, info); using var encoder = new OpusEncoder(new() { Bitrate = 32000 });
+        byte[] packet = new byte[1275]; int length = encoder.Encode(Tone(), packet); int[] output = new int[1920];
+        for (uint i = 0; i < 64; i++) stream.Enqueue(new(MediaFrameType.Voice, 0, 0, 42, i, i * 960), packet.AsSpan(0, length));
+        stream.Mix(output, false, null); Assert.InRange(stream.Depth, 0, 6);
+        for (int i = 0; i < 20; i++) { output.AsSpan().Clear(); stream.Mix(output, false, null); }
+        Assert.All(output, value => Assert.Equal(0, value)); Assert.InRange(stream.ConcealedFrames, 1, 10);
+    }
+
+    [Fact]
+    public void PttResumeAndCaptureChannelChangesKeepTimestampProgressAndAudio()
+    {
+        var info = Stream(60, true); using var receive = new AudioEngine((_, _, _, _) => true, false);
+        receive.SetRemoteStreams([new() { Id = 2, ChannelId = 1, Streams = { info } }], 1, 1);
+        using var send = new AudioEngine((ssrc, timestamp, payload, flags) => { receive.Receive(new(MediaFrameType.Voice, flags, 0, ssrc, 0, timestamp), payload); return true; }, false);
+        send.InputMode = AudioInputMode.PushToTalk; send.PushToTalk = true; send.AddLocalStream(info);
+        short[] mono = Tone(), stereo = Tone(2); long energy = 0;
+        receive.MixedPcm += pcm => { foreach (short value in pcm) energy += Math.Abs((int)value); };
+        for (int i = 0; i < 15; i++) { send.FeedPcm(1, mono, 1); send.ProcessCycle(); receive.ProcessCycle(); }
+        send.PushToTalk = false;
+        for (int i = 0; i < 16; i++) { send.FeedPcm(1, mono, 1); send.ProcessCycle(); receive.ProcessCycle(); }
+        send.SetCaptureChannels(1, 2); send.PushToTalk = true; energy = 0;
+        for (int i = 0; i < 15; i++) { send.FeedPcm(1, stereo, 2); send.ProcessCycle(); receive.ProcessCycle(); }
+        Assert.True(energy > 100000);
+    }
+}

dotnet/tests/VoiceCat.Tests/ManagedClientTests.cs

@@ -0,0 +1,80 @@
+using VoiceCat.Core;
+using VoiceCat.Crypto;
+using Voicecat.V1;
+using static VoiceCat.Tests.ServerTests;
+using static VoiceCat.Tests.MediaRelayTests;
+
+namespace VoiceCat.Tests;
+
+public class ManagedClientTests
+{
+    private static VoiceCatClient NewClient(ServerFixture fixture, string name) => new(name, "test", Path.Combine(fixture.Directory, name + ".pins"));
+    private static Task Connect(VoiceCatClient client, ServerFixture fixture) => client.ConnectAsync("127.0.0.1", (ushort)fixture.Server.EndPoint.Port, (_, _) => ValueTask.FromResult(true));
+    private static async Task<Envelope> Event(VoiceCatClient client, Func<Envelope, bool> predicate)
+    {
+        using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+        await foreach (Envelope message in client.ReadEventsAsync(timeout.Token)) if (predicate(message)) return message;
+        throw new IOException("Expected client event was not received.");
+    }
+
+    [Fact]
+    public async Task ManagedClientsAuthenticateChatAndCorrelateConcurrentRequests()
+    {
+        await using var fixture = new ServerFixture();
+        await using var alice = NewClient(fixture, "Alice"); await using var bob = NewClient(fixture, "Bob");
+        await Connect(alice, fixture); await Connect(bob, fixture);
+        Assert.True((await alice.AuthenticateGuestAsync("Alice")).Ok); Assert.True((await bob.AuthenticateGuestAsync("Bob")).Ok);
+        await Event(bob, e => e.ServerState is not null); await Event(alice, e => e.UserEvent?.User?.Nickname == "Bob");
+        Assert.Equal(2, alice.Users.Count);
+        var copy = alice.Users[0]; copy.Nickname = "Mutated"; Assert.DoesNotContain(alice.Users, u => u.Nickname == "Mutated");
+        alice.Send(new() { TextMessage = new() { Scope = TextScope.TextChannel, TargetId = 1, Body = "Managed conversation", ClientMsgId = "a1" } });
+        Assert.Equal("Managed conversation", (await Event(bob, e => e.TextMessage is not null)).TextMessage.Body);
+        var requests = Enumerable.Range(1, 20).Select(async i =>
+        {
+            Envelope response = await alice.RequestAsync(new() { Ping = new() { Nonce = (ulong)i } });
+            Assert.Equal((ulong)i, response.Pong.Nonce); return response.RequestId;
+        });
+        Assert.Equal(20, (await Task.WhenAll(requests)).Distinct().Count());
+        await Assert.ThrowsAsync<InvalidOperationException>(() => Connect(alice, fixture));
+        Assert.Equal(ClientConnectionState.Connected, alice.State);
+        await alice.DisconnectAsync();
+        await Event(bob, e => e.UserEvent?.Kind == UserEvent.Types.Kind.Left);
+        await Connect(alice, fixture); Assert.True((await alice.AuthenticateGuestAsync("Returned")).Ok);
+    }
+
+    [Fact]
+    public async Task TofuRequiresApprovalPinsAcceptedCertificateAndRejectsChanges()
+    {
+        await using var first = new ServerFixture(); await using var second = new ServerFixture();
+        await using var client = NewClient(first, "Tofu");
+        await Assert.ThrowsAsync<System.Security.Authentication.AuthenticationException>(() => client.ConnectAsync("127.0.0.1", (ushort)first.Server.EndPoint.Port));
+        await client.ConnectAsync("127.0.0.1", (ushort)first.Server.EndPoint.Port, (challenge, _) =>
+        { Assert.Equal(TofuStatus.FirstConnect, challenge.Status); return ValueTask.FromResult(true); });
+        await client.DisconnectAsync();
+        await client.ConnectAsync("127.0.0.1", (ushort)first.Server.EndPoint.Port); await client.DisconnectAsync();
+        // Pin the other server's certificate to this endpoint, simulating a changed server certificate.
+        using var credentials = ServerCredentials.LoadOrCreate(second.Directory, "VoiceCat Server");
+        new TofuStore(Path.Combine(first.Directory, "Other.pins")).Pin("127.0.0.1", (ushort)first.Server.EndPoint.Port, credentials.CertificateFingerprint);
+        await using var changed = new VoiceCatClient(tofuStorePath: Path.Combine(first.Directory, "Other.pins"));
+        await Assert.ThrowsAsync<System.Security.Authentication.AuthenticationException>(() => changed.ConnectAsync("127.0.0.1", (ushort)first.Server.EndPoint.Port,
+            (challenge, _) => { Assert.Equal(TofuStatus.Mismatch, challenge.Status); return ValueTask.FromResult(false); }));
+    }
+
+    [Fact]
+    public async Task ManagedClientSendsAndReceivesAuthenticatedEncodedVoice()
+    {
+        await using var fixture = new ServerFixture();
+        await using var managed = NewClient(fixture, "Managed"); await Connect(managed, fixture); await managed.AuthenticateGuestAsync("Managed");
+        Assert.True((await managed.SubscribeVoiceAsync()).Ok);
+        await using var peer = await VoicePeer.ConnectAsync(fixture, "Peer");
+        var remote = await peer.AnnounceAsync(StreamKind.StreamMic);
+        var local = (await managed.RequestAsync(new() { StreamAnnounce = new() { Kind = StreamKind.StreamMic } })).StreamAnnounceResult;
+        Assert.True(local.Ok);
+        var received = new TaskCompletionSource<byte[]>(TaskCreationOptions.RunContinuationsAsynchronously);
+        managed.VoiceReceived += (header, payload) => { Assert.Equal(remote.Ssrc, header.Ssrc); received.TrySetResult(payload.ToArray()); };
+        await peer.SendAsync(peer.Seal(remote.Ssrc, [1, 2, 3]));
+        Assert.Equal(new byte[] { 1, 2, 3 }, await received.Task.WaitAsync(TimeSpan.FromSeconds(5)));
+        Assert.True(managed.TrySendEncodedVoice(local.Ssrc, 960, [4, 5, 6]));
+        Assert.Equal(new byte[] { 4, 5, 6 }, (await peer.ReceiveVoiceAsync()).Payload);
+    }
+}

dotnet/tests/VoiceCat.Tests/MediaRelayTests.cs

@@ -1,3 +1,4 @@
+using VoiceCat.Transport;
 using System.Net;
 using System.Diagnostics;
 using System.Net.Sockets;

dotnet/tests/VoiceCat.Tests/ProductionServerTests.cs

@@ -1,3 +1,4 @@
+using VoiceCat.Transport;
 using System.Net;
 using System.Net.Sockets;
 using System.Text.Json;

dotnet/tests/VoiceCat.Tests/ServerTests.cs

@@ -1,3 +1,4 @@
+using VoiceCat.Transport;
 using System.Diagnostics;
 using System.Net;
 using System.Net.Sockets;

dotnet/tests/VoiceCat.Tests/VoiceCat.Tests.csproj

@@ -4,6 +4,8 @@
     <IsTestProject>true</IsTestProject>
   </PropertyGroup>
   <ItemGroup>
+    <ProjectReference Include="../../../clients/windows/VoiceCat.Managed/VoiceCat.Managed.csproj" />
+    <ProjectReference Include="../../src/VoiceCat.Core/VoiceCat.Core.csproj" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
     <PackageReference Include="xunit" Version="2.9.3" />
     <PackageReference Include="xunit.runner.visualstudio" Version="3.1.1" PrivateAssets="all" />

dotnet/tests/VoiceCat.Tests/WindowsManagedClientTests.cs

@@ -0,0 +1,58 @@
+using VoiceCat.Interop;
+using VoiceCat.Server.Data;
+using Voicecat.V1;
+using static VoiceCat.Tests.ServerTests;
+using Client = VoiceCat.Interop.VoiceCatClient;
+
+namespace VoiceCat.Tests;
+
+public class WindowsManagedClientTests
+{
+    private static async Task Until(Client client, Func<bool> predicate)
+    {
+        using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+        do { client.PumpEvents(); if (predicate()) return; await Task.Delay(10, timeout.Token); } while (true);
+    }
+    private static async Task Login(Client client, ServerFixture fixture, bool admin = false)
+    {
+        client.EventReceived += e => { if (e.Type == VcEventType.ServerIdentity) client.ConfirmServerIdentity(true); };
+        Assert.Equal(VcResult.Ok, client.Connect("127.0.0.1", (ushort)fixture.Server.EndPoint.Port));
+        if (admin) client.AuthenticateUser("Admin", "secret"); else client.AuthenticateGuest("Guest");
+        await Until(client, () => client.ListUsers().Count > 0);
+    }
+    [Fact]
+    public async Task ShippedWindowsFacadeChatsExchangesPcmAndKeepsCaptureIdsAcrossChannelMoves()
+    {
+        await using var fixture = new ServerFixture();
+        using (var accounts = new AccountStore(Path.Combine(fixture.Directory, "voicecat.db"))) await accounts.CreateAccountAsync("Admin", "secret", true);
+        using var alice = new Client("Alice", "test", tofuStorePath: Path.Combine(fixture.Directory, "alice.pins"));
+        using var bob = new Client("Bob", "test", tofuStorePath: Path.Combine(fixture.Directory, "bob.pins"));
+        await Login(alice, fixture, true); await Login(bob, fixture);
+        Assert.True(alice.GetPermissions().IsAdmin);
+        string? body = null; bob.EventReceived += e => { if (e.Type == VcEventType.TextMessage) body = e.Text; };
+        Assert.Equal(VcResult.Ok, alice.SendText(VcTextScope.Channel, 1, "Managed Windows chat"));
+        await Until(bob, () => body is not null); Assert.Equal("Managed Windows chat", body);
+        Assert.Equal(VcResult.Ok, alice.JoinVoice()); Assert.Equal(VcResult.Ok, bob.JoinVoice());
+        alice.SetInputMode(VcInputMode.AlwaysOn); bob.SetInputMode(VcInputMode.AlwaysOn);
+        var a = alice.StartStreamExternalFeed(VcStreamKind.Mic, "Mic"); var b = bob.StartStreamExternalFeed(VcStreamKind.Mic, "Mic");
+        Assert.Equal(VcResult.Ok, a.Result); Assert.Equal(VcResult.Ok, b.Result);
+        await Until(bob, () => bob.ManagedClient.Users.Any(u => u.Streams.Count > 0 && u.Id != bob.ManagedClient.Authentication!.Self.Id));
+        long aliceEnergy = 0, bobEnergy = 0;
+        alice.ManagedClient.Audio.MixedPcm += pcm => { long sum = 0; foreach (short sample in pcm) sum += Math.Abs((int)sample); Interlocked.Add(ref aliceEnergy, sum); };
+        bob.ManagedClient.Audio.MixedPcm += pcm => { long sum = 0; foreach (short sample in pcm) sum += Math.Abs((int)sample); Interlocked.Add(ref bobEnergy, sum); };
+        short[] tone = Enumerable.Range(0, 960).Select(i => (short)(8000 * Math.Sin(i * Math.PI * 880 / 48000))).ToArray();
+        for (int i = 0; i < 40; i++) { alice.StreamFeedPcm(a.StreamId, tone, 960, 1); bob.StreamFeedPcm(b.StreamId, tone, 960, 1); alice.PumpEvents(); bob.PumpEvents(); await Task.Delay(20); }
+        Assert.True(Interlocked.Read(ref aliceEnergy) > 100000); Assert.True(Interlocked.Read(ref bobEnergy) > 100000);
+        uint oldId = alice.ManagedClient.LocalStreams.Single().StreamId;
+        await alice.ManagedClient.RequestAsync(new() { CreateChannel = new() { Channel = new() { Name = "Stereo", ParentId = 1, Audio = AudioEngineTests.Stream(20, true).Audio } } });
+        await Until(alice, () => alice.ListChannels().Any(c => c.Name == "Stereo"));
+        uint channel = alice.ListChannels().Single(c => c.Name == "Stereo").Id;
+        Assert.Equal(VcResult.Ok, alice.JoinChannel(channel));
+        await Until(alice, () => alice.ManagedClient.LocalStreams.Any(s => s.StreamId != oldId));
+        Assert.Equal(a.StreamId, Assert.Single(alice.ListUserStreams(alice.ManagedClient.Authentication!.Self.Id)).StreamId);
+        Assert.True(alice.GetStreamAudioConfig(alice.ManagedClient.Authentication!.Self.Id, a.StreamId).Config!.Stereo);
+        Assert.Equal(VcResult.Ok, alice.StreamFeedPcm(a.StreamId, tone, 960, 1));
+        Assert.Equal(VcResult.Ok, alice.StopStream(a.StreamId));
+        Assert.Empty(alice.ManagedClient.LocalStreams);
+    }
+}

dotnet/tests/VoiceCat.Tests/packages.lock.json

@@ -146,9 +146,24 @@
           "xunit.extensibility.core": "[2.9.3]"
         }
       },
+      "voicecat.audio": {
+        "type": "Project",
+        "dependencies": {
+          "VoiceCat.Codec": "[1.0.0, )",
+          "VoiceCat.Dsp": "[1.0.0, )",
+          "VoiceCat.Protocol": "[1.0.0, )"
+        }
+      },
       "voicecat.codec": {
         "type": "Project"
       },
+      "voicecat.core": {
+        "type": "Project",
+        "dependencies": {
+          "VoiceCat.Audio": "[1.0.0, )",
+          "VoiceCat.Crypto": "[1.0.0, )"
+        }
+      },
       "voicecat.crypto": {
         "type": "Project",
         "dependencies": {
@@ -159,6 +174,12 @@
       "voicecat.dsp": {
         "type": "Project"
       },
+      "voicecat.managed": {
+        "type": "Project",
+        "dependencies": {
+          "VoiceCat.Core": "[1.0.0, )"
+        }
+      },
       "voicecat.protocol": {
         "type": "Project",
         "dependencies": {