Fix audio clock drift and adaptive jitter buffering

2026-09-20 16:26:03 +02:00
parent 3ee0df11a9
commit dd811a0bb8
23 changed files with 401 additions and 61 deletions
@@ -7,6 +7,84 @@ namespace VoiceCat.Tests;

 public class AudioEngineTests
 {
+    [Theory]
+    [InlineData(-1000)]
+    [InlineData(1000)]
+    public void AdaptivePcmBufferAbsorbsIndependentClockDrift(int partsPerMillion)
+    {
+        var buffer = new AdaptivePcmBuffer(1, 40); short[] input = new short[962], output = new short[960];
+        input.AsSpan().Fill(1234); Assert.True(buffer.TryWrite(input.AsSpan(0, 960)));
+        Assert.True(buffer.TryWrite(input.AsSpan(0, 960)));
+        double produced = 0;
+        for (int cycle = 0; cycle < 10_000; cycle++)
+        {
+            produced += 960 * (1 + partsPerMillion / 1_000_000.0);
+            int frames = (int)produced; produced -= frames;
+            Assert.True(buffer.TryWrite(input.AsSpan(0, frames)));
+            Assert.Equal(output.Length, buffer.Read(output));
+        }
+        Assert.InRange(buffer.CountFrames, 480, 3840);
+        long before = GC.GetAllocatedBytesForCurrentThread();
+        for (int i = 0; i < 100; i++) { buffer.TryWrite(input.AsSpan(0, 960)); buffer.Read(output); }
+        Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before);
+    }
+
+    [Fact]
+    public void OneCaptureMissDoesNotRestartTalkspurtButSustainedStarvationDoes()
+    {
+        var sent = new List<(uint Timestamp, VoiceFrameFlags Flags)>();
+        using var engine = new AudioEngine((_, timestamp, _, flags) => { sent.Add((timestamp, flags)); return true; }, false)
+        { InputMode = AudioInputMode.AlwaysOn, DeviceBufferMilliseconds = 20 };
+        engine.AddLocalStream(Stream()); short[] tone = Tone();
+        engine.FeedPcm(1, tone, 1); engine.ProcessCycle();
+        engine.ProcessCycle();
+        engine.FeedPcm(1, tone, 1); engine.ProcessCycle();
+        Assert.Equal(2, sent.Count); Assert.True((sent[0].Flags & VoiceFrameFlags.Marker) != 0); Assert.Equal(VoiceFrameFlags.None, sent[1].Flags & VoiceFrameFlags.Marker);
+        for (int i = 0; i < 10; i++) engine.ProcessCycle();
+        engine.FeedPcm(1, tone, 1); engine.ProcessCycle();
+        Assert.True((sent[^1].Flags & VoiceFrameFlags.Marker) != 0);
+    }
+
+    [Theory]
+    [InlineData(5)] [InlineData(10)] [InlineData(20)] [InlineData(40)] [InlineData(60)]
+    public void RecoveryLookaheadTracksChannelFrameDuration(int frameMilliseconds)
+    {
+        using var stream = new ReceiveStream(2, Stream(frameMilliseconds));
+        Assert.Equal(frameMilliseconds * 48, stream.TargetDepthSamples);
+    }
+
+    [Fact]
+    public void JitterTargetAdaptsInSampleTimeAndRemainsCapped()
+    {
+        var clock = new ManualAudioClock(); StreamInfo info = Stream(); info.Audio.Fec = false;
+        using var stream = new ReceiveStream(2, info, clock); using var encoder = new OpusEncoder(new() { Bitrate = 32000 });
+        byte[] packet = new byte[1275]; int length = encoder.Encode(Tone(), packet);
+        for (uint i = 0; i < 40; i++)
+        {
+            clock.Advance(i % 2 == 0 ? 5 : 35);
+            stream.Enqueue(new(MediaFrameType.Voice, 0, 0, 42, i, i * 960), packet.AsSpan(0, length));
+        }
+        int[] output = new int[1920]; stream.Mix(output, false, null);
+        Assert.InRange(stream.TargetDepthSamples, 960, 5760);
+    }
+
+    [Fact]
+    public void DredUsesTimestampOffsetForConsecutiveMissingShortFrames()
+    {
+        using var probe = new OpusEncoder();
+        if (!probe.SupportsDeepRedundancy) return;
+        StreamInfo info = Stream(10, dred: true); using var stream = new ReceiveStream(2, info);
+        using var encoder = new OpusEncoder(new() { FrameDurationMilliseconds = 10, DeepRedundancy = true, ExpectedPacketLossPercent = 30, Bitrate = 64000 });
+        byte[] packet = new byte[1275]; short[] tone = new short[480]; int[] output = new int[1920];
+        for (uint i = 0; i < 50; i++)
+        {
+            CodecTests.FillTone(tone, 480, 1, 48000, (int)i); int length = encoder.Encode(tone, packet);
+            if (i is not (25 or 26)) stream.Enqueue(new(MediaFrameType.Voice, 0, 0, 42, i, i * 480), packet.AsSpan(0, length));
+            if (i % 2 == 1) { output.AsSpan().Clear(); stream.Mix(output, false, null); }
+        }
+        Assert.True(stream.DredFrames >= 2);
+    }
+
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
@@ -114,4 +192,12 @@ public class AudioEngineTests
        for (int i = 0; i < 15; i++) { send.FeedPcm(1, stereo, 2); send.ProcessCycle(); receive.ProcessCycle(); }
        Assert.True(energy > 100000);
    }
+
+    private sealed class ManualAudioClock : TimeProvider
+    {
+        private long milliseconds;
+        public override long TimestampFrequency => 1000;
+        public override long GetTimestamp() => milliseconds;
+        internal void Advance(int value) => milliseconds += value;
+    }
 }