Files
voice-cat/tests/VoiceCat.Tests/NetworkImpairmentTests.cs
T
Talon 0dad40c9d7
Build and test / test (macos-latest) (push) Waiting to run
Build and test / test (ubuntu-24.04) (push) Waiting to run
Build and test / test (windows-latest) (push) Waiting to run
Build and test / apple-client (push) Waiting to run
Reconnect on a real handover instead of waiting for a dead path
A Wi-Fi to cellular switch left the session visibly dropping: the media transport rebound
itself within a few seconds, but nothing noticed the blackholed control connection until an
unanswered keepalive proved it, and the teardown that followed announced a lost connection and
waited another second before dialling again.

Watch the system path on iOS and fail the control connection the moment the carrying interface
changes, which is the only path change TCP cannot survive. Roaming between access points and a
link that is merely unusable for a while keep the same interface and the same source address,
so ControlPathWatcher reports neither; an unsatisfied path holds the last signature rather than
reporting, so a reconnect is never started into a route that cannot carry it. Tighten the
keepalive window on the phone as the backstop for what the monitor cannot see, run the first
reconnect attempt immediately, and defer the lost-connection announcement until an attempt has
actually failed, so a sub-second handover is silent and only a real outage is announced.

A control reconnect still re-authenticates and rejoins: the media keys come from the TLS
exporter of the connection that was lost, so seamless handover needs control-plane session
resumption rather than a faster reconnect.
2026-09-25 17:20:52 +02:00

402 lines
21 KiB
C#

using System.Net;
using System.Net.Sockets;
using VoiceCat.Audio;
using VoiceCat.Core;
using VoiceCat.Codec;
using VoiceCat.Protocol;
using Voicecat.V1;
using Xunit;
using static VoiceCat.Tests.ServerTests;
using Xunit.Abstractions;
namespace VoiceCat.Tests;
// Deterministic network-impairment simulation for the receive path. A virtual millisecond clock
// drives an encoder, an impairment model, and the mixer so bursty loss, jitter, reordering,
// link outages, and a stalled (backgrounded) consumer are reproducible without hardware.
public class NetworkImpairmentTests(ITestOutputHelper output)
{
// A scheduled packet. Arrival is virtual-clock milliseconds; duplicates share a timestamp.
private readonly record struct Wire(int Arrival, uint Sequence, uint Timestamp, bool Marker, int Length, byte[] Payload);
private sealed class Impairment(int seed)
{
private readonly Random random = new(seed);
private bool bursting;
internal double LossPercent, BurstLossPercent, BurstEntryPercent, BurstExitPercent = 30, JitterMs, ReorderPercent, DuplicatePercent;
internal int BaseDelayMs = 20, OutageStartMs = -1, OutageEndMs = -1;
internal bool Dropped(int sendTime)
{
if (OutageStartMs >= 0 && sendTime >= OutageStartMs && sendTime < OutageEndMs) return true;
if (BurstEntryPercent > 0)
{
bursting = bursting ? random.NextDouble() * 100 >= BurstExitPercent : random.NextDouble() * 100 < BurstEntryPercent;
if (bursting) return random.NextDouble() * 100 < BurstLossPercent;
}
return random.NextDouble() * 100 < LossPercent;
}
internal int Arrival(int sendTime)
{
double delay = BaseDelayMs + (JitterMs > 0 ? random.NextDouble() * JitterMs : 0);
if (ReorderPercent > 0 && random.NextDouble() * 100 < ReorderPercent) delay += 45;
return sendTime + (int)delay;
}
internal bool Duplicated() => DuplicatePercent > 0 && random.NextDouble() * 100 < DuplicatePercent;
}
private sealed record Report(string Name, int Frames, int SilentFrames, int LongestSilentRunMs, int Concealed, int Overruns, int Sent, int TargetDepth)
{
internal double SilentPercent => Frames == 0 ? 0 : SilentFrames * 100.0 / Frames;
public override string ToString() =>
$"{Name,-28} silent={SilentPercent,5:F1}% worstGap={LongestSilentRunMs,5}ms concealed={Concealed,4} overruns={Overruns,3} sent={Sent,4} target={TargetDepth,4}";
}
// Runs `durationMs` of a 20 ms mono talkspurt through the impairment model. The consumer
// pumps the mixer every 20 ms of virtual time except inside a stall window, which models an
// iOS render callback that stops being serviced while backgrounded.
private Report Simulate(string name, Impairment impairment, int durationMs = 20_000, bool dred = false, bool fec = true,
int stallStartMs = -1, int stallEndMs = -1)
{
StreamInfo info = AudioEngineTests.Stream(20, dred: dred);
info.Audio.Fec = fec;
var clock = new VirtualClock();
using var stream = new ReceiveStream(7, info, clock);
using var encoder = new OpusEncoder(new() { Bitrate = 32000, ForwardErrorCorrection = fec, DeepRedundancy = dred, ExpectedPacketLossPercent = 20, Complexity = 5 });
var pending = new List<Wire>();
short[] tone = new short[960];
int frames = 0, silent = 0, run = 0, longest = 0, sent = 0;
int[] mix = new int[1920];
for (int now = 0; now <= durationMs; now += 20)
{
CodecTests.FillTone(tone, 960, 1, 48000, now / 20);
byte[] packet = new byte[1275];
int length = encoder.Encode(tone, packet);
uint sequence = (uint)(now / 20);
if (!impairment.Dropped(now))
{
pending.Add(new(impairment.Arrival(now), sequence, sequence * 960, sequence == 0, length, packet));
if (impairment.Duplicated()) pending.Add(new(impairment.Arrival(now) + 5, sequence, sequence * 960, false, length, packet));
}
clock.Set(now);
foreach (Wire wire in pending.Where(w => w.Arrival <= now).OrderBy(w => w.Arrival).ToArray())
{
var flags = wire.Marker ? VoiceFrameFlags.Marker : VoiceFrameFlags.None;
sent++;
stream.Enqueue(new(MediaFrameType.Voice, flags, 0, 42, wire.Sequence, wire.Timestamp), wire.Payload.AsSpan(0, wire.Length));
pending.Remove(wire);
}
if (stallStartMs >= 0 && now >= stallStartMs && now < stallEndMs) continue;
mix.AsSpan().Clear();
stream.Mix(mix, false, null);
frames++;
bool quiet = true;
foreach (int sample in mix) if (sample != 0) { quiet = false; break; }
if (quiet) { silent++; run += 20; longest = Math.Max(longest, run); }
else run = 0;
}
var report = new Report(name, frames, silent, longest, stream.ConcealedFrames, stream.Overruns, sent, stream.TargetDepthSamples);
output.WriteLine(report.ToString());
return report;
}
[Fact]
public void ImpairmentProfileReport()
{
output.WriteLine("--- FEC on ---");
Simulate("clean", new Impairment(1));
Simulate("random loss 2%", new Impairment(2) { LossPercent = 2 });
Simulate("random loss 10%", new Impairment(3) { LossPercent = 10 });
Simulate("bursty loss", new Impairment(4) { BurstEntryPercent = 4, BurstLossPercent = 80, BurstExitPercent = 25 });
Simulate("jitter 60ms", new Impairment(5) { JitterMs = 60 });
Simulate("jitter 120ms", new Impairment(6) { JitterMs = 120 });
Simulate("reorder 5%", new Impairment(7) { ReorderPercent = 5 });
Simulate("duplicate 5%", new Impairment(8) { DuplicatePercent = 5 });
Simulate("wifi switch 3s outage", new Impairment(9) { OutageStartMs = 6000, OutageEndMs = 9000 });
Simulate("bad wifi (loss+jitter)", new Impairment(10) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 });
Simulate("background stall 2s", new Impairment(11), stallStartMs: 6000, stallEndMs: 8000);
Simulate("background stall 10s", new Impairment(13), stallStartMs: 6000, stallEndMs: 16000);
Simulate("stall + jitter", new Impairment(12) { JitterMs = 60 }, stallStartMs: 6000, stallEndMs: 8000);
output.WriteLine("--- FEC off and DRED off ---");
Simulate("nofec clean", new Impairment(21), fec: false);
Simulate("nofec loss 2%", new Impairment(22) { LossPercent = 2 }, fec: false);
Simulate("nofec loss 10%", new Impairment(23) { LossPercent = 10 }, fec: false);
Simulate("nofec jitter 30ms", new Impairment(24) { JitterMs = 30 }, fec: false);
Simulate("nofec reorder 5%", new Impairment(25) { ReorderPercent = 5 }, fec: false);
Simulate("nofec bad wifi", new Impairment(26) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 }, fec: false);
}
// Bounds are set well above the measured result so ordinary codec variation does not make
// them flaky; they exist to catch a structural regression in the receive path, such as the
// depth floor or the arrival estimator being lost again.
[Theory]
// impairment, maxConcealedPercent, maxGapMs
[InlineData("loss2", 12)]
[InlineData("loss10", 12)]
[InlineData("burst", 20)]
[InlineData("jitter60", 12)]
[InlineData("jitter120", 12)]
[InlineData("reorder", 5)]
[InlineData("badwifi", 12)]
public void ImpairedLinksStayIntelligible(string profile, int maxConcealedPercent)
{
Impairment impairment = profile switch
{
"loss2" => new(2) { LossPercent = 2 },
"loss10" => new(3) { LossPercent = 10 },
"burst" => new(4) { BurstEntryPercent = 4, BurstLossPercent = 80, BurstExitPercent = 25 },
"jitter60" => new(5) { JitterMs = 60 },
"jitter120" => new(6) { JitterMs = 120 },
"reorder" => new(7) { ReorderPercent = 5 },
_ => new(10) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 },
};
Report report = Simulate(profile, impairment);
Assert.True(report.Concealed * 100 / report.Frames <= maxConcealedPercent,
$"{profile} concealed {report.Concealed} of {report.Frames} frames.");
Assert.True(report.LongestSilentRunMs <= 200, $"{profile} went silent for {report.LongestSilentRunMs} ms.");
}
// Pure reordering loses no data at all, so it must be absorbed by depth rather than concealed.
// Without the depth floor and an estimator that observes late arrivals this was 47 frames.
[Fact]
public void ReorderingWithoutLossIsAbsorbedRatherThanConcealed()
{
Report report = Simulate("reorder no fec", new Impairment(25) { ReorderPercent = 5 }, fec: false);
Assert.Equal(1000, report.Sent);
Assert.True(report.Concealed <= 15, $"Concealed {report.Concealed} frames despite losing none.");
}
// A consumer that stops draining (an interrupted or rebuilding iOS graph) must not cost the
// live talkspurt. The handoff previously refused new packets while full, discarding 437 of
// 1000 packets across a ten second stall.
[Fact]
public void AStalledConsumerLosesBoundedAudioRatherThanTheLiveTalkspurt()
{
Report report = Simulate("stall", new Impairment(13), stallStartMs: 6000, stallEndMs: 16000);
Assert.Equal(1000, report.Sent);
Assert.InRange(report.Overruns, 1, 4);
Assert.True(report.LongestSilentRunMs <= 200, $"Silent for {report.LongestSilentRunMs} ms after the stall.");
}
// A Wi-Fi/cellular handover changes the client's media source address while TLS survives.
// The relay binds a peer's endpoint once and refuses to move it, and the client stops
// offering its binding token after the first bind, so media must not silently die here.
[Fact]
public async Task MediaSurvivesAHandoverThatChangesTheClientSourceAddress()
{
await using var fixture = new ServerFixture();
await using var alice = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Alice");
await using var bob = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Bob");
StreamAnnounceResult stream = await alice.AnnounceAsync(StreamKind.StreamMic);
await alice.SendAsync(alice.Seal(stream.Ssrc, [1]));
await bob.ReceiveVoiceAsync();
await alice.HandoverAsync();
Assert.True(await alice.KeepaliveEchoesAsync(), "Relay stopped routing to Alice after her media source address changed.");
await alice.SendAsync(alice.Seal(stream.Ssrc, [2]));
(_, byte[] payload) = await bob.ReceiveVoiceAsync();
Assert.Equal<byte[]>([2], payload);
}
// The rebind token travels in the clear so the relay can locate the peer without trialling
// every key. Authorization comes from the AEAD tag and the replay window, so an on-path
// observer who captures a rebind must not be able to redirect the peer's downlink.
[Fact]
public async Task ReplayedRebindFromAnotherAddressCannotStealTheDownlink()
{
await using var fixture = new ServerFixture();
await using var alice = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Alice");
await using var bob = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Bob");
StreamAnnounceResult stream = await alice.AnnounceAsync(StreamKind.StreamMic);
byte[] captured = await alice.CaptureRebindAsync();
Assert.True(await alice.KeepaliveEchoesAsync());
using var attacker = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
attacker.Bind(new IPEndPoint(IPAddress.Loopback, 0));
await attacker.SendToAsync(captured, SocketFlags.None, fixture.Server.MediaEndPoint);
using (var timeout = new CancellationTokenSource(500))
{
byte[] buffer = new byte[65535];
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
await attacker.ReceiveAsync(buffer, SocketFlags.None, timeout.Token));
}
// Alice still owns the path, so her media keeps flowing to Bob.
await alice.SendAsync(alice.Seal(stream.Ssrc, [9]));
(_, byte[] payload) = await bob.ReceiveVoiceAsync();
Assert.Equal<byte[]>([9], payload);
Assert.True(await alice.KeepaliveEchoesAsync());
}
// A phone that changes interface leaves TCP blackholed rather than reset: the socket stays
// open and the OS reports nothing for minutes. Only an unanswered keepalive exposes it, and
// until it does the app shows a live session over a dead path and never reconnects.
[Fact]
public async Task ABlackholedControlConnectionIsDetectedInsteadOfAppearingConnected()
{
await using var fixture = new ServerFixture();
await using var proxy = new BlackholeProxy(fixture.Server.EndPoint);
await using var client = new VoiceCatClient("Test", "0.0.1", Path.Combine(fixture.Directory, "tofu.txt"));
client.SetControlLiveness(TimeSpan.FromMilliseconds(200), TimeSpan.FromSeconds(2));
var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
client.ConnectionStateChanged += state => { if (state == ClientConnectionState.Disconnected) disconnected.TrySetResult(); };
await client.ConnectAsync("127.0.0.1", (ushort)proxy.EndPoint.Port, (_, _) => ValueTask.FromResult(true));
await client.AuthenticateGuestAsync("Alice");
Assert.Equal(ClientConnectionState.Connected, client.State);
proxy.Freeze();
await disconnected.Task.WaitAsync(TimeSpan.FromSeconds(15));
Assert.Equal(ClientConnectionState.Disconnected, client.State);
Assert.IsType<IOException>(client.ConnectionFailure);
}
// The reconnect is only worth paying for a path change that TCP genuinely cannot survive.
// Walking a house between access points, or a link that is merely unusable for a while, keeps
// the same interface and the same source address, and a reconnect there would turn a
// recoverable glitch into the visible drop this whole change exists to remove.
[Fact]
public void OnlyAChangedInterfaceCountsAsAHandover()
{
var watcher = new ControlPathWatcher();
// The first usable path is a baseline, not a handover.
Assert.False(watcher.Observe(true, ["wifi:en0"]));
// Roaming between access points, and the same path reported again.
Assert.False(watcher.Observe(true, ["wifi:en0"]));
// A dead spot: unusable, then back on the interface it left from.
Assert.False(watcher.Observe(false, []));
Assert.False(watcher.Observe(false, []));
Assert.False(watcher.Observe(true, ["wifi:en0"]));
// Wi-Fi gives out and cellular takes over: the source address is gone.
Assert.True(watcher.Observe(true, ["cellular:pdp_ip0"]));
Assert.False(watcher.Observe(true, ["cellular:pdp_ip0"]));
// And back onto Wi-Fi on arriving home.
Assert.True(watcher.Observe(true, ["wifi:en0"]));
// Interface order is a reporting detail, not a change.
Assert.True(watcher.Observe(true, ["wifi:en0", "cellular:pdp_ip0"]));
Assert.False(watcher.Observe(true, ["cellular:pdp_ip0", "wifi:en0"]));
// After a reset the next usable path is a baseline again.
watcher.Reset();
Assert.False(watcher.Observe(true, ["wired:en5"]));
Assert.True(watcher.Observe(true, ["wifi:en0"]));
}
// The platform knows the path changed long before an unanswered keepalive can prove it. A
// client told directly must fail the connection at once, with the real cause, so the reconnect
// runs while the audio session is still up instead of after the silence timeout expires.
[Fact]
public async Task ADroppedControlConnectionReconnectsWithoutWaitingOutTheSilenceTimeout()
{
await using var fixture = new ServerFixture();
await using var proxy = new BlackholeProxy(fixture.Server.EndPoint);
await using var client = new VoiceCatClient("Test", "0.0.1", Path.Combine(fixture.Directory, "tofu.txt"));
client.SetControlLiveness(TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5));
var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
client.ConnectionStateChanged += state => { if (state == ClientConnectionState.Disconnected) disconnected.TrySetResult(); };
await client.ConnectAsync("127.0.0.1", (ushort)proxy.EndPoint.Port, (_, _) => ValueTask.FromResult(true));
await client.AuthenticateGuestAsync("Alice");
Assert.Equal(ClientConnectionState.Connected, client.State);
proxy.Freeze();
client.DropForReconnect("The network interface changed to cellular:pdp_ip0.");
await disconnected.Task.WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(ClientConnectionState.Disconnected, client.State);
Assert.Contains("cellular:pdp_ip0", Assert.IsType<IOException>(client.ConnectionFailure).Message);
// Dropping an already dead connection is a no-op, not a second failure.
client.DropForReconnect("Ignored.");
Assert.Contains("cellular:pdp_ip0", client.ConnectionFailure!.Message);
}
// A silence timeout at or below the keepalive interval would fail every healthy connection on
// its first tick, so a mobile client tightening these cannot be allowed to invert them.
[Fact]
public async Task ControlLivenessRejectsAWindowThatCannotBeMet()
{
await using var client = new VoiceCatClient("Test", "0.0.1", Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
Assert.Throws<ArgumentOutOfRangeException>(() => client.ConfigureControlLiveness(TimeSpan.Zero, TimeSpan.FromSeconds(12)));
Assert.Throws<ArgumentOutOfRangeException>(() => client.ConfigureControlLiveness(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5)));
client.ConfigureControlLiveness(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(12));
}
// Forwards TCP both ways until frozen, after which bytes are swallowed and the sockets are
// left open — what a vanished route looks like to the client, unlike a close or a reset.
private sealed class BlackholeProxy : IAsyncDisposable
{
private readonly Socket listener = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
private readonly CancellationTokenSource stop = new();
private readonly IPEndPoint origin;
private volatile bool frozen;
internal IPEndPoint EndPoint { get; }
internal void Freeze() => frozen = true;
internal BlackholeProxy(IPEndPoint origin)
{
this.origin = origin;
listener.Bind(new IPEndPoint(IPAddress.Loopback, 0));
listener.Listen(4);
EndPoint = (IPEndPoint)listener.LocalEndPoint!;
_ = AcceptAsync();
}
private async Task AcceptAsync()
{
try
{
while (!stop.IsCancellationRequested)
{
Socket inbound = await listener.AcceptAsync(stop.Token);
Socket outbound = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await outbound.ConnectAsync(origin, stop.Token);
_ = PumpAsync(inbound, outbound);
_ = PumpAsync(outbound, inbound);
}
}
catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) { }
}
private async Task PumpAsync(Socket from, Socket to)
{
byte[] buffer = new byte[16384];
try
{
while (!stop.IsCancellationRequested)
{
int length = await from.ReceiveAsync(buffer, SocketFlags.None, stop.Token);
if (length == 0) break;
if (frozen) continue;
await to.SendAsync(buffer.AsMemory(0, length), SocketFlags.None, stop.Token);
}
}
catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) { }
}
public ValueTask DisposeAsync()
{
stop.Cancel(); listener.Dispose(); stop.Dispose();
return ValueTask.CompletedTask;
}
}
private sealed class VirtualClock : TimeProvider
{
private long milliseconds;
public override long TimestampFrequency => 1000;
public override long GetTimestamp() => milliseconds;
internal void Set(long value) => milliseconds = value;
}
}