Talon
f3ac779bf4
Media died silently whenever a client's source address changed. The relay bound a peer's endpoint once and refused to move it, and the client stopped offering its binding token after the first bind, so a Wi-Fi/cellular handover stranded the session in both directions. Add an authenticated Rebind media frame: the binding token travels in the clear for peer lookup only, and the AEAD tag over header and token plus the peer's existing replay window are what authorize the move, so a captured rebind cannot be replayed to redirect someone else's downlink. The client rebuilds its UDP socket instead of retrying on one still pinned to a vanished interface. Nothing judged the control connection live: pings were sent and pongs ignored, so a blackholed TCP path went unnoticed for minutes while the UI showed a live session. Treat any server traffic as liveness and fail the connection when it stops, which drives the existing reconnect. The receive jitter buffer had lost its depth floor, so a channel without FEC or DRED played out with no buffer at all and ordinary reordering became concealment. Restore a one-frame floor, observe every arrival rather than only accepted ones — a shallow buffer was rejecting the late arrivals that should have deepened it — and allow playout to hold a frame so depth can follow a degrading link. A stalled consumer now sheds the oldest queued packet instead of refusing the live talkspurt. Add a deterministic network-impairment simulation covering bursty loss, jitter, reordering, duplication, outages and a stalled consumer, a handover test against a real relay, a replay test for the rebind path, and a blackholed control connection driven through a freezable TCP proxy.
331 lines
16 KiB
C#
331 lines
16 KiB
C#
using System.Net;
|
|
using System.Net.Sockets;
|
|
using VoiceCat.Audio;
|
|
using VoiceCat.Core;
|
|
using VoiceCat.Codec;
|
|
using VoiceCat.Protocol;
|
|
using Voicecat.V1;
|
|
using Xunit;
|
|
using static VoiceCat.Tests.ServerTests;
|
|
using Xunit.Abstractions;
|
|
|
|
namespace VoiceCat.Tests;
|
|
|
|
// Deterministic network-impairment simulation for the receive path. A virtual millisecond clock
|
|
// drives an encoder, an impairment model, and the mixer so bursty loss, jitter, reordering,
|
|
// link outages, and a stalled (backgrounded) consumer are reproducible without hardware.
|
|
public class NetworkImpairmentTests(ITestOutputHelper output)
|
|
{
|
|
// A scheduled packet. Arrival is virtual-clock milliseconds; duplicates share a timestamp.
|
|
private readonly record struct Wire(int Arrival, uint Sequence, uint Timestamp, bool Marker, int Length, byte[] Payload);
|
|
|
|
private sealed class Impairment(int seed)
|
|
{
|
|
private readonly Random random = new(seed);
|
|
private bool bursting;
|
|
internal double LossPercent, BurstLossPercent, BurstEntryPercent, BurstExitPercent = 30, JitterMs, ReorderPercent, DuplicatePercent;
|
|
internal int BaseDelayMs = 20, OutageStartMs = -1, OutageEndMs = -1;
|
|
|
|
internal bool Dropped(int sendTime)
|
|
{
|
|
if (OutageStartMs >= 0 && sendTime >= OutageStartMs && sendTime < OutageEndMs) return true;
|
|
if (BurstEntryPercent > 0)
|
|
{
|
|
bursting = bursting ? random.NextDouble() * 100 >= BurstExitPercent : random.NextDouble() * 100 < BurstEntryPercent;
|
|
if (bursting) return random.NextDouble() * 100 < BurstLossPercent;
|
|
}
|
|
return random.NextDouble() * 100 < LossPercent;
|
|
}
|
|
|
|
internal int Arrival(int sendTime)
|
|
{
|
|
double delay = BaseDelayMs + (JitterMs > 0 ? random.NextDouble() * JitterMs : 0);
|
|
if (ReorderPercent > 0 && random.NextDouble() * 100 < ReorderPercent) delay += 45;
|
|
return sendTime + (int)delay;
|
|
}
|
|
|
|
internal bool Duplicated() => DuplicatePercent > 0 && random.NextDouble() * 100 < DuplicatePercent;
|
|
}
|
|
|
|
private sealed record Report(string Name, int Frames, int SilentFrames, int LongestSilentRunMs, int Concealed, int Overruns, int Sent, int TargetDepth)
|
|
{
|
|
internal double SilentPercent => Frames == 0 ? 0 : SilentFrames * 100.0 / Frames;
|
|
public override string ToString() =>
|
|
$"{Name,-28} silent={SilentPercent,5:F1}% worstGap={LongestSilentRunMs,5}ms concealed={Concealed,4} overruns={Overruns,3} sent={Sent,4} target={TargetDepth,4}";
|
|
}
|
|
|
|
// Runs `durationMs` of a 20 ms mono talkspurt through the impairment model. The consumer
|
|
// pumps the mixer every 20 ms of virtual time except inside a stall window, which models an
|
|
// iOS render callback that stops being serviced while backgrounded.
|
|
private Report Simulate(string name, Impairment impairment, int durationMs = 20_000, bool dred = false, bool fec = true,
|
|
int stallStartMs = -1, int stallEndMs = -1)
|
|
{
|
|
StreamInfo info = AudioEngineTests.Stream(20, dred: dred);
|
|
info.Audio.Fec = fec;
|
|
var clock = new VirtualClock();
|
|
using var stream = new ReceiveStream(7, info, clock);
|
|
using var encoder = new OpusEncoder(new() { Bitrate = 32000, ForwardErrorCorrection = fec, DeepRedundancy = dred, ExpectedPacketLossPercent = 20, Complexity = 5 });
|
|
var pending = new List<Wire>();
|
|
short[] tone = new short[960];
|
|
int frames = 0, silent = 0, run = 0, longest = 0, sent = 0;
|
|
int[] mix = new int[1920];
|
|
|
|
for (int now = 0; now <= durationMs; now += 20)
|
|
{
|
|
CodecTests.FillTone(tone, 960, 1, 48000, now / 20);
|
|
byte[] packet = new byte[1275];
|
|
int length = encoder.Encode(tone, packet);
|
|
uint sequence = (uint)(now / 20);
|
|
if (!impairment.Dropped(now))
|
|
{
|
|
pending.Add(new(impairment.Arrival(now), sequence, sequence * 960, sequence == 0, length, packet));
|
|
if (impairment.Duplicated()) pending.Add(new(impairment.Arrival(now) + 5, sequence, sequence * 960, false, length, packet));
|
|
}
|
|
|
|
clock.Set(now);
|
|
foreach (Wire wire in pending.Where(w => w.Arrival <= now).OrderBy(w => w.Arrival).ToArray())
|
|
{
|
|
var flags = wire.Marker ? VoiceFrameFlags.Marker : VoiceFrameFlags.None;
|
|
sent++;
|
|
stream.Enqueue(new(MediaFrameType.Voice, flags, 0, 42, wire.Sequence, wire.Timestamp), wire.Payload.AsSpan(0, wire.Length));
|
|
pending.Remove(wire);
|
|
}
|
|
|
|
if (stallStartMs >= 0 && now >= stallStartMs && now < stallEndMs) continue;
|
|
|
|
mix.AsSpan().Clear();
|
|
stream.Mix(mix, false, null);
|
|
frames++;
|
|
bool quiet = true;
|
|
foreach (int sample in mix) if (sample != 0) { quiet = false; break; }
|
|
if (quiet) { silent++; run += 20; longest = Math.Max(longest, run); }
|
|
else run = 0;
|
|
}
|
|
var report = new Report(name, frames, silent, longest, stream.ConcealedFrames, stream.Overruns, sent, stream.TargetDepthSamples);
|
|
output.WriteLine(report.ToString());
|
|
return report;
|
|
}
|
|
|
|
[Fact]
|
|
public void ImpairmentProfileReport()
|
|
{
|
|
output.WriteLine("--- FEC on ---");
|
|
Simulate("clean", new Impairment(1));
|
|
Simulate("random loss 2%", new Impairment(2) { LossPercent = 2 });
|
|
Simulate("random loss 10%", new Impairment(3) { LossPercent = 10 });
|
|
Simulate("bursty loss", new Impairment(4) { BurstEntryPercent = 4, BurstLossPercent = 80, BurstExitPercent = 25 });
|
|
Simulate("jitter 60ms", new Impairment(5) { JitterMs = 60 });
|
|
Simulate("jitter 120ms", new Impairment(6) { JitterMs = 120 });
|
|
Simulate("reorder 5%", new Impairment(7) { ReorderPercent = 5 });
|
|
Simulate("duplicate 5%", new Impairment(8) { DuplicatePercent = 5 });
|
|
Simulate("wifi switch 3s outage", new Impairment(9) { OutageStartMs = 6000, OutageEndMs = 9000 });
|
|
Simulate("bad wifi (loss+jitter)", new Impairment(10) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 });
|
|
Simulate("background stall 2s", new Impairment(11), stallStartMs: 6000, stallEndMs: 8000);
|
|
Simulate("background stall 10s", new Impairment(13), stallStartMs: 6000, stallEndMs: 16000);
|
|
Simulate("stall + jitter", new Impairment(12) { JitterMs = 60 }, stallStartMs: 6000, stallEndMs: 8000);
|
|
|
|
output.WriteLine("--- FEC off and DRED off ---");
|
|
Simulate("nofec clean", new Impairment(21), fec: false);
|
|
Simulate("nofec loss 2%", new Impairment(22) { LossPercent = 2 }, fec: false);
|
|
Simulate("nofec loss 10%", new Impairment(23) { LossPercent = 10 }, fec: false);
|
|
Simulate("nofec jitter 30ms", new Impairment(24) { JitterMs = 30 }, fec: false);
|
|
Simulate("nofec reorder 5%", new Impairment(25) { ReorderPercent = 5 }, fec: false);
|
|
Simulate("nofec bad wifi", new Impairment(26) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 }, fec: false);
|
|
}
|
|
|
|
// Bounds are set well above the measured result so ordinary codec variation does not make
|
|
// them flaky; they exist to catch a structural regression in the receive path, such as the
|
|
// depth floor or the arrival estimator being lost again.
|
|
[Theory]
|
|
// impairment, maxConcealedPercent, maxGapMs
|
|
[InlineData("loss2", 12)]
|
|
[InlineData("loss10", 12)]
|
|
[InlineData("burst", 20)]
|
|
[InlineData("jitter60", 12)]
|
|
[InlineData("jitter120", 12)]
|
|
[InlineData("reorder", 5)]
|
|
[InlineData("badwifi", 12)]
|
|
public void ImpairedLinksStayIntelligible(string profile, int maxConcealedPercent)
|
|
{
|
|
Impairment impairment = profile switch
|
|
{
|
|
"loss2" => new(2) { LossPercent = 2 },
|
|
"loss10" => new(3) { LossPercent = 10 },
|
|
"burst" => new(4) { BurstEntryPercent = 4, BurstLossPercent = 80, BurstExitPercent = 25 },
|
|
"jitter60" => new(5) { JitterMs = 60 },
|
|
"jitter120" => new(6) { JitterMs = 120 },
|
|
"reorder" => new(7) { ReorderPercent = 5 },
|
|
_ => new(10) { LossPercent = 8, JitterMs = 80, ReorderPercent = 3 },
|
|
};
|
|
Report report = Simulate(profile, impairment);
|
|
Assert.True(report.Concealed * 100 / report.Frames <= maxConcealedPercent,
|
|
$"{profile} concealed {report.Concealed} of {report.Frames} frames.");
|
|
Assert.True(report.LongestSilentRunMs <= 200, $"{profile} went silent for {report.LongestSilentRunMs} ms.");
|
|
}
|
|
|
|
// Pure reordering loses no data at all, so it must be absorbed by depth rather than concealed.
|
|
// Without the depth floor and an estimator that observes late arrivals this was 47 frames.
|
|
[Fact]
|
|
public void ReorderingWithoutLossIsAbsorbedRatherThanConcealed()
|
|
{
|
|
Report report = Simulate("reorder no fec", new Impairment(25) { ReorderPercent = 5 }, fec: false);
|
|
Assert.Equal(1000, report.Sent);
|
|
Assert.True(report.Concealed <= 15, $"Concealed {report.Concealed} frames despite losing none.");
|
|
}
|
|
|
|
// A consumer that stops draining (an interrupted or rebuilding iOS graph) must not cost the
|
|
// live talkspurt. The handoff previously refused new packets while full, discarding 437 of
|
|
// 1000 packets across a ten second stall.
|
|
[Fact]
|
|
public void AStalledConsumerLosesBoundedAudioRatherThanTheLiveTalkspurt()
|
|
{
|
|
Report report = Simulate("stall", new Impairment(13), stallStartMs: 6000, stallEndMs: 16000);
|
|
Assert.Equal(1000, report.Sent);
|
|
Assert.InRange(report.Overruns, 1, 4);
|
|
Assert.True(report.LongestSilentRunMs <= 200, $"Silent for {report.LongestSilentRunMs} ms after the stall.");
|
|
}
|
|
|
|
// A Wi-Fi/cellular handover changes the client's media source address while TLS survives.
|
|
// The relay binds a peer's endpoint once and refuses to move it, and the client stops
|
|
// offering its binding token after the first bind, so media must not silently die here.
|
|
[Fact]
|
|
public async Task MediaSurvivesAHandoverThatChangesTheClientSourceAddress()
|
|
{
|
|
await using var fixture = new ServerFixture();
|
|
await using var alice = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Alice");
|
|
await using var bob = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Bob");
|
|
StreamAnnounceResult stream = await alice.AnnounceAsync(StreamKind.StreamMic);
|
|
|
|
await alice.SendAsync(alice.Seal(stream.Ssrc, [1]));
|
|
await bob.ReceiveVoiceAsync();
|
|
|
|
await alice.HandoverAsync();
|
|
Assert.True(await alice.KeepaliveEchoesAsync(), "Relay stopped routing to Alice after her media source address changed.");
|
|
|
|
await alice.SendAsync(alice.Seal(stream.Ssrc, [2]));
|
|
(_, byte[] payload) = await bob.ReceiveVoiceAsync();
|
|
Assert.Equal<byte[]>([2], payload);
|
|
}
|
|
|
|
// The rebind token travels in the clear so the relay can locate the peer without trialling
|
|
// every key. Authorization comes from the AEAD tag and the replay window, so an on-path
|
|
// observer who captures a rebind must not be able to redirect the peer's downlink.
|
|
[Fact]
|
|
public async Task ReplayedRebindFromAnotherAddressCannotStealTheDownlink()
|
|
{
|
|
await using var fixture = new ServerFixture();
|
|
await using var alice = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Alice");
|
|
await using var bob = await MediaRelayTests.VoicePeer.ConnectAsync(fixture, "Bob");
|
|
StreamAnnounceResult stream = await alice.AnnounceAsync(StreamKind.StreamMic);
|
|
|
|
byte[] captured = await alice.CaptureRebindAsync();
|
|
Assert.True(await alice.KeepaliveEchoesAsync());
|
|
|
|
using var attacker = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
|
attacker.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
|
await attacker.SendToAsync(captured, SocketFlags.None, fixture.Server.MediaEndPoint);
|
|
using (var timeout = new CancellationTokenSource(500))
|
|
{
|
|
byte[] buffer = new byte[65535];
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
|
|
await attacker.ReceiveAsync(buffer, SocketFlags.None, timeout.Token));
|
|
}
|
|
|
|
// Alice still owns the path, so her media keeps flowing to Bob.
|
|
await alice.SendAsync(alice.Seal(stream.Ssrc, [9]));
|
|
(_, byte[] payload) = await bob.ReceiveVoiceAsync();
|
|
Assert.Equal<byte[]>([9], payload);
|
|
Assert.True(await alice.KeepaliveEchoesAsync());
|
|
}
|
|
|
|
// A phone that changes interface leaves TCP blackholed rather than reset: the socket stays
|
|
// open and the OS reports nothing for minutes. Only an unanswered keepalive exposes it, and
|
|
// until it does the app shows a live session over a dead path and never reconnects.
|
|
[Fact]
|
|
public async Task ABlackholedControlConnectionIsDetectedInsteadOfAppearingConnected()
|
|
{
|
|
await using var fixture = new ServerFixture();
|
|
await using var proxy = new BlackholeProxy(fixture.Server.EndPoint);
|
|
await using var client = new VoiceCatClient("Test", "0.0.1", Path.Combine(fixture.Directory, "tofu.txt"));
|
|
client.SetControlLiveness(TimeSpan.FromMilliseconds(200), TimeSpan.FromSeconds(2));
|
|
|
|
var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
client.ConnectionStateChanged += state => { if (state == ClientConnectionState.Disconnected) disconnected.TrySetResult(); };
|
|
await client.ConnectAsync("127.0.0.1", (ushort)proxy.EndPoint.Port, (_, _) => ValueTask.FromResult(true));
|
|
await client.AuthenticateGuestAsync("Alice");
|
|
Assert.Equal(ClientConnectionState.Connected, client.State);
|
|
|
|
proxy.Freeze();
|
|
await disconnected.Task.WaitAsync(TimeSpan.FromSeconds(15));
|
|
Assert.Equal(ClientConnectionState.Disconnected, client.State);
|
|
Assert.IsType<IOException>(client.ConnectionFailure);
|
|
}
|
|
|
|
// Forwards TCP both ways until frozen, after which bytes are swallowed and the sockets are
|
|
// left open — what a vanished route looks like to the client, unlike a close or a reset.
|
|
private sealed class BlackholeProxy : IAsyncDisposable
|
|
{
|
|
private readonly Socket listener = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
|
private readonly CancellationTokenSource stop = new();
|
|
private readonly IPEndPoint origin;
|
|
private volatile bool frozen;
|
|
internal IPEndPoint EndPoint { get; }
|
|
internal void Freeze() => frozen = true;
|
|
|
|
internal BlackholeProxy(IPEndPoint origin)
|
|
{
|
|
this.origin = origin;
|
|
listener.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
|
listener.Listen(4);
|
|
EndPoint = (IPEndPoint)listener.LocalEndPoint!;
|
|
_ = AcceptAsync();
|
|
}
|
|
|
|
private async Task AcceptAsync()
|
|
{
|
|
try
|
|
{
|
|
while (!stop.IsCancellationRequested)
|
|
{
|
|
Socket inbound = await listener.AcceptAsync(stop.Token);
|
|
Socket outbound = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
|
await outbound.ConnectAsync(origin, stop.Token);
|
|
_ = PumpAsync(inbound, outbound);
|
|
_ = PumpAsync(outbound, inbound);
|
|
}
|
|
}
|
|
catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) { }
|
|
}
|
|
|
|
private async Task PumpAsync(Socket from, Socket to)
|
|
{
|
|
byte[] buffer = new byte[16384];
|
|
try
|
|
{
|
|
while (!stop.IsCancellationRequested)
|
|
{
|
|
int length = await from.ReceiveAsync(buffer, SocketFlags.None, stop.Token);
|
|
if (length == 0) break;
|
|
if (frozen) continue;
|
|
await to.SendAsync(buffer.AsMemory(0, length), SocketFlags.None, stop.Token);
|
|
}
|
|
}
|
|
catch (Exception exception) when (exception is OperationCanceledException or SocketException or ObjectDisposedException) { }
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
stop.Cancel(); listener.Dispose(); stop.Dispose();
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
}
|
|
|
|
private sealed class VirtualClock : TimeProvider
|
|
{
|
|
private long milliseconds;
|
|
public override long TimestampFrequency => 1000;
|
|
public override long GetTimestamp() => milliseconds;
|
|
internal void Set(long value) => milliseconds = value;
|
|
}
|
|
}
|