Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Fix audio clock drift and adaptive jitter buffering
.NET port / test (macos-latest) (push) Canceled after 0s
Details
.NET port / test (ubuntu-24.04) (push) Canceled after 0s
Details
.NET port / test (windows-latest) (push) Canceled after 0s
Details
.NET port / apple-client (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
Talon
2026-09-20 16:26:03 +02:00
parent 3ee0df11a9
commit dd811a0bb8
23 changed files with 401 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clients/apple/dotnet/VoiceCat.Mac/MacAudioBackend.cs
+2 -3
View File
@@ -139,7 +139,7 @@ internal sealed class MacAudioBackend : IAudioDeviceBackend
private sealed class Playback : IAudioPlayback
{
private readonly PcmRing pcm = new(32_768);
private readonly AdaptivePcmBuffer pcm = new(2);
private readonly short[] renderScratch = new short[8_192];
private readonly AVAudioEngine engine = new();
private readonly AVAudioFormat format;
@@ -187,6 +187,7 @@ internal sealed class MacAudioBackend : IAudioDeviceBackend
}
public void Write(ReadOnlySpan<short> stereoPcm) => pcm.TryWrite(stereoPcm);
public int BufferMilliseconds { get => pcm.BufferMilliseconds; set => pcm.BufferMilliseconds = value; }
private unsafe int Render(IntPtr isSilence, IntPtr _, uint frameCount, IntPtr outputData)
{
@@ -199,8 +200,6 @@ internal sealed class MacAudioBackend : IAudioDeviceBackend
int frames = checked((int)frameCount);
int requested = checked(frames * 2);
if (bufferCount != 2 || requested > renderScratch.Length) return Fail("Core Audio returned an invalid planar stereo playback buffer.");
Span<short> discard = stackalloc short[1_920];
while (pcm.Count > 11_520) pcm.Read(discard[..Math.Min(discard.Length, pcm.Count - 11_520)]);
Span<short> source = renderScratch.AsSpan(0, requested);
int read = pcm.Read(source);
source[read..].Clear();
clients/apple/dotnet/VoiceCat.Mac/MacSettings.cs
+5 -3
View File
@@ -19,6 +19,7 @@ internal sealed class MacSettings
internal bool AuxiliaryEnabled { get; set; }
internal string? AuxiliaryDeviceId { get; set; }
internal float AuxiliaryGain { get; set; } = 1f;
internal int AudioBufferMilliseconds { get; set; } = 40;
internal bool EventSounds { get; set; } = true;
internal bool SpokenEvents { get; set; }
internal float EventVolume { get; set; } = 1f;
@@ -28,9 +29,9 @@ internal sealed class MacSettings
internal void Load()
{
NSString[] keys = [(NSString)"voice.inputMode", (NSString)"voice.vadThreshold", (NSString)"voice.inputGain", (NSString)"voice.outputGain",
(NSString)"voice.pttKeyCode", (NSString)"voice.auxGain", (NSString)"feedback.sounds", (NSString)"feedback.volume"];
(NSString)"voice.pttKeyCode", (NSString)"voice.auxGain", (NSString)"voice.audioBufferMs", (NSString)"feedback.sounds", (NSString)"feedback.volume"];
NSObject[] defaults = [NSNumber.FromInt32((int)AudioInputMode.VoiceActivation), NSNumber.FromFloat(0.05f),
NSNumber.FromFloat(1f), NSNumber.FromFloat(0.8f), NSNumber.FromInt32(0x60), NSNumber.FromFloat(1f),
NSNumber.FromFloat(1f), NSNumber.FromFloat(0.8f), NSNumber.FromInt32(0x60), NSNumber.FromFloat(1f), NSNumber.FromInt32(40),
NSNumber.FromBoolean(true), NSNumber.FromFloat(1f)];
values.RegisterDefaults(new NSDictionary<NSString, NSObject>(keys, defaults));
InputMode = Enum.IsDefined(typeof(AudioInputMode), (int)values.IntForKey("voice.inputMode"))
@@ -46,6 +47,7 @@ internal sealed class MacSettings
AuxiliaryEnabled = values.BoolForKey("voice.auxEnabled");
AuxiliaryDeviceId = EmptyToNull(values.StringForKey("voice.auxDeviceUID"));
AuxiliaryGain = Math.Clamp(values.FloatForKey("voice.auxGain"), 0f, 4f);
int audioBuffer = checked((int)values.IntForKey("voice.audioBufferMs")); AudioBufferMilliseconds = audioBuffer is 20 or 40 or 60 ? audioBuffer : 40;
EventSounds = values.BoolForKey("feedback.sounds");
SpokenEvents = values.BoolForKey("feedback.speech");
EventVolume = Math.Clamp(values.FloatForKey("feedback.volume"), 0f, 1f);
@@ -59,7 +61,7 @@ internal sealed class MacSettings
values.SetFloat(InputGain, "voice.inputGain"); values.SetFloat(OutputGain, "voice.outputGain");
values.SetBool(InputNoiseReduction, "voice.inputNoiseReduction"); values.SetBool(StereoMicrophone, "voice.stereoMic");
values.SetInt(PushToTalkKeyCode, "voice.pttKeyCode"); Set("voice.inputDevice", InputDeviceId); Set("voice.outputDevice", OutputDeviceId);
values.SetBool(AuxiliaryEnabled, "voice.auxEnabled"); Set("voice.auxDeviceUID", AuxiliaryDeviceId); values.SetFloat(AuxiliaryGain, "voice.auxGain");
values.SetBool(AuxiliaryEnabled, "voice.auxEnabled"); Set("voice.auxDeviceUID", AuxiliaryDeviceId); values.SetFloat(AuxiliaryGain, "voice.auxGain"); values.SetInt(AudioBufferMilliseconds, "voice.audioBufferMs");
values.SetBool(EventSounds, "feedback.sounds"); values.SetBool(SpokenEvents, "feedback.speech"); values.SetFloat(EventVolume, "feedback.volume");
values.SetBool(SelfTalkSounds, "feedback.selfTalk"); values.SetBool(PushToTalkSound, "feedback.ptt"); values.Synchronize();
}
clients/apple/dotnet/VoiceCat.Mac/MainWindowController.cs
+3
View File
@@ -210,6 +210,8 @@ internal sealed class MainWindowController : NSWindowController
if (settings.InputMode != AudioInputMode.PushToTalk) { client.Audio.PushToTalk = false; pushToTalkEngaged = false; }
client.Audio.InputGain = settings.InputGain; client.Audio.OutputGain = settings.OutputGain;
client.Audio.InputNoiseReduction = settings.InputNoiseReduction;
client.Audio.DeviceBufferMilliseconds = settings.AudioBufferMilliseconds;
if (playback is not null) playback.BufferMilliseconds = settings.AudioBufferMilliseconds;
if (auxiliaryStreamId != 0) client.Audio.SetLocalGain(auxiliaryStreamId, settings.AuxiliaryGain);
}
@@ -367,6 +369,7 @@ internal sealed class MainWindowController : NSWindowController
activeOutputDevice = settings.OutputDeviceId ?? SelectedDevice(outputDevice);
activeStereo = settings.StereoMicrophone;
playback = audioBackend.OpenPlayback(activeOutputDevice);
playback.BufferMilliseconds = settings.AudioBufferMilliseconds;
client.Audio.MixedPcm += PlayMixedPcm;
StreamInfo stream = await client.StartStreamAsync(StreamKind.StreamMic, "Microphone", settings.StereoMicrophone ? 2 : 1);
microphoneStreamId = stream.StreamId;
clients/apple/dotnet/VoiceCat.Mac/SettingsWindowController.cs
+8 -2
View File
@@ -26,15 +26,18 @@ internal sealed class SettingsWindowController : NSWindowController
private readonly NSButton pttSound = NSButton.CreateCheckbox("Push-to-talk cue", () => { });
private readonly NSButton pttKey = new(new CGRect(220, 12, 200, 32)) { Title = "Change PTT key…" };
private readonly NSSlider eventVolume = new(new CGRect(160, 505, 260, 24)) { MinValue = 0, MaxValue = 100 };
private readonly NSPopUpButton audioBuffer = new(new CGRect(160, 545, 260, 28), false);
internal SettingsWindowController(MainWindowController owner, MacSettings settings, MacAudioBackend audio) : base(
new NSWindow(new CGRect(0, 0, 460, 620), NSWindowStyle.Titled | NSWindowStyle.Closable,
new NSWindow(new CGRect(0, 0, 460, 660), NSWindowStyle.Titled | NSWindowStyle.Closable,
NSBackingStore.Buffered, false))
{
this.owner = owner; this.settings = settings; this.audio = audio;
Window!.Title = "VoiceCat Settings"; Window.Center();
NSView view = Window.ContentView!;
mode.AddItems(["Voice activation", "Push to talk", "Always on"]);
audioBuffer.AddItems(["Low latency (20 ms)", "Balanced (40 ms)", "Stable (60 ms)"]);
Add(view, "Audio buffering", audioBuffer, 555);
Add(view, "Sound volume", eventVolume, 515);
Add(view, "Input mode", mode, 465); Add(view, "VAD sensitivity", vad, 425); Add(view, "Microphone volume", inputGain, 385);
noiseReduction.Frame = new CGRect(160, 340, 260, 24); stereo.Frame = new CGRect(160, 315, 260, 24);
@@ -46,7 +49,7 @@ internal sealed class SettingsWindowController : NSWindowController
selfTalk.Frame = new CGRect(265, 45, 180, 24); pttSound.Frame = new CGRect(25, 18, 180, 24);
view.AddSubview(sounds); view.AddSubview(speech); view.AddSubview(selfTalk); view.AddSubview(pttSound); view.AddSubview(pttKey);
foreach (NSControl control in new NSControl[] { mode, vad, inputGain, noiseReduction, stereo, input, output,
outputGain, auxiliary, auxiliaryDevice, auxiliaryGain, sounds, speech, selfTalk, pttSound, eventVolume }) control.Activated += Changed;
outputGain, auxiliary, auxiliaryDevice, auxiliaryGain, sounds, speech, selfTalk, pttSound, eventVolume, audioBuffer }) control.Activated += Changed;
pttKey.Activated += CapturePushToTalkKey;
Populate(input, true, settings.InputDeviceId); Populate(output, false, settings.OutputDeviceId); Populate(auxiliaryDevice, true, settings.AuxiliaryDeviceId);
LoadValues(); SetAccessibility();
@@ -76,6 +79,7 @@ internal sealed class SettingsWindowController : NSWindowController
eventVolume.DoubleValue = settings.EventVolume * 100;
selfTalk.State = State(settings.SelfTalkSounds); pttSound.State = State(settings.PushToTalkSound);
pttKey.Title = $"PTT key code: {settings.PushToTalkKeyCode}";
audioBuffer.SelectItem(settings.AudioBufferMilliseconds switch { 20 => 0, 60 => 2, _ => 1 });
UpdateEnabled();
}
@@ -102,6 +106,7 @@ internal sealed class SettingsWindowController : NSWindowController
settings.AuxiliaryGain = (float)auxiliaryGain.DoubleValue / 100; settings.EventSounds = On(sounds);
settings.EventVolume = (float)eventVolume.DoubleValue / 100; settings.SpokenEvents = On(speech);
settings.SelfTalkSounds = On(selfTalk); settings.PushToTalkSound = On(pttSound);
settings.AudioBufferMilliseconds = audioBuffer.IndexOfSelectedItem switch { 0 => 20, 2 => 60, _ => 40 };
settings.Save(); UpdateEnabled(); await owner.ApplySettingsAsync();
}
@@ -113,6 +118,7 @@ internal sealed class SettingsWindowController : NSWindowController
((INSAccessibility)output).AccessibilityLabel = "Output device"; ((INSAccessibility)outputGain).AccessibilityLabel = "Output volume";
((INSAccessibility)auxiliaryDevice).AccessibilityLabel = "Auxiliary input device"; ((INSAccessibility)auxiliaryGain).AccessibilityLabel = "Auxiliary input volume";
((INSAccessibility)eventVolume).AccessibilityLabel = "Event sound volume";
((INSAccessibility)audioBuffer).AccessibilityLabel = "Audio buffering";
}
private static bool On(NSButton button) => button.State == NSCellStateValue.On;
private static NSCellStateValue State(bool value) => value ? NSCellStateValue.On : NSCellStateValue.Off;