Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(clients): add aux outgoing stream (mic + second input device) on Windows + macOS

Browse Source 
Lets a user transmit a second hardware input device (e.g. line-in / aux)
alongside the mic, with its own device picker and volume, from Audio Settings.

No core/ABI/proto changes: the aux is a VC_STREAM_AUX_DEVICE stream started
with external_feed=1 and fed via vc_stream_feed_pcm (the same external-feed
pipeline screen-audio uses). Per-kind local_streams_ already allows mic +
screen + one aux to coexist; volume is a client-side gain multiply (the core's
vc_set_input_gain is mic-only/global). Aux is always-on (core never gates
AUX_DEVICE on VAD/PTT) and is tied to the voice session.

Windows: new Audio/InputDeviceCapture.cs (WASAPI shared-mode capture from a
real input endpoint + capture-endpoint enumeration); aux section in
AudioSettingsForm.cs; lifecycle in MainForm.cs; persistence in VoiceSettings.cs.

macOS: new Audio/InputDeviceCapture.swift (AVAudioEngine input-node tap pinned
to the chosen Core Audio device + device enumeration by stable UID); aux section
in SettingsWindowController.swift; lifecycle + UserDefaults persistence in
MainWindowController.swift; file registered in project.pbxproj.

Windows verified (C# solution builds clean; aux confirmed working). macOS build
+ E2E pending a Mac.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-23 12:56:09 +02:00
parent a48b47d4ca
commit 7249a8fd30
9 changed files with 1210 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift
View File

@@ -31,6 +31,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
internal var micStreamId: UInt32 = 0
private var screenStreamId: UInt32 = 0
private var screenCapture: ScreenAudioCapture?
private var auxStreamId: UInt32 = 0 // 0 = aux (second input device) stream not active
private var auxCapture: InputDeviceCapture? // client-side capture feeding the aux stream
// Last app/exclusion choice from the share picker; reused as the default next time.
private var screenAudioSelection: ScreenAudioSelection = .default
internal var pttKeyCode: UInt16 = 0x60 { // F8
@@ -60,6 +62,9 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
static let vadThreshold = "voice.vadThreshold"
static let inputGain = "voice.inputGain"
static let pttKeyCode = "voice.pttKeyCode"
static let auxEnabled = "voice.auxEnabled"
static let auxDeviceUID = "voice.auxDeviceUID"
static let auxGain = "voice.auxGain"
}
internal var selectedInputMode: VoiceCatInputMode = .voiceActivation {
@@ -73,6 +78,20 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
internal var selectedInputDeviceId: String?
// Aux outgoing stream: a second hardware input device the client captures itself and feeds to
// the core (kind = AUX_DEVICE, external_feed). Device + volume only — aux is always-on (the
// core never gates AUX_DEVICE on VAD/PTT). auxDeviceUID is a Core Audio device UID (stable),
// NOT a core/miniaudio id. auxGain is read live by the capture feed, so the slider is instant.
internal var auxEnabled: Bool = false {
didSet { UserDefaults.standard.set(auxEnabled, forKey: AudioDefaults.auxEnabled) }
}
internal var auxDeviceUID: String? {
didSet { UserDefaults.standard.set(auxDeviceUID, forKey: AudioDefaults.auxDeviceUID) }
}
internal var auxGain: Float = 1.0 {
didSet { UserDefaults.standard.set(auxGain, forKey: AudioDefaults.auxGain) }
}
// MARK: - UI components
private let channelOutlineView = NSOutlineView()
private let userTableView = NSTableView()
@@ -140,6 +159,13 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
if d.object(forKey: AudioDefaults.pttKeyCode) != nil {
pttKeyCode = UInt16(d.integer(forKey: AudioDefaults.pttKeyCode))
}
auxEnabled = d.bool(forKey: AudioDefaults.auxEnabled)
if d.object(forKey: AudioDefaults.auxDeviceUID) != nil {
auxDeviceUID = d.string(forKey: AudioDefaults.auxDeviceUID)
}
if d.object(forKey: AudioDefaults.auxGain) != nil {
auxGain = d.float(forKey: AudioDefaults.auxGain)
}
}
required init?(coder: NSCoder) { fatalError() }
@@ -675,7 +701,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
displayedUsers = []; userTableView.reloadData()
users.removeAll(); talkingUsers.removeAll()
stopScreenCapture()
currentChannelId = 0; micStreamId = 0; screenStreamId = 0
auxCapture?.stop(); auxCapture = nil // connection gone — drop capture, no stopStream
currentChannelId = 0; micStreamId = 0; screenStreamId = 0; auxStreamId = 0
composeField.isEnabled = false; sendButton.isEnabled = false
joinVoiceButton?.isEnabled = false
shareScreenButton?.isEnabled = false
@@ -757,10 +784,12 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
EventFeedback.shared.play(.voiceOn)
NSAccessibility.post(element: logTextView, notification: .announcementRequested,
userInfo: [.announcement: "Joined voice", .priority: NSAccessibilityPriorityLevel.medium])
startAuxStream() // no-op unless the aux stream is enabled in settings
} else {
addActivity("Failed to start microphone: \(result)")
}
} else {
stopAuxStream()
client.setPushToTalk(false)
client.stopStream(micStreamId)
micStreamId = 0
@@ -771,6 +800,84 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
}
// MARK: - Aux input stream (second hardware input device)
/// Called by SettingsWindowController when the user toggles the aux checkbox.
func applyAuxEnabled(_ on: Bool) {
auxEnabled = on
guard micStreamId != 0 else { return } // not in voice — applied on next Join Voice
if on { startAuxStream() } else { stopAuxStream() }
}
/// Called by SettingsWindowController when the user picks a different aux device.
func applyAuxDevice(_ uid: String?) {
auxDeviceUID = uid
if auxStreamId != 0 { restartAuxCapture() }
}
private func startAuxStream() {
guard auxStreamId == 0, auxEnabled else { return }
let (result, streamId) = client.startStream(
StreamDescriptor(kind: .auxDevice, deviceId: nil, label: "Aux device", externalFeed: true))
guard result == .ok else {
addActivity("Failed to start aux stream: \(result)")
return
}
auxStreamId = streamId
startAuxCapture()
addActivity("Aux input stream active")
}
private func startAuxCapture() {
let capture = InputDeviceCapture(deviceUID: auxDeviceUID) { [weak self] ptr, spc, ch in
self?.feedAux(ptr, samplesPerChannel: spc, channels: ch)
}
auxCapture = capture
do { try capture.start() }
catch {
addActivity("Failed to open aux input device")
stopAuxStream()
}
}
// Re-open the capture on a different device while the aux stream stays up (the core stream id
// is unchanged — only the client-side capture source changes).
private func restartAuxCapture() {
guard auxStreamId != 0 else { return }
auxCapture?.stop()
auxCapture = nil
startAuxCapture()
}
private func stopAuxStream() {
auxCapture?.stop()
auxCapture = nil
if auxStreamId != 0 {
client.stopStream(auxStreamId)
auxStreamId = 0
}
}
// Fired on the capture's realtime thread. feedPcm is thread-safe. Gain is read live from
// auxGain each frame so the volume slider takes effect immediately.
private func feedAux(_ pcm: UnsafePointer<Int16>, samplesPerChannel: Int, channels: UInt32) {
guard auxStreamId != 0 else { return }
let gain = auxGain
if gain != 1.0 {
let n = samplesPerChannel * Int(channels)
var scaled = [Int16](repeating: 0, count: n)
for i in 0..<n {
let v = (Float(pcm[i]) * gain).rounded()
scaled[i] = Int16(max(-32768, min(32767, v)))
}
client.feedPcm(streamId: auxStreamId, pcm: scaled,
samplesPerChannel: samplesPerChannel, channels: channels)
} else {
client.feedPcm(streamId: auxStreamId, pcm: pcm,
samplesPerChannel: samplesPerChannel, channels: channels)
}
}
/// Update toolbar button labels/states to reflect whether voice is active. Mirrors the
/// Windows client's `SetVoiceJoinedState`.
private func setVoiceJoinedState(_ joined: Bool) {
@@ -1158,6 +1265,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
client.setPushToTalk(false)
stopScreenCapture()
if auxStreamId != 0 { stopAuxStream() }
if screenStreamId != 0 { client.stopStream(screenStreamId) }
if micStreamId != 0 { client.stopStream(micStreamId) }
client.onLevel = nil