Talon/voice-cat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(clients): add aux outgoing stream (mic + second input device) on Windows + macOS

Browse Source 
Lets a user transmit a second hardware input device (e.g. line-in / aux)
alongside the mic, with its own device picker and volume, from Audio Settings.

No core/ABI/proto changes: the aux is a VC_STREAM_AUX_DEVICE stream started
with external_feed=1 and fed via vc_stream_feed_pcm (the same external-feed
pipeline screen-audio uses). Per-kind local_streams_ already allows mic +
screen + one aux to coexist; volume is a client-side gain multiply (the core's
vc_set_input_gain is mic-only/global). Aux is always-on (core never gates
AUX_DEVICE on VAD/PTT) and is tied to the voice session.

Windows: new Audio/InputDeviceCapture.cs (WASAPI shared-mode capture from a
real input endpoint + capture-endpoint enumeration); aux section in
AudioSettingsForm.cs; lifecycle in MainForm.cs; persistence in VoiceSettings.cs.

macOS: new Audio/InputDeviceCapture.swift (AVAudioEngine input-node tap pinned
to the chosen Core Audio device + device enumeration by stable UID); aux section
in SettingsWindowController.swift; lifecycle + UserDefaults persistence in
MainWindowController.swift; file registered in project.pbxproj.

Windows verified (C# solution builds clean; aux confirmed working). macOS build
+ E2E pending a Mac.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Talon
2026-06-23 12:56:09 +02:00
parent a48b47d4ca
commit 7249a8fd30
9 changed files with 1210 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift
View File

@@ -31,6 +31,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
internal var micStreamId: UInt32 = 0
private var screenStreamId: UInt32 = 0
private var screenCapture: ScreenAudioCapture?
private var auxStreamId: UInt32 = 0 // 0 = aux (second input device) stream not active
private var auxCapture: InputDeviceCapture? // client-side capture feeding the aux stream
// Last app/exclusion choice from the share picker; reused as the default next time.
private var screenAudioSelection: ScreenAudioSelection = .default
internal var pttKeyCode: UInt16 = 0x60 { // F8
@@ -60,6 +62,9 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
static let vadThreshold = "voice.vadThreshold"
static let inputGain = "voice.inputGain"
static let pttKeyCode = "voice.pttKeyCode"
static let auxEnabled = "voice.auxEnabled"
static let auxDeviceUID = "voice.auxDeviceUID"
static let auxGain = "voice.auxGain"
}
internal var selectedInputMode: VoiceCatInputMode = .voiceActivation {
@@ -73,6 +78,20 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
internal var selectedInputDeviceId: String?
// Aux outgoing stream: a second hardware input device the client captures itself and feeds to
// the core (kind = AUX_DEVICE, external_feed). Device + volume only — aux is always-on (the
// core never gates AUX_DEVICE on VAD/PTT). auxDeviceUID is a Core Audio device UID (stable),
// NOT a core/miniaudio id. auxGain is read live by the capture feed, so the slider is instant.
internal var auxEnabled: Bool = false {
didSet { UserDefaults.standard.set(auxEnabled, forKey: AudioDefaults.auxEnabled) }
}
internal var auxDeviceUID: String? {
didSet { UserDefaults.standard.set(auxDeviceUID, forKey: AudioDefaults.auxDeviceUID) }
}
internal var auxGain: Float = 1.0 {
didSet { UserDefaults.standard.set(auxGain, forKey: AudioDefaults.auxGain) }
}
// MARK: - UI components
private let channelOutlineView = NSOutlineView()
private let userTableView = NSTableView()
@@ -140,6 +159,13 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
if d.object(forKey: AudioDefaults.pttKeyCode) != nil {
pttKeyCode = UInt16(d.integer(forKey: AudioDefaults.pttKeyCode))
}
auxEnabled = d.bool(forKey: AudioDefaults.auxEnabled)
if d.object(forKey: AudioDefaults.auxDeviceUID) != nil {
auxDeviceUID = d.string(forKey: AudioDefaults.auxDeviceUID)
}
if d.object(forKey: AudioDefaults.auxGain) != nil {
auxGain = d.float(forKey: AudioDefaults.auxGain)
}
}
required init?(coder: NSCoder) { fatalError() }
@@ -675,7 +701,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
displayedUsers = []; userTableView.reloadData()
users.removeAll(); talkingUsers.removeAll()
stopScreenCapture()
currentChannelId = 0; micStreamId = 0; screenStreamId = 0
auxCapture?.stop(); auxCapture = nil // connection gone — drop capture, no stopStream
currentChannelId = 0; micStreamId = 0; screenStreamId = 0; auxStreamId = 0
composeField.isEnabled = false; sendButton.isEnabled = false
joinVoiceButton?.isEnabled = false
shareScreenButton?.isEnabled = false
@@ -757,10 +784,12 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
EventFeedback.shared.play(.voiceOn)
NSAccessibility.post(element: logTextView, notification: .announcementRequested,
userInfo: [.announcement: "Joined voice", .priority: NSAccessibilityPriorityLevel.medium])
startAuxStream() // no-op unless the aux stream is enabled in settings
} else {
addActivity("Failed to start microphone: \(result)")
}
} else {
stopAuxStream()
client.setPushToTalk(false)
client.stopStream(micStreamId)
micStreamId = 0
@@ -771,6 +800,84 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
}
// MARK: - Aux input stream (second hardware input device)
/// Called by SettingsWindowController when the user toggles the aux checkbox.
func applyAuxEnabled(_ on: Bool) {
auxEnabled = on
guard micStreamId != 0 else { return } // not in voice — applied on next Join Voice
if on { startAuxStream() } else { stopAuxStream() }
}
/// Called by SettingsWindowController when the user picks a different aux device.
func applyAuxDevice(_ uid: String?) {
auxDeviceUID = uid
if auxStreamId != 0 { restartAuxCapture() }
}
private func startAuxStream() {
guard auxStreamId == 0, auxEnabled else { return }
let (result, streamId) = client.startStream(
StreamDescriptor(kind: .auxDevice, deviceId: nil, label: "Aux device", externalFeed: true))
guard result == .ok else {
addActivity("Failed to start aux stream: \(result)")
return
}
auxStreamId = streamId
startAuxCapture()
addActivity("Aux input stream active")
}
private func startAuxCapture() {
let capture = InputDeviceCapture(deviceUID: auxDeviceUID) { [weak self] ptr, spc, ch in
self?.feedAux(ptr, samplesPerChannel: spc, channels: ch)
}
auxCapture = capture
do { try capture.start() }
catch {
addActivity("Failed to open aux input device")
stopAuxStream()
}
}
// Re-open the capture on a different device while the aux stream stays up (the core stream id
// is unchanged — only the client-side capture source changes).
private func restartAuxCapture() {
guard auxStreamId != 0 else { return }
auxCapture?.stop()
auxCapture = nil
startAuxCapture()
}
private func stopAuxStream() {
auxCapture?.stop()
auxCapture = nil
if auxStreamId != 0 {
client.stopStream(auxStreamId)
auxStreamId = 0
}
}
// Fired on the capture's realtime thread. feedPcm is thread-safe. Gain is read live from
// auxGain each frame so the volume slider takes effect immediately.
private func feedAux(_ pcm: UnsafePointer<Int16>, samplesPerChannel: Int, channels: UInt32) {
guard auxStreamId != 0 else { return }
let gain = auxGain
if gain != 1.0 {
let n = samplesPerChannel * Int(channels)
var scaled = [Int16](repeating: 0, count: n)
for i in 0..<n {
let v = (Float(pcm[i]) * gain).rounded()
scaled[i] = Int16(max(-32768, min(32767, v)))
}
client.feedPcm(streamId: auxStreamId, pcm: scaled,
samplesPerChannel: samplesPerChannel, channels: channels)
} else {
client.feedPcm(streamId: auxStreamId, pcm: pcm,
samplesPerChannel: samplesPerChannel, channels: channels)
}
}
/// Update toolbar button labels/states to reflect whether voice is active. Mirrors the
/// Windows client's `SetVoiceJoinedState`.
private func setVoiceJoinedState(_ joined: Bool) {
@@ -1158,6 +1265,7 @@ final class MainWindowController: NSWindowController, NSWindowDelegate {
}
client.setPushToTalk(false)
stopScreenCapture()
if auxStreamId != 0 { stopAuxStream() }
if screenStreamId != 0 { client.stopStream(screenStreamId) }
if micStreamId != 0 { client.stopStream(micStreamId) }
client.onLevel = nil

121
clients/apple/macOS/VoiceCatMac/Windows/SettingsWindowController.swift
View File

@@ -58,6 +58,22 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
}()
private let inputGainValueLabel = NSTextField(labelWithString: "100%")
// Aux input stream: a second outgoing stream from another hardware input device (e.g. line-in
// / aux), captured client-side. Device + volume only — aux is always-on. Persisted via
// MainWindowController.auxEnabled / auxDeviceUID / auxGain.
private let auxCheckbox = NSButton(checkboxWithTitle: "Aux input stream (second device)",
target: nil, action: nil)
private let auxDevicePicker = NSPopUpButton()
private let auxRefreshButton = NSButton()
private let auxGainSlider: NSSlider = {
let s = NSSlider(value: 100, minValue: 0, maxValue: 300, target: nil, action: nil)
s.numberOfTickMarks = 0
return s
}()
private let auxGainValueLabel = NSTextField(labelWithString: "100%")
private let auxDeviceLabel = NSTextField(labelWithString: "Aux device:")
private let auxGainLabel = NSTextField(labelWithString: "Aux volume:")
// Notification feedback controls. Read/write UserDefaults with the same keys VoiceCatCore's
// FeedbackSettings reads, so EventFeedback honours these immediately.
private let soundsCheckbox = NSButton(checkboxWithTitle: "Event sounds", target: nil, action: nil)
@@ -94,6 +110,7 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
buildUI()
syncFromMainController()
loadInputDevices()
loadAuxDevices()
}
required init?(coder: NSCoder) { fatalError() }
@@ -175,6 +192,41 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
inputGainRow.orientation = .horizontal
inputGainRow.spacing = 8
// ── Aux input stream ──────────────────────────────────────────────────
let auxHeader = NSTextField(labelWithString: "Aux input stream")
auxHeader.font = .boldSystemFont(ofSize: NSFont.systemFontSize)
auxCheckbox.target = self
auxCheckbox.action = #selector(auxEnabledChanged)
auxCheckbox.setAccessibilityLabel("Enable aux input stream")
auxCheckbox.setAccessibilityHelp("Transmit a second hardware input device alongside your microphone.")
auxDeviceLabel.setAccessibilityLabel("Aux input device")
auxDevicePicker.setAccessibilityLabel("Aux input device")
auxDevicePicker.target = self
auxDevicePicker.action = #selector(auxDeviceChanged)
auxRefreshButton.title = "↺"
auxRefreshButton.bezelStyle = .rounded
auxRefreshButton.target = self
auxRefreshButton.action = #selector(refreshAuxDevicesClicked)
auxRefreshButton.setAccessibilityLabel("Refresh aux device list")
auxRefreshButton.toolTip = "Refresh"
auxGainLabel.setAccessibilityLabel("Aux volume")
auxGainSlider.target = self
auxGainSlider.action = #selector(auxGainChanged)
auxGainSlider.setAccessibilityLabel("Aux volume")
auxGainSlider.setAccessibilityHelp("Volume of the aux input stream. 100% is unity.")
auxGainValueLabel.setAccessibilityLabel("Aux volume value")
let auxDeviceRow = NSStackView(views: [auxDeviceLabel, auxDevicePicker, auxRefreshButton])
auxDeviceRow.orientation = .horizontal
auxDeviceRow.spacing = 8
let auxGainRow = NSStackView(views: [auxGainLabel, auxGainSlider, auxGainValueLabel])
auxGainRow.orientation = .horizontal
auxGainRow.spacing = 8
// Notifications
let notificationsHeader = NSTextField(labelWithString: "Notifications")
notificationsHeader.font = .boldSystemFont(ofSize: NSFont.systemFontSize)
@@ -198,7 +250,8 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
volumeRow.spacing = 8
let stack = NSStackView(views: [inputModeRow, vadRow, inputGainRow, pttRow, deviceRow,
levelRow, notificationsHeader, soundsCheckbox, volumeRow,
levelRow, auxHeader, auxCheckbox, auxDeviceRow, auxGainRow,
notificationsHeader, soundsCheckbox, volumeRow,
speechCheckbox, selfTalkCheckbox, pttSoundCheckbox])
stack.orientation = .vertical
stack.spacing = 12
@@ -218,6 +271,8 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
levelMeter.widthAnchor.constraint(equalToConstant: 200),
devicePicker.widthAnchor.constraint(greaterThanOrEqualToConstant: 180),
soundsVolumeSlider.widthAnchor.constraint(greaterThanOrEqualToConstant: 200),
auxDevicePicker.widthAnchor.constraint(greaterThanOrEqualToConstant: 180),
auxGainSlider.widthAnchor.constraint(greaterThanOrEqualToConstant: 180),
])
syncNotificationControls()
@@ -265,6 +320,11 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
inputGainSlider.doubleValue = Double(mc.inputGain * 100)
updateInputGainLabel()
auxCheckbox.state = mc.auxEnabled ? .on : .off
auxGainSlider.doubleValue = Double(mc.auxGain * 100)
updateAuxGainLabel()
updateAuxControlsEnabled()
updateConditionalControls()
}
@@ -338,6 +398,65 @@ final class SettingsWindowController: NSWindowController, NSWindowDelegate {
}
}
// MARK: - Aux input stream actions
@objc private func auxEnabledChanged() {
let on = auxCheckbox.state == .on
updateAuxControlsEnabled()
mainController?.applyAuxEnabled(on)
}
@objc private func refreshAuxDevicesClicked() { loadAuxDevices() }
@objc private func auxDeviceChanged() {
let uid = auxDevicePicker.selectedItem?.representedObject as? String
mainController?.applyAuxDevice(uid)
}
@objc private func auxGainChanged() {
mainController?.auxGain = Float(auxGainSlider.doubleValue) / 100.0
updateAuxGainLabel()
}
private func updateAuxGainLabel() {
let pct = Int(auxGainSlider.doubleValue.rounded())
auxGainValueLabel.stringValue = "\(pct)%"
auxGainSlider.setAccessibilityValue("\(pct) percent")
}
private func updateAuxControlsEnabled() {
let on = auxCheckbox.state == .on
auxDeviceLabel.isEnabled = on
auxDevicePicker.isEnabled = on
auxRefreshButton.isEnabled = on
auxGainLabel.isEnabled = on
auxGainSlider.isEnabled = on
auxGainValueLabel.isEnabled = on
}
private func loadAuxDevices() {
let devices = InputDeviceEnumerator.list()
let prevSelected = (auxDevicePicker.selectedItem?.representedObject as? String)
?? mainController?.auxDeviceUID
auxDevicePicker.removeAllItems()
for d in devices {
let item = NSMenuItem(title: d.name, action: nil, keyEquivalent: "")
item.representedObject = d.uid
auxDevicePicker.menu?.addItem(item)
}
if let prev = prevSelected,
let item = auxDevicePicker.itemArray.first(where: { ($0.representedObject as? String) == prev }) {
auxDevicePicker.select(item)
} else if let def = devices.first(where: { $0.isDefault }),
let item = auxDevicePicker.itemArray.first(where: { ($0.representedObject as? String) == def.uid }) {
auxDevicePicker.select(item)
} else if auxDevicePicker.numberOfItems > 0 {
auxDevicePicker.selectItem(at: 0)
}
// Persist the resolved selection so a relaunch (or Join Voice) opens the same device.
mainController?.auxDeviceUID = auxDevicePicker.selectedItem?.representedObject as? String
}
// MARK: - Level meter (called by MainWindowController)
func updateLevel(rms: Float) {