feat(clients): add aux outgoing stream (mic + second input device) on Windows + macOS
Lets a user transmit a second hardware input device (e.g. line-in / aux) alongside the mic, with its own device picker and volume, from Audio Settings. No core/ABI/proto changes: the aux is a VC_STREAM_AUX_DEVICE stream started with external_feed=1 and fed via vc_stream_feed_pcm (the same external-feed pipeline screen-audio uses). Per-kind local_streams_ already allows mic + screen + one aux to coexist; volume is a client-side gain multiply (the core's vc_set_input_gain is mic-only/global). Aux is always-on (core never gates AUX_DEVICE on VAD/PTT) and is tied to the voice session. Windows: new Audio/InputDeviceCapture.cs (WASAPI shared-mode capture from a real input endpoint + capture-endpoint enumeration); aux section in AudioSettingsForm.cs; lifecycle in MainForm.cs; persistence in VoiceSettings.cs. macOS: new Audio/InputDeviceCapture.swift (AVAudioEngine input-node tap pinned to the chosen Core Audio device + device enumeration by stable UID); aux section in SettingsWindowController.swift; lifecycle + UserDefaults persistence in MainWindowController.swift; file registered in project.pbxproj. Windows verified (C# solution builds clean; aux confirmed working). macOS build + E2E pending a Mac. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
229
clients/apple/macOS/VoiceCatMac/Audio/InputDeviceCapture.swift
Normal file
229
clients/apple/macOS/VoiceCatMac/Audio/InputDeviceCapture.swift
Normal file
@@ -0,0 +1,229 @@
|
||||
import AVFoundation
|
||||
import CoreAudio
|
||||
|
||||
// InputDeviceCapture — captures a single hardware INPUT device (mic / line-in / aux) on macOS and
|
||||
// emits 20 ms (960 samples/channel @ 48 kHz, interleaved int16) frames for the aux outgoing stream.
|
||||
//
|
||||
// The input-device analogue of ScreenAudioCapture (which captures system audio via
|
||||
// ScreenCaptureKit). The core already owns ONE capture device (the mic) and can't open a second
|
||||
// arbitrary input, so for the aux stream the client captures the chosen device here and feeds PCM
|
||||
// into the core via `vc_stream_feed_pcm` — the same external-feed pipeline screen audio uses.
|
||||
//
|
||||
// Device selection: an AVAudioEngine's input node wraps an AUHAL audio unit; setting
|
||||
// kAudioOutputUnitProperty_CurrentDevice on it pins capture to a specific Core Audio device. We
|
||||
// pass the device's *UID* (stable across reboots/replugs, unlike AudioDeviceID) and resolve it at
|
||||
// start. The tap delivers Float32; AVAudioConverter resamples/quantises to 48 kHz int16.
|
||||
final class InputDeviceCapture {
|
||||
|
||||
/// Receives a full 20 ms frame: (interleaved int16 PCM, samplesPerChannel = 960, channels).
|
||||
typealias PcmHandler = (UnsafePointer<Int16>, Int, UInt32) -> Void
|
||||
|
||||
enum CaptureError: Error { case deviceNotFound, engineStartFailed(OSStatus) }
|
||||
|
||||
private static let frameSamplesPerChannel = 960 // 20 ms @ 48 kHz
|
||||
|
||||
private let deviceUID: String? // nil = system default input device
|
||||
private let onPcm: PcmHandler
|
||||
private let engine = AVAudioEngine()
|
||||
private var converter: AVAudioConverter?
|
||||
private var outFormat: AVAudioFormat?
|
||||
private var channels = 1
|
||||
|
||||
/// Interleaved int16 carry-over between tap callbacks (the tap buffer doesn't align to 20 ms),
|
||||
/// drained in whole `frameSamplesPerChannel * channels` chunks. Only touched on the tap queue.
|
||||
private var pending: [Int16] = []
|
||||
|
||||
init(deviceUID: String?, onPcm: @escaping PcmHandler) {
|
||||
self.deviceUID = deviceUID
|
||||
self.onPcm = onPcm
|
||||
}
|
||||
|
||||
/// Begin capture. Throws if the device can't be resolved or the engine fails to start.
|
||||
func start() throws {
|
||||
let input = engine.inputNode
|
||||
|
||||
// Pin the engine's AUHAL to the chosen device (skip for default — the engine already uses
|
||||
// the system default input). Must happen before reading inputFormat, which changes with
|
||||
// the selected device.
|
||||
if let deviceUID, let devId = Self.deviceID(forUID: deviceUID) {
|
||||
var dev = devId
|
||||
let status = AudioUnitSetProperty(input.audioUnit!,
|
||||
kAudioOutputUnitProperty_CurrentDevice,
|
||||
kAudioUnitScope_Global, 0,
|
||||
&dev, UInt32(MemoryLayout<AudioDeviceID>.size))
|
||||
if status != noErr { throw CaptureError.engineStartFailed(status) }
|
||||
} else if deviceUID != nil {
|
||||
throw CaptureError.deviceNotFound
|
||||
}
|
||||
|
||||
let inFormat = input.inputFormat(forBus: 0)
|
||||
channels = max(1, min(2, Int(inFormat.channelCount)))
|
||||
guard let out = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000,
|
||||
channels: AVAudioChannelCount(channels), interleaved: true)
|
||||
else { throw CaptureError.deviceNotFound }
|
||||
outFormat = out
|
||||
converter = AVAudioConverter(from: inFormat, to: out)
|
||||
|
||||
input.installTap(onBus: 0, bufferSize: 960, format: inFormat) { [weak self] buf, _ in
|
||||
self?.process(buf)
|
||||
}
|
||||
engine.prepare()
|
||||
do { try engine.start() }
|
||||
catch { throw CaptureError.engineStartFailed(-1) }
|
||||
}
|
||||
|
||||
/// Stop capture and tear down the engine. Safe to call multiple times.
|
||||
func stop() {
|
||||
engine.inputNode.removeTap(onBus: 0)
|
||||
if engine.isRunning { engine.stop() }
|
||||
converter = nil
|
||||
pending.removeAll(keepingCapacity: false)
|
||||
}
|
||||
|
||||
// MARK: - Conversion (called on the tap's realtime thread)
|
||||
|
||||
private func process(_ inBuf: AVAudioPCMBuffer) {
|
||||
guard let converter, let outFormat else { return }
|
||||
|
||||
// Output capacity must cover up-sampling (e.g. 44.1 → 48 kHz) plus slack.
|
||||
let ratio = outFormat.sampleRate / inBuf.format.sampleRate
|
||||
let cap = AVAudioFrameCount(Double(inBuf.frameLength) * ratio + 32)
|
||||
guard cap > 0, let outBuf = AVAudioPCMBuffer(pcmFormat: outFormat, frameCapacity: cap)
|
||||
else { return }
|
||||
|
||||
var fed = false
|
||||
var err: NSError?
|
||||
let status = converter.convert(to: outBuf, error: &err) { _, outStatus in
|
||||
if fed { outStatus.pointee = .noDataNow; return nil }
|
||||
fed = true
|
||||
outStatus.pointee = .haveData
|
||||
return inBuf
|
||||
}
|
||||
guard status != .error, outBuf.frameLength > 0,
|
||||
let ch = outBuf.int16ChannelData else { return }
|
||||
|
||||
// Interleaved int16: all channels live in the first buffer (ch[0]).
|
||||
let n = Int(outBuf.frameLength) * channels
|
||||
pending.append(contentsOf: UnsafeBufferPointer(start: ch[0], count: n))
|
||||
emit()
|
||||
}
|
||||
|
||||
/// Fire `onPcm` for every whole 20 ms frame accumulated.
|
||||
private func emit() {
|
||||
let full = Self.frameSamplesPerChannel * channels
|
||||
while pending.count >= full {
|
||||
pending.withUnsafeBufferPointer { buf in
|
||||
onPcm(buf.baseAddress!, Self.frameSamplesPerChannel, UInt32(channels))
|
||||
}
|
||||
pending.removeFirst(full)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - UID → AudioDeviceID resolution
|
||||
|
||||
private static func deviceID(forUID uid: String) -> AudioDeviceID? {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioHardwarePropertyTranslateUIDToDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var deviceID = AudioDeviceID(0)
|
||||
var cfUID = uid as CFString
|
||||
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
|
||||
let status = withUnsafeMutablePointer(to: &cfUID) { uidPtr -> OSStatus in
|
||||
AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr,
|
||||
UInt32(MemoryLayout<CFString>.size), uidPtr,
|
||||
&size, &deviceID)
|
||||
}
|
||||
return (status == noErr && deviceID != 0) ? deviceID : nil
|
||||
}
|
||||
}
|
||||
|
||||
// InputDeviceInfo / InputDeviceEnumerator — Core Audio input-device enumeration for the aux-stream
|
||||
// picker. Separate from the core's vc_list_devices (whose ids are miniaudio-opaque and can't be
|
||||
// passed to Core Audio); the aux device is client-captured, so the picker uses device UIDs.
|
||||
struct InputDeviceInfo: Equatable {
|
||||
let uid: String // stable across reboots/replugs — what we persist
|
||||
let name: String
|
||||
let isDefault: Bool
|
||||
}
|
||||
|
||||
enum InputDeviceEnumerator {
|
||||
|
||||
/// All Core Audio devices that expose at least one input channel.
|
||||
static func list() -> [InputDeviceInfo] {
|
||||
let defaultUID = defaultInputUID()
|
||||
var result: [InputDeviceInfo] = []
|
||||
|
||||
for devID in allDeviceIDs() {
|
||||
guard inputChannelCount(devID) > 0 else { continue }
|
||||
guard let uid = stringProperty(devID, kAudioDevicePropertyDeviceUID) else { continue }
|
||||
let name = stringProperty(devID, kAudioObjectPropertyName)
|
||||
?? stringProperty(devID, kAudioDevicePropertyDeviceNameCFString)
|
||||
?? "Unknown input device"
|
||||
result.append(InputDeviceInfo(uid: uid, name: name, isDefault: uid == defaultUID))
|
||||
}
|
||||
return result.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
|
||||
}
|
||||
|
||||
// MARK: - Core Audio helpers
|
||||
|
||||
private static func allDeviceIDs() -> [AudioDeviceID] {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioHardwarePropertyDevices,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var size = UInt32(0)
|
||||
guard AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil,
|
||||
&size) == noErr, size > 0 else { return [] }
|
||||
let count = Int(size) / MemoryLayout<AudioDeviceID>.size
|
||||
var ids = [AudioDeviceID](repeating: 0, count: count)
|
||||
let status = AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr, 0,
|
||||
nil, &size, &ids)
|
||||
return status == noErr ? ids : []
|
||||
}
|
||||
|
||||
private static func inputChannelCount(_ devID: AudioDeviceID) -> Int {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioDevicePropertyStreamConfiguration,
|
||||
mScope: kAudioObjectPropertyScopeInput,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var size = UInt32(0)
|
||||
guard AudioObjectGetPropertyDataSize(devID, &addr, 0, nil, &size) == noErr, size > 0
|
||||
else { return 0 }
|
||||
let bufList = UnsafeMutableRawPointer.allocate(byteCount: Int(size),
|
||||
alignment: MemoryLayout<AudioBufferList>.alignment)
|
||||
defer { bufList.deallocate() }
|
||||
guard AudioObjectGetPropertyData(devID, &addr, 0, nil, &size, bufList) == noErr
|
||||
else { return 0 }
|
||||
let abl = UnsafeMutableAudioBufferListPointer(
|
||||
bufList.assumingMemoryBound(to: AudioBufferList.self))
|
||||
return abl.reduce(0) { $0 + Int($1.mNumberChannels) }
|
||||
}
|
||||
|
||||
private static func defaultInputUID() -> String? {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var devID = AudioDeviceID(0)
|
||||
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
|
||||
guard AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil,
|
||||
&size, &devID) == noErr, devID != 0 else { return nil }
|
||||
return stringProperty(devID, kAudioDevicePropertyDeviceUID)
|
||||
}
|
||||
|
||||
private static func stringProperty(_ devID: AudioDeviceID,
|
||||
_ selector: AudioObjectPropertySelector) -> String? {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: selector,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var cf: CFString? = nil
|
||||
var size = UInt32(MemoryLayout<CFString?>.size)
|
||||
let status = withUnsafeMutablePointer(to: &cf) { ptr in
|
||||
AudioObjectGetPropertyData(devID, &addr, 0, nil, &size, ptr)
|
||||
}
|
||||
guard status == noErr else { return nil }
|
||||
return cf as String?
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user