diff --git a/PROGRESS.md b/PROGRESS.md index 01338c8..3e92110 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -10,6 +10,30 @@ up instantly. Newest status at the top. ## ▶ Where we left off / next action +- **Done (2026-06-23):** **Aux outgoing stream (mic + a second input device) — Windows + macOS.** + Users can now transmit a second hardware input device (e.g. line-in / aux) alongside the mic, with + its own device picker and volume, from Audio Settings. **No core/ABI/proto changes** — the aux is + a `VC_STREAM_AUX_DEVICE` stream started with `external_feed=1`, captured client-side, and fed via + `vc_stream_feed_pcm` (the same external-feed pipeline screen-audio uses). Per-kind `local_streams_` + already allows mic + screen + one aux to coexist; volume is a client-side gain multiply (the core's + `vc_set_input_gain` is mic-only/global). The aux is always-on (the core never gates `AUX_DEVICE` on + VAD/PTT) and is tied to the voice session (started on Join Voice when enabled, stopped on Leave). + - **Windows:** new `Audio/InputDeviceCapture.cs` (WASAPI shared-mode capture from a real input + endpoint via `IMMDevice.Activate(IAudioClient)`, 48 kHz/s16, 20 ms frames) + `InputDeviceEnumerator` + (WASAPI capture-endpoint list — separate from the core's miniaudio ids). Aux section in + `AudioSettingsForm.cs` (enable checkbox, device combo, refresh, volume slider, accessible names, + live-apply + Cancel revert via callbacks). Lifecycle in `MainForm.cs` (`_auxStreamId` + + `InputDeviceCapture`). Persisted in `VoiceSettings.cs` (`AuxEnabled/AuxDeviceId/AuxGain`). + - **macOS:** new `Audio/InputDeviceCapture.swift` (AVAudioEngine input-node tap pinned to the chosen + Core Audio device via `kAudioOutputUnitProperty_CurrentDevice`; AVAudioConverter → 48 kHz int16; + 20 ms framing modelled on `ScreenAudioCapture`) + `InputDeviceEnumerator` (Core Audio device list + by stable UID). Aux section in `SettingsWindowController.swift`; lifecycle + UserDefaults + persistence (`voice.aux*`) in `MainWindowController.swift`. New file added to `project.pbxproj`. + - **Verify status:** Windows C# solution builds clean (0 warn/0 err); `ctest` core suite unchanged + (no core edits). **Next (manual):** on a Mac, build `VoiceCatMac.xcodeproj`; then two-client E2E — + enable aux on a second input device, confirm two distinct streams for the sender and that the aux + volume slider moves the aux level independently of the mic; confirm persistence across relaunch. + - **Done (2026-06-23):** **Input-settings persistence, mic input gain, + two iOS bugs (all 3 clients).** Four fixes: 1. **Input settings now persist.** Transmission mode (VAD/PTT/Always-On), VAD threshold, and the diff --git a/clients/apple/macOS/VoiceCatMac.xcodeproj/project.pbxproj b/clients/apple/macOS/VoiceCatMac.xcodeproj/project.pbxproj index 3fd4342..a626aa4 100644 --- a/clients/apple/macOS/VoiceCatMac.xcodeproj/project.pbxproj +++ b/clients/apple/macOS/VoiceCatMac.xcodeproj/project.pbxproj @@ -31,6 +31,7 @@ AAAA00000000000000000048 /* UserPickerSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000047 /* UserPickerSheet.swift */; }; AAAA0000000000000000004A /* SettingsWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000049 /* SettingsWindowController.swift */; }; AAAA0000000000000000004C /* ScreenAudioCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA0000000000000000004B /* ScreenAudioCapture.swift */; }; + AAAA00000000000000000051 /* InputDeviceCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA00000000000000000050 /* InputDeviceCapture.swift */; }; AAAA0000000000000000004E /* ScreenSharePickerSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAAA0000000000000000004F /* ScreenSharePickerSheet.swift */; }; /* End PBXBuildFile section */ @@ -45,6 +46,7 @@ AAAA00000000000000000019 /* ConnectWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectWindowController.swift; sourceTree = ""; }; AAAA0000000000000000001A /* MainWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainWindowController.swift; sourceTree = ""; }; AAAA0000000000000000004B /* ScreenAudioCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScreenAudioCapture.swift; sourceTree = ""; }; + AAAA00000000000000000050 /* InputDeviceCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputDeviceCapture.swift; sourceTree = ""; }; AAAA0000000000000000001B /* AddServerSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddServerSheet.swift; sourceTree = ""; }; AAAA0000000000000000001C /* ServerIdentitySheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerIdentitySheet.swift; sourceTree = ""; }; AAAA0000000000000000001D /* PasswordPromptSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PasswordPromptSheet.swift; sourceTree = ""; }; @@ -104,6 +106,7 @@ isa = PBXGroup; children = ( AAAA0000000000000000004B /* ScreenAudioCapture.swift */, + AAAA00000000000000000050 /* InputDeviceCapture.swift */, ); path = Audio; sourceTree = ""; @@ -247,6 +250,7 @@ AAAA0000000000000000004E /* ScreenSharePickerSheet.swift in Sources */, AAAA0000000000000000004A /* SettingsWindowController.swift in Sources */, AAAA0000000000000000004C /* ScreenAudioCapture.swift in Sources */, + AAAA00000000000000000051 /* InputDeviceCapture.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/clients/apple/macOS/VoiceCatMac/Audio/InputDeviceCapture.swift b/clients/apple/macOS/VoiceCatMac/Audio/InputDeviceCapture.swift new file mode 100644 index 0000000..c669cc6 --- /dev/null +++ b/clients/apple/macOS/VoiceCatMac/Audio/InputDeviceCapture.swift @@ -0,0 +1,229 @@ +import AVFoundation +import CoreAudio + +// InputDeviceCapture — captures a single hardware INPUT device (mic / line-in / aux) on macOS and +// emits 20 ms (960 samples/channel @ 48 kHz, interleaved int16) frames for the aux outgoing stream. +// +// The input-device analogue of ScreenAudioCapture (which captures system audio via +// ScreenCaptureKit). The core already owns ONE capture device (the mic) and can't open a second +// arbitrary input, so for the aux stream the client captures the chosen device here and feeds PCM +// into the core via `vc_stream_feed_pcm` — the same external-feed pipeline screen audio uses. +// +// Device selection: an AVAudioEngine's input node wraps an AUHAL audio unit; setting +// kAudioOutputUnitProperty_CurrentDevice on it pins capture to a specific Core Audio device. We +// pass the device's *UID* (stable across reboots/replugs, unlike AudioDeviceID) and resolve it at +// start. The tap delivers Float32; AVAudioConverter resamples/quantises to 48 kHz int16. +final class InputDeviceCapture { + + /// Receives a full 20 ms frame: (interleaved int16 PCM, samplesPerChannel = 960, channels). + typealias PcmHandler = (UnsafePointer, Int, UInt32) -> Void + + enum CaptureError: Error { case deviceNotFound, engineStartFailed(OSStatus) } + + private static let frameSamplesPerChannel = 960 // 20 ms @ 48 kHz + + private let deviceUID: String? // nil = system default input device + private let onPcm: PcmHandler + private let engine = AVAudioEngine() + private var converter: AVAudioConverter? + private var outFormat: AVAudioFormat? + private var channels = 1 + + /// Interleaved int16 carry-over between tap callbacks (the tap buffer doesn't align to 20 ms), + /// drained in whole `frameSamplesPerChannel * channels` chunks. Only touched on the tap queue. + private var pending: [Int16] = [] + + init(deviceUID: String?, onPcm: @escaping PcmHandler) { + self.deviceUID = deviceUID + self.onPcm = onPcm + } + + /// Begin capture. Throws if the device can't be resolved or the engine fails to start. + func start() throws { + let input = engine.inputNode + + // Pin the engine's AUHAL to the chosen device (skip for default — the engine already uses + // the system default input). Must happen before reading inputFormat, which changes with + // the selected device. + if let deviceUID, let devId = Self.deviceID(forUID: deviceUID) { + var dev = devId + let status = AudioUnitSetProperty(input.audioUnit!, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, 0, + &dev, UInt32(MemoryLayout.size)) + if status != noErr { throw CaptureError.engineStartFailed(status) } + } else if deviceUID != nil { + throw CaptureError.deviceNotFound + } + + let inFormat = input.inputFormat(forBus: 0) + channels = max(1, min(2, Int(inFormat.channelCount))) + guard let out = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000, + channels: AVAudioChannelCount(channels), interleaved: true) + else { throw CaptureError.deviceNotFound } + outFormat = out + converter = AVAudioConverter(from: inFormat, to: out) + + input.installTap(onBus: 0, bufferSize: 960, format: inFormat) { [weak self] buf, _ in + self?.process(buf) + } + engine.prepare() + do { try engine.start() } + catch { throw CaptureError.engineStartFailed(-1) } + } + + /// Stop capture and tear down the engine. Safe to call multiple times. + func stop() { + engine.inputNode.removeTap(onBus: 0) + if engine.isRunning { engine.stop() } + converter = nil + pending.removeAll(keepingCapacity: false) + } + + // MARK: - Conversion (called on the tap's realtime thread) + + private func process(_ inBuf: AVAudioPCMBuffer) { + guard let converter, let outFormat else { return } + + // Output capacity must cover up-sampling (e.g. 44.1 → 48 kHz) plus slack. + let ratio = outFormat.sampleRate / inBuf.format.sampleRate + let cap = AVAudioFrameCount(Double(inBuf.frameLength) * ratio + 32) + guard cap > 0, let outBuf = AVAudioPCMBuffer(pcmFormat: outFormat, frameCapacity: cap) + else { return } + + var fed = false + var err: NSError? + let status = converter.convert(to: outBuf, error: &err) { _, outStatus in + if fed { outStatus.pointee = .noDataNow; return nil } + fed = true + outStatus.pointee = .haveData + return inBuf + } + guard status != .error, outBuf.frameLength > 0, + let ch = outBuf.int16ChannelData else { return } + + // Interleaved int16: all channels live in the first buffer (ch[0]). + let n = Int(outBuf.frameLength) * channels + pending.append(contentsOf: UnsafeBufferPointer(start: ch[0], count: n)) + emit() + } + + /// Fire `onPcm` for every whole 20 ms frame accumulated. + private func emit() { + let full = Self.frameSamplesPerChannel * channels + while pending.count >= full { + pending.withUnsafeBufferPointer { buf in + onPcm(buf.baseAddress!, Self.frameSamplesPerChannel, UInt32(channels)) + } + pending.removeFirst(full) + } + } + + // MARK: - UID → AudioDeviceID resolution + + private static func deviceID(forUID uid: String) -> AudioDeviceID? { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyTranslateUIDToDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var deviceID = AudioDeviceID(0) + var cfUID = uid as CFString + var size = UInt32(MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &cfUID) { uidPtr -> OSStatus in + AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr, + UInt32(MemoryLayout.size), uidPtr, + &size, &deviceID) + } + return (status == noErr && deviceID != 0) ? deviceID : nil + } +} + +// InputDeviceInfo / InputDeviceEnumerator — Core Audio input-device enumeration for the aux-stream +// picker. Separate from the core's vc_list_devices (whose ids are miniaudio-opaque and can't be +// passed to Core Audio); the aux device is client-captured, so the picker uses device UIDs. +struct InputDeviceInfo: Equatable { + let uid: String // stable across reboots/replugs — what we persist + let name: String + let isDefault: Bool +} + +enum InputDeviceEnumerator { + + /// All Core Audio devices that expose at least one input channel. + static func list() -> [InputDeviceInfo] { + let defaultUID = defaultInputUID() + var result: [InputDeviceInfo] = [] + + for devID in allDeviceIDs() { + guard inputChannelCount(devID) > 0 else { continue } + guard let uid = stringProperty(devID, kAudioDevicePropertyDeviceUID) else { continue } + let name = stringProperty(devID, kAudioObjectPropertyName) + ?? stringProperty(devID, kAudioDevicePropertyDeviceNameCFString) + ?? "Unknown input device" + result.append(InputDeviceInfo(uid: uid, name: name, isDefault: uid == defaultUID)) + } + return result.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + } + + // MARK: - Core Audio helpers + + private static func allDeviceIDs() -> [AudioDeviceID] { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var size = UInt32(0) + guard AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, + &size) == noErr, size > 0 else { return [] } + let count = Int(size) / MemoryLayout.size + var ids = [AudioDeviceID](repeating: 0, count: count) + let status = AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr, 0, + nil, &size, &ids) + return status == noErr ? ids : [] + } + + private static func inputChannelCount(_ devID: AudioDeviceID) -> Int { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioObjectPropertyScopeInput, + mElement: kAudioObjectPropertyElementMain) + var size = UInt32(0) + guard AudioObjectGetPropertyDataSize(devID, &addr, 0, nil, &size) == noErr, size > 0 + else { return 0 } + let bufList = UnsafeMutableRawPointer.allocate(byteCount: Int(size), + alignment: MemoryLayout.alignment) + defer { bufList.deallocate() } + guard AudioObjectGetPropertyData(devID, &addr, 0, nil, &size, bufList) == noErr + else { return 0 } + let abl = UnsafeMutableAudioBufferListPointer( + bufList.assumingMemoryBound(to: AudioBufferList.self)) + return abl.reduce(0) { $0 + Int($1.mNumberChannels) } + } + + private static func defaultInputUID() -> String? { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var devID = AudioDeviceID(0) + var size = UInt32(MemoryLayout.size) + guard AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, + &size, &devID) == noErr, devID != 0 else { return nil } + return stringProperty(devID, kAudioDevicePropertyDeviceUID) + } + + private static func stringProperty(_ devID: AudioDeviceID, + _ selector: AudioObjectPropertySelector) -> String? { + var addr = AudioObjectPropertyAddress( + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var cf: CFString? = nil + var size = UInt32(MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &cf) { ptr in + AudioObjectGetPropertyData(devID, &addr, 0, nil, &size, ptr) + } + guard status == noErr else { return nil } + return cf as String? + } +} diff --git a/clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift b/clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift index 3e53b6e..240d9f4 100644 --- a/clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift +++ b/clients/apple/macOS/VoiceCatMac/Windows/MainWindowController.swift @@ -31,6 +31,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { internal var micStreamId: UInt32 = 0 private var screenStreamId: UInt32 = 0 private var screenCapture: ScreenAudioCapture? + private var auxStreamId: UInt32 = 0 // 0 = aux (second input device) stream not active + private var auxCapture: InputDeviceCapture? // client-side capture feeding the aux stream // Last app/exclusion choice from the share picker; reused as the default next time. private var screenAudioSelection: ScreenAudioSelection = .default internal var pttKeyCode: UInt16 = 0x60 { // F8 @@ -60,6 +62,9 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { static let vadThreshold = "voice.vadThreshold" static let inputGain = "voice.inputGain" static let pttKeyCode = "voice.pttKeyCode" + static let auxEnabled = "voice.auxEnabled" + static let auxDeviceUID = "voice.auxDeviceUID" + static let auxGain = "voice.auxGain" } internal var selectedInputMode: VoiceCatInputMode = .voiceActivation { @@ -73,6 +78,20 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { } internal var selectedInputDeviceId: String? + // Aux outgoing stream: a second hardware input device the client captures itself and feeds to + // the core (kind = AUX_DEVICE, external_feed). Device + volume only — aux is always-on (the + // core never gates AUX_DEVICE on VAD/PTT). auxDeviceUID is a Core Audio device UID (stable), + // NOT a core/miniaudio id. auxGain is read live by the capture feed, so the slider is instant. + internal var auxEnabled: Bool = false { + didSet { UserDefaults.standard.set(auxEnabled, forKey: AudioDefaults.auxEnabled) } + } + internal var auxDeviceUID: String? { + didSet { UserDefaults.standard.set(auxDeviceUID, forKey: AudioDefaults.auxDeviceUID) } + } + internal var auxGain: Float = 1.0 { + didSet { UserDefaults.standard.set(auxGain, forKey: AudioDefaults.auxGain) } + } + // MARK: - UI components private let channelOutlineView = NSOutlineView() private let userTableView = NSTableView() @@ -140,6 +159,13 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { if d.object(forKey: AudioDefaults.pttKeyCode) != nil { pttKeyCode = UInt16(d.integer(forKey: AudioDefaults.pttKeyCode)) } + auxEnabled = d.bool(forKey: AudioDefaults.auxEnabled) + if d.object(forKey: AudioDefaults.auxDeviceUID) != nil { + auxDeviceUID = d.string(forKey: AudioDefaults.auxDeviceUID) + } + if d.object(forKey: AudioDefaults.auxGain) != nil { + auxGain = d.float(forKey: AudioDefaults.auxGain) + } } required init?(coder: NSCoder) { fatalError() } @@ -675,7 +701,8 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { displayedUsers = []; userTableView.reloadData() users.removeAll(); talkingUsers.removeAll() stopScreenCapture() - currentChannelId = 0; micStreamId = 0; screenStreamId = 0 + auxCapture?.stop(); auxCapture = nil // connection gone — drop capture, no stopStream + currentChannelId = 0; micStreamId = 0; screenStreamId = 0; auxStreamId = 0 composeField.isEnabled = false; sendButton.isEnabled = false joinVoiceButton?.isEnabled = false shareScreenButton?.isEnabled = false @@ -757,10 +784,12 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { EventFeedback.shared.play(.voiceOn) NSAccessibility.post(element: logTextView, notification: .announcementRequested, userInfo: [.announcement: "Joined voice", .priority: NSAccessibilityPriorityLevel.medium]) + startAuxStream() // no-op unless the aux stream is enabled in settings } else { addActivity("Failed to start microphone: \(result)") } } else { + stopAuxStream() client.setPushToTalk(false) client.stopStream(micStreamId) micStreamId = 0 @@ -771,6 +800,84 @@ final class MainWindowController: NSWindowController, NSWindowDelegate { } } + // MARK: - Aux input stream (second hardware input device) + + /// Called by SettingsWindowController when the user toggles the aux checkbox. + func applyAuxEnabled(_ on: Bool) { + auxEnabled = on + guard micStreamId != 0 else { return } // not in voice — applied on next Join Voice + if on { startAuxStream() } else { stopAuxStream() } + } + + /// Called by SettingsWindowController when the user picks a different aux device. + func applyAuxDevice(_ uid: String?) { + auxDeviceUID = uid + if auxStreamId != 0 { restartAuxCapture() } + } + + private func startAuxStream() { + guard auxStreamId == 0, auxEnabled else { return } + let (result, streamId) = client.startStream( + StreamDescriptor(kind: .auxDevice, deviceId: nil, label: "Aux device", externalFeed: true)) + guard result == .ok else { + addActivity("Failed to start aux stream: \(result)") + return + } + auxStreamId = streamId + startAuxCapture() + addActivity("Aux input stream active") + } + + private func startAuxCapture() { + let capture = InputDeviceCapture(deviceUID: auxDeviceUID) { [weak self] ptr, spc, ch in + self?.feedAux(ptr, samplesPerChannel: spc, channels: ch) + } + auxCapture = capture + do { try capture.start() } + catch { + addActivity("Failed to open aux input device") + stopAuxStream() + } + } + + // Re-open the capture on a different device while the aux stream stays up (the core stream id + // is unchanged — only the client-side capture source changes). + private func restartAuxCapture() { + guard auxStreamId != 0 else { return } + auxCapture?.stop() + auxCapture = nil + startAuxCapture() + } + + private func stopAuxStream() { + auxCapture?.stop() + auxCapture = nil + if auxStreamId != 0 { + client.stopStream(auxStreamId) + auxStreamId = 0 + } + } + + // Fired on the capture's realtime thread. feedPcm is thread-safe. Gain is read live from + // auxGain each frame so the volume slider takes effect immediately. + private func feedAux(_ pcm: UnsafePointer, samplesPerChannel: Int, channels: UInt32) { + guard auxStreamId != 0 else { return } + let gain = auxGain + if gain != 1.0 { + let n = samplesPerChannel * Int(channels) + var scaled = [Int16](repeating: 0, count: n) + for i in 0.. 0 { + auxDevicePicker.selectItem(at: 0) + } + // Persist the resolved selection so a relaunch (or Join Voice) opens the same device. + mainController?.auxDeviceUID = auxDevicePicker.selectedItem?.representedObject as? String + } + // MARK: - Level meter (called by MainWindowController) func updateLevel(rms: Float) { diff --git a/clients/windows/VoiceCat.App/Audio/InputDeviceCapture.cs b/clients/windows/VoiceCat.App/Audio/InputDeviceCapture.cs new file mode 100644 index 0000000..f115b2e --- /dev/null +++ b/clients/windows/VoiceCat.App/Audio/InputDeviceCapture.cs @@ -0,0 +1,454 @@ +using System.Runtime.InteropServices; + +// WASAPI shared-mode capture from a real hardware INPUT device (a microphone / line-in / aux +// device), plus enumeration of capture endpoints for the aux-stream picker. +// +// This is the input-device analogue of ProcessLoopbackCapture (which captures *render* loopback +// via the process-loopback activation hack). Here the source is an ordinary capture endpoint, so +// we use the standard IMMDevice.Activate(IAudioClient) path with RCW interfaces — no vtable +// gymnastics needed (a normal device's COM objects honour QueryInterface). +// +// Why client-side capture at all? The core already owns ONE capture device (the mic). It can't +// open a second arbitrary input device, so for the aux stream the client captures the device and +// feeds 48 kHz / 20 ms int16 frames into the core via vc_stream_feed_pcm — the same external-feed +// pipeline screen-audio sharing uses. The device ids here are WASAPI endpoint ids and are NOT the +// core's miniaudio ids, so the aux picker is populated independently of vc_list_devices. +namespace VoiceCat.App.Audio; + +/// An audio input (capture) endpoint for the aux-stream device picker. +/// is a WASAPI endpoint id (round-trip only; never construct by hand) — pass null to capture the +/// system default. returns the friendly name for ComboBox display. +public sealed record InputDeviceInfo(string Id, string Name, bool IsDefault) +{ + public override string ToString() => Name; +} + +/// Enumerates WASAPI capture endpoints. Separate from the core's vc_list_devices because +/// the aux device is opened client-side and needs a WASAPI id, not a miniaudio one. +public static class InputDeviceEnumerator +{ + public static IReadOnlyList List() + { + var result = new List(); + InputDeviceCapture.IMMDeviceEnumerator? enumerator = null; + IntPtr collectionPtr = IntPtr.Zero; + string? defaultId = null; + + try + { + enumerator = (InputDeviceCapture.IMMDeviceEnumerator)Activator.CreateInstance( + Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!; + + // Resolve the default capture endpoint id so the picker can flag it. + if (enumerator.GetDefaultAudioEndpoint(1 /*eCapture*/, 0 /*eConsole*/, + out var defDev) == 0 && defDev != null) + { + try { if (defDev.GetId(out string id) == 0) defaultId = id; } + finally { Marshal.ReleaseComObject(defDev); } + } + + // DEVICE_STATE_ACTIVE = 0x1 — only currently-usable endpoints. + if (enumerator.EnumAudioEndpoints(1 /*eCapture*/, 0x1, out collectionPtr) != 0 + || collectionPtr == IntPtr.Zero) + return result; + + var collection = (InputDeviceCapture.IMMDeviceCollection) + Marshal.GetObjectForIUnknown(collectionPtr); + collection.GetCount(out int count); + + for (int i = 0; i < count; i++) + { + if (collection.Item(i, out var dev) != 0 || dev == null) continue; + try + { + if (dev.GetId(out string id) != 0) continue; + string name = ReadFriendlyName(dev) ?? "Unknown input device"; + result.Add(new InputDeviceInfo(id, name, id == defaultId)); + } + finally { Marshal.ReleaseComObject(dev); } + } + } + catch { /* no audio subsystem / WASAPI unavailable — return what we have */ } + finally + { + if (collectionPtr != IntPtr.Zero) Marshal.Release(collectionPtr); + if (enumerator != null) Marshal.ReleaseComObject(enumerator); + } + + return result.OrderBy(d => d.Name, StringComparer.OrdinalIgnoreCase).ToList(); + } + + private static string? ReadFriendlyName(InputDeviceCapture.IMMDevice dev) + { + if (dev.OpenPropertyStore(0 /*STGM_READ*/, out IntPtr storePtr) != 0 + || storePtr == IntPtr.Zero) + return null; + try + { + var store = (InputDeviceCapture.IPropertyStore)Marshal.GetObjectForIUnknown(storePtr); + // PKEY_Device_FriendlyName = {a45c254e-df1c-4efd-8020-67d146a850e0}, pid 14. + var key = new InputDeviceCapture.PropertyKey + { + fmtid = new Guid("a45c254e-df1c-4efd-8020-67d146a850e0"), + pid = 14, + }; + if (store.GetValue(ref key, out var pv) != 0) return null; + try + { + // VT_LPWSTR = 31. + return pv.vt == 31 ? Marshal.PtrToStringUni(pv.pointerValue) : null; + } + finally { InputDeviceCapture.PropVariantClear(ref pv); } + } + finally { Marshal.Release(storePtr); } + } +} + +/// Captures a single hardware input device in WASAPI shared mode and raises a 20 ms +/// (960 samples/channel @ 48 kHz, interleaved s16) frame event. The caller feeds these to the +/// core's external-feed stream. Threading mirrors ProcessLoopbackCapture: all WASAPI work runs on +/// a dedicated background (MTA) thread; the event fires on that thread. +public sealed class InputDeviceCapture : IDisposable +{ + /// Fired on the capture thread every 20 ms: (interleaved s16 PCM, samplesPerChannel + /// = 960, channels). + public event Action? PcmFrameReady; + + private const int SampleRate = 48000; + private const int FrameSamples = 960; // 20 ms + + private readonly string? _deviceId; // null = system default capture endpoint + + private IAudioClient? _audioClient; + private IAudioCaptureClient? _captureClient; + + private AutoResetEvent? _bufferEvent; + private Thread? _captureThread; + private volatile bool _running; + private int _channels; + + // Accumulator: assembles driver-callback-sized fragments into FrameSamples chunks. + private short[] _accumBuf = []; + private int _accumCount; + + // Init-done signal: Set() by the capture thread after activation completes. + private readonly ManualResetEventSlim _initDone = new(false); + private bool _initOk; + + public InputDeviceCapture(string? deviceId) => _deviceId = deviceId; + + /// Starts capture. Blocks until WASAPI activation completes (typically <100 ms). + /// Returns false if the device cannot be opened. + public bool Start() + { + if (_running) return false; + _running = true; + _captureThread = new Thread(CaptureThreadProc) + { + IsBackground = true, + Name = "AuxInputCapture", + }; + _captureThread.Start(); + + bool ok = _initDone.Wait(5000) && _initOk; + if (!ok) _running = false; + return ok; + } + + public void Stop() + { + _running = false; + _bufferEvent?.Set(); + _captureThread?.Join(500); + try { _audioClient?.Stop(); } catch { /* device already gone */ } + } + + public void Dispose() + { + Stop(); + if (_captureClient != null) { Marshal.ReleaseComObject(_captureClient); _captureClient = null; } + if (_audioClient != null) { Marshal.ReleaseComObject(_audioClient); _audioClient = null; } + _bufferEvent?.Dispose(); + _initDone.Dispose(); + } + + // ── Capture thread (MTA) ────────────────────────────────────────────────── + + private void CaptureThreadProc() + { + _initOk = ActivateAndStart(); + _initDone.Set(); + if (!_initOk) return; + CaptureLoop(); + } + + private bool ActivateAndStart() + { + if (!ActivateClient()) return false; + + _bufferEvent = new AutoResetEvent(false); + if (_audioClient!.SetEventHandle(_bufferEvent.SafeWaitHandle.DangerousGetHandle()) < 0) + return false; + + return _audioClient.Start() >= 0; + } + + private bool ActivateClient() + { + IMMDeviceEnumerator? enumerator = null; + IMMDevice? device = null; + try + { + enumerator = (IMMDeviceEnumerator)Activator.CreateInstance( + Type.GetTypeFromCLSID(new Guid("BCDE0395-E52F-467C-8E3D-C4579291692E"))!)!; + + int hr = _deviceId is null + ? enumerator.GetDefaultAudioEndpoint(1 /*eCapture*/, 0 /*eConsole*/, out device) + : enumerator.GetDevice(_deviceId, out device); + if (hr != 0 || device == null) return false; + + var iidAudioClient = new Guid("1CB9AD4C-DBFA-4c32-B178-C2F568A703B2"); + if (device.Activate(ref iidAudioClient, 0x17 /*CLSCTX_ALL*/, IntPtr.Zero, + out object acObj) != 0 || acObj is not IAudioClient ac) + return false; + _audioClient = ac; + + return InitializeStream(); + } + catch { return false; } + finally + { + if (device != null) Marshal.ReleaseComObject(device); + if (enumerator != null) Marshal.ReleaseComObject(enumerator); + } + } + + private bool InitializeStream() + { + // Try s16 stereo first; fall back to s16 mono. AUTOCONVERTPCM lets the audio engine + // resample/convert the device's native format to our requested 48 kHz s16; EVENTCALLBACK + // drives the buffer-ready event. Shared mode (0), no LOOPBACK (this is a capture device). + // AUDCLNT_STREAMFLAGS_EVENTCALLBACK = 0x00040000 + // AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM = 0x80000000 + // AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY = 0x08000000 + const uint streamFlags = 0x00040000u | 0x80000000u | 0x08000000u; + + foreach (int ch in new[] { 2, 1 }) + { + var fmt = new WaveFormatEx + { + wFormatTag = 1, // WAVE_FORMAT_PCM + nChannels = (ushort)ch, + nSamplesPerSec = SampleRate, + wBitsPerSample = 16, + nBlockAlign = (ushort)(ch * 2), + nAvgBytesPerSec = (uint)(SampleRate * ch * 2), + cbSize = 0, + }; + + IntPtr pFmt = Marshal.AllocHGlobal(Marshal.SizeOf()); + try + { + Marshal.StructureToPtr(fmt, pFmt, false); + int hr = _audioClient!.Initialize(0 /*AUDCLNT_SHAREMODE_SHARED*/, streamFlags, + 2_000_000 /*200 ms hns*/, 0, pFmt, IntPtr.Zero); + if (hr >= 0) + { + _channels = ch; + _accumBuf = new short[FrameSamples * ch]; + _accumCount = 0; + + var iidCapture = new Guid("C8ADBD64-E71E-48a0-A4DE-185C395CD317"); + if (_audioClient.GetService(ref iidCapture, out object ccObj) != 0 + || ccObj is not IAudioCaptureClient cc) + return false; + _captureClient = cc; + return true; + } + if (ch == 1) return false; + } + finally { Marshal.FreeHGlobal(pFmt); } + } + return false; + } + + // ── Capture loop ───────────────────────────────────────────────────────── + + private void CaptureLoop() + { + while (_running) + { + _bufferEvent!.WaitOne(100); + if (!_running) break; + + while (_running) + { + if (_captureClient!.GetNextPacketSize(out uint packetSize) < 0 || packetSize == 0) + break; + + if (_captureClient.GetBuffer(out IntPtr dataPtr, out uint framesAvailable, + out uint flags, out _, out _) < 0) + break; + + bool silent = (flags & 2) != 0; // AUDCLNT_BUFFERFLAGS_SILENT + if (framesAvailable > 0) + { + if (silent) AccumulateSilence((int)framesAvailable); + else AccumulatePcm(dataPtr, (int)framesAvailable); + } + + _captureClient.ReleaseBuffer(framesAvailable); + } + } + } + + private unsafe void AccumulatePcm(IntPtr data, int frames) + { + var src = (short*)data.ToPointer(); + int total = frames * _channels; + int idx = 0; + while (idx < total) + { + int space = _accumBuf.Length - _accumCount; + int copy = Math.Min(total - idx, space); + fixed (short* dst = _accumBuf) + Buffer.MemoryCopy(src + idx, dst + _accumCount, copy * 2L, copy * 2L); + _accumCount += copy; + idx += copy; + if (_accumCount == _accumBuf.Length) + FlushFrame(); + } + } + + private void AccumulateSilence(int frames) + { + int total = frames * _channels; + int idx = 0; + while (idx < total) + { + int space = _accumBuf.Length - _accumCount; + int fill = Math.Min(total - idx, space); + Array.Clear(_accumBuf, _accumCount, fill); + _accumCount += fill; + idx += fill; + if (_accumCount == _accumBuf.Length) + FlushFrame(); + } + } + + private void FlushFrame() + { + var copy = new short[_accumBuf.Length]; + _accumBuf.AsSpan().CopyTo(copy); + PcmFrameReady?.Invoke(copy, FrameSamples, _channels); + _accumCount = 0; + } + + // ── COM declarations ─────────────────────────────────────────────────────── + // + // Declared internal here so InputDeviceEnumerator can share them. These are standard MMDevice + // / WASAPI interfaces; a normal capture endpoint honours QueryInterface, so RCW marshalling is + // safe (unlike ProcessLoopbackCapture's process-loopback objects, which need raw vtable calls). + + [ComImport, Guid("A95664D2-9614-4F35-A746-DE8DB63617E6"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IMMDeviceEnumerator + { + [PreserveSig] int EnumAudioEndpoints(int dataFlow, int stateMask, out IntPtr devices); + [PreserveSig] int GetDefaultAudioEndpoint(int dataFlow, int role, out IMMDevice endpoint); + [PreserveSig] int GetDevice([MarshalAs(UnmanagedType.LPWStr)] string id, out IMMDevice device); + [PreserveSig] int RegisterEndpointNotificationCallback(IntPtr client); + [PreserveSig] int UnregisterEndpointNotificationCallback(IntPtr client); + } + + [ComImport, Guid("0BD7A1BE-7A1A-44DB-8397-CC5392387B5E"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IMMDeviceCollection + { + [PreserveSig] int GetCount(out int count); + [PreserveSig] int Item(int index, out IMMDevice device); + } + + [ComImport, Guid("D666063F-1587-4E43-81F1-B948E807363F"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IMMDevice + { + [PreserveSig] int Activate(ref Guid iid, int clsCtx, IntPtr activationParams, + [MarshalAs(UnmanagedType.IUnknown)] out object ppInterface); + [PreserveSig] int OpenPropertyStore(int stgmAccess, out IntPtr propStore); + [PreserveSig] int GetId([MarshalAs(UnmanagedType.LPWStr)] out string id); + [PreserveSig] int GetState(out int state); + } + + [ComImport, Guid("886D8EEB-8CF2-4446-8D02-CDBA1DBDCF99"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IPropertyStore + { + [PreserveSig] int GetCount(out int count); + [PreserveSig] int GetAt(int index, out PropertyKey key); + [PreserveSig] int GetValue(ref PropertyKey key, out PropVariant value); + [PreserveSig] int SetValue(ref PropertyKey key, ref PropVariant value); + [PreserveSig] int Commit(); + } + + [ComImport, Guid("1CB9AD4C-DBFA-4C32-B178-C2F568A703B2"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioClient + { + [PreserveSig] int Initialize(int shareMode, uint streamFlags, long hnsBufferDuration, + long hnsPeriodicity, IntPtr pFormat, IntPtr audioSessionGuid); + [PreserveSig] int GetBufferSize(out uint numBufferFrames); + [PreserveSig] int GetStreamLatency(out long latency); + [PreserveSig] int GetCurrentPadding(out uint numPaddingFrames); + [PreserveSig] int IsFormatSupported(int shareMode, IntPtr pFormat, out IntPtr closestMatch); + [PreserveSig] int GetMixFormat(out IntPtr deviceFormat); + [PreserveSig] int GetDevicePeriod(out long defaultDevicePeriod, out long minimumDevicePeriod); + [PreserveSig] int Start(); + [PreserveSig] int Stop(); + [PreserveSig] int Reset(); + [PreserveSig] int SetEventHandle(IntPtr eventHandle); + [PreserveSig] int GetService(ref Guid riid, + [MarshalAs(UnmanagedType.IUnknown)] out object ppv); + } + + [ComImport, Guid("C8ADBD64-E71E-48A0-A4DE-185C395CD317"), + InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IAudioCaptureClient + { + [PreserveSig] int GetBuffer(out IntPtr data, out uint numFramesToRead, out uint flags, + out ulong devicePosition, out ulong qpcPosition); + [PreserveSig] int ReleaseBuffer(uint numFramesRead); + [PreserveSig] int GetNextPacketSize(out uint numFramesInNextPacket); + } + + [StructLayout(LayoutKind.Sequential, Pack = 2)] + private struct WaveFormatEx + { + public ushort wFormatTag; + public ushort nChannels; + public uint nSamplesPerSec; + public uint nAvgBytesPerSec; + public ushort nBlockAlign; + public ushort wBitsPerSample; + public ushort cbSize; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PropertyKey + { + public Guid fmtid; + public int pid; + } + + // Minimal PROPVARIANT: we only ever read VT_LPWSTR (friendly name). x64 layout — the value + // union starts at offset 8 after vt(2)+reserved(6). + [StructLayout(LayoutKind.Explicit)] + internal struct PropVariant + { + [FieldOffset(0)] public ushort vt; + [FieldOffset(8)] public IntPtr pointerValue; + } + + [DllImport("ole32.dll")] + internal static extern int PropVariantClear(ref PropVariant pvar); +} diff --git a/clients/windows/VoiceCat.App/Forms/AudioSettingsForm.cs b/clients/windows/VoiceCat.App/Forms/AudioSettingsForm.cs index 35ca910..3a85e01 100644 --- a/clients/windows/VoiceCat.App/Forms/AudioSettingsForm.cs +++ b/clients/windows/VoiceCat.App/Forms/AudioSettingsForm.cs @@ -1,4 +1,5 @@ using System.ComponentModel; +using VoiceCat.App.Audio; using VoiceCat.App.Models; using VoiceCat.Interop; @@ -15,12 +16,21 @@ public sealed class AudioSettingsForm : Form private readonly VoiceSettings _settings; private readonly uint _micStreamId; + // Aux-stream live-apply callbacks, supplied by MainForm (which owns the aux capture). Null + // when not connected — the controls still edit settings, just without live effect. + private readonly Action? _applyAuxEnabled; + private readonly Action? _applyAuxDevice; + private readonly Action? _applyAuxGain; + // Snapshot of original values so Cancel can restore them private readonly string? _origDeviceId; private readonly VcInputMode _origMode; private readonly int _origVadSlider; private readonly int _origMicGain; private readonly Keys _origPttKey; + private readonly bool _origAuxEnabled; + private readonly string? _origAuxDeviceId; + private readonly int _origAuxGain; private readonly ComboBox _cboDevice; private readonly Button _btnRefresh; @@ -32,14 +42,25 @@ public sealed class AudioSettingsForm : Form private readonly Label _lblSensitivity; private readonly TrackBar _trkVad; private readonly TrackBar _trkGain; + private readonly CheckBox _chkAux; + private readonly Label _lblAuxDevice; + private readonly ComboBox _cboAuxDevice; + private readonly Button _btnAuxRefresh; + private readonly Label _lblAuxGain; + private readonly TrackBar _trkAuxGain; private Keys _pttKey; - public AudioSettingsForm(VoiceCatClient client, VoiceSettings settings, uint micStreamId) + public AudioSettingsForm(VoiceCatClient client, VoiceSettings settings, uint micStreamId, + Action? applyAuxEnabled = null, Action? applyAuxDevice = null, + Action? applyAuxGain = null) { _client = client; _settings = settings; _micStreamId = micStreamId; + _applyAuxEnabled = applyAuxEnabled; + _applyAuxDevice = applyAuxDevice; + _applyAuxGain = applyAuxGain; _pttKey = (Keys)settings.PttKey; _origDeviceId = settings.InputDeviceId; @@ -47,6 +68,9 @@ public sealed class AudioSettingsForm : Form _origVadSlider = settings.VadThresholdSlider; _origMicGain = settings.MicGain; _origPttKey = _pttKey; + _origAuxEnabled = settings.AuxEnabled; + _origAuxDeviceId = settings.AuxDeviceId; + _origAuxGain = settings.AuxGain; Text = "Audio settings"; FormBorderStyle = FormBorderStyle.FixedDialog; @@ -54,7 +78,7 @@ public sealed class AudioSettingsForm : Form MinimizeBox = false; StartPosition = FormStartPosition.CenterParent; AutoScaleMode = AutoScaleMode.Font; - ClientSize = new Size(420, 370); + ClientSize = new Size(420, 555); // ── Device row ──────────────────────────────────────────────────────── var lblDevice = new Label @@ -175,19 +199,83 @@ public sealed class AudioSettingsForm : Form "Boost a quiet microphone. 100 is unity gain; range 0–300 percent."; _trkGain.Scroll += TrkGain_Scroll; + // ── Aux input stream ────────────────────────────────────────────────── + // A second outgoing stream from another hardware input device (e.g. line-in / aux), + // captured client-side and fed to the core. Device + volume only — aux is always-on + // (the core never gates AUX_DEVICE on VAD/PTT). + _chkAux = new CheckBox + { + Text = "&Aux stream (second input device)", + Location = new Point(12, 348), + AutoSize = true, + Checked = settings.AuxEnabled, + AccessibleName = "Enable aux input stream", + AccessibleDescription = + "Transmit a second hardware input device alongside your microphone.", + }; + _chkAux.CheckedChanged += ChkAux_CheckedChanged; + + _lblAuxDevice = new Label + { + Text = "Aux d&evice:", + Location = new Point(12, 378), + AutoSize = true, + }; + _cboAuxDevice = new ComboBox + { + Location = new Point(12, 398), + Width = 300, + DropDownStyle = ComboBoxStyle.DropDownList, + DisplayMember = "Name", + ValueMember = "Id", + AccessibleName = "Aux input device", + AccessibleDescription = "Select the second audio input device to transmit.", + }; + _cboAuxDevice.SelectedIndexChanged += CboAuxDevice_SelectedIndexChanged; + + _btnAuxRefresh = new Button + { + Text = "Re&fresh", + Location = new Point(320, 396), + Size = new Size(80, 26), + }; + _btnAuxRefresh.Click += (_, _) => LoadAuxDevices(); + + _lblAuxGain = new Label + { + Text = "Aux vo&lume:", + Location = new Point(12, 434), + AutoSize = true, + }; + _trkAuxGain = new TrackBar + { + Location = new Point(12, 454), + Size = new Size(200, 45), + Minimum = 0, + Maximum = 300, + TickFrequency = 25, + SmallChange = 5, + LargeChange = 25, + Value = Math.Clamp(settings.AuxGain, 0, 300), + }; + _trkAuxGain.AccessibleName = "Aux volume"; + _trkAuxGain.AccessibleDescription = + "Volume of the aux input stream. 100 is unity gain; range 0–300 percent."; + _trkAuxGain.Scroll += TrkAuxGain_Scroll; + // ── OK / Cancel ─────────────────────────────────────────────────────── var btnOk = new Button { Text = "&OK", DialogResult = DialogResult.OK, - Location = new Point(228, 334), + Location = new Point(228, 516), Size = new Size(80, 27), }; var btnCancel = new Button { Text = "&Cancel", DialogResult = DialogResult.Cancel, - Location = new Point(316, 334), + Location = new Point(316, 516), Size = new Size(80, 27), }; @@ -205,6 +293,7 @@ public sealed class AudioSettingsForm : Form lblMode, _radioVad, _lblSensitivity, _trkVad, _radioPtt, _lblPttKey, _btnChangePtt, _radioAlwaysOn, lblGain, _trkGain, + _chkAux, _lblAuxDevice, _cboAuxDevice, _btnAuxRefresh, _lblAuxGain, _trkAuxGain, btnOk, btnCancel, ]); @@ -217,6 +306,8 @@ public sealed class AudioSettingsForm : Form } LoadDevices(); + LoadAuxDevices(); + UpdateAuxControlsEnabled(); } private void LoadDevices() @@ -245,6 +336,65 @@ public sealed class AudioSettingsForm : Form _client.SetInputDevice(_micStreamId, dev.IsDefault ? null : dev.Id); } + // ── Aux input stream ────────────────────────────────────────────────────── + + private void LoadAuxDevices() + { + string? currentId = (_cboAuxDevice.SelectedItem as InputDeviceInfo)?.Id + ?? _settings.AuxDeviceId; + + var devices = InputDeviceEnumerator.List().ToList(); + + // Set the data source AND restore the selection while unsubscribed so neither the initial + // populate nor a Refresh fires a spurious device-change (which would restart the capture). + _cboAuxDevice.SelectedIndexChanged -= CboAuxDevice_SelectedIndexChanged; + _cboAuxDevice.DataSource = new BindingList(devices); + + int idx = -1; + if (currentId is not null) + idx = devices.FindIndex(d => d.Id == currentId); + if (idx < 0) + idx = devices.FindIndex(d => d.IsDefault); + _cboAuxDevice.SelectedIndex = idx >= 0 ? idx : (devices.Count > 0 ? 0 : -1); + _cboAuxDevice.SelectedIndexChanged += CboAuxDevice_SelectedIndexChanged; + + // Persist the resolved selection (null = default) so Join Voice opens the same device. + if (_cboAuxDevice.SelectedItem is InputDeviceInfo dev) + _settings.AuxDeviceId = dev.IsDefault ? null : dev.Id; + } + + private void ChkAux_CheckedChanged(object? sender, EventArgs e) + { + _settings.AuxEnabled = _chkAux.Checked; + UpdateAuxControlsEnabled(); + _applyAuxEnabled?.Invoke(_chkAux.Checked); + } + + private void CboAuxDevice_SelectedIndexChanged(object? sender, EventArgs e) + { + if (_cboAuxDevice.SelectedItem is not InputDeviceInfo dev) return; + _settings.AuxDeviceId = dev.IsDefault ? null : dev.Id; + if (_chkAux.Checked) + _applyAuxDevice?.Invoke(dev.IsDefault ? null : dev.Id); + } + + private void TrkAuxGain_Scroll(object? sender, EventArgs e) + { + _settings.AuxGain = _trkAuxGain.Value; + if (_chkAux.Checked) + _applyAuxGain?.Invoke(_trkAuxGain.Value / 100f); + } + + private void UpdateAuxControlsEnabled() + { + bool on = _chkAux.Checked; + _lblAuxDevice.Enabled = on; + _cboAuxDevice.Enabled = on; + _btnAuxRefresh.Enabled = on; + _lblAuxGain.Enabled = on; + _trkAuxGain.Enabled = on; + } + private void RadioMode_CheckedChanged(object? sender, EventArgs e) { var mode = CurrentMode(); @@ -310,6 +460,15 @@ public sealed class AudioSettingsForm : Form _client.SetVadThreshold(0.1f * (1f - (_origVadSlider - 1f) / 99f)); _client.SetInputGain(_origMicGain / 100f); } + + // Aux: restore originals to settings and re-apply live (order: device + gain first so a + // re-enable starts with the right configuration). + _settings.AuxEnabled = _origAuxEnabled; + _settings.AuxDeviceId = _origAuxDeviceId; + _settings.AuxGain = _origAuxGain; + _applyAuxDevice?.Invoke(_origAuxDeviceId); + _applyAuxGain?.Invoke(_origAuxGain / 100f); + _applyAuxEnabled?.Invoke(_origAuxEnabled); } private void UpdatePttKeyLabel() => diff --git a/clients/windows/VoiceCat.App/Forms/MainForm.cs b/clients/windows/VoiceCat.App/Forms/MainForm.cs index e0e5c70..1e81e74 100644 --- a/clients/windows/VoiceCat.App/Forms/MainForm.cs +++ b/clients/windows/VoiceCat.App/Forms/MainForm.cs @@ -28,6 +28,8 @@ public partial class MainForm : Form private uint _micStreamId; // 0 = not started private uint _screenStreamId; // 0 = not sharing screen audio private ProcessAudioMixer? _screenMixer; // non-null only in per-app capture mode + private uint _auxStreamId; // 0 = aux (second input device) stream not active + private InputDeviceCapture? _auxCapture; // client-side capture feeding the aux stream private Keys _pttKey = Keys.F8; private bool _pttEngaged; // guards the PTT cue against key-repeat private bool _serverMuted; @@ -169,7 +171,16 @@ public partial class MainForm : Form var miAudio = new ToolStripMenuItem("&Audio..."); miAudio.Click += (_, _) => { - using var dlg = new AudioSettingsForm(_client, _voiceSettings, _micStreamId); + using var dlg = new AudioSettingsForm(_client, _voiceSettings, _micStreamId, + applyAuxEnabled: on => + { + if (_micStreamId == 0) return; // not in voice — applied on next Join Voice + if (on) StartAuxStream(); else StopAuxStream(); + }, + applyAuxDevice: _ => + { + if (_auxStreamId != 0) RestartAuxCapture(); // settings.AuxDeviceId already updated + }); dlg.ShowDialog(this); _pttKey = (Keys)_voiceSettings.PttKey; }; @@ -511,6 +522,7 @@ public partial class MainForm : Form _micStreamId = 0; _screenStreamId = 0; _screenMixer?.Stop(); _screenMixer?.Dispose(); _screenMixer = null; + DisposeAuxCapture(); _auxStreamId = 0; // connection gone — drop capture, no StopStream txtCompose.Enabled = false; btnSend.Enabled = false; tsbJoinVoice.Enabled = false; @@ -641,6 +653,7 @@ public partial class MainForm : Form SetVoiceJoinedState(true); AddActivity("Joined voice — microphone active"); _feedback.PlaySound(SoundEvent.VoiceOn); + StartAuxStream(); // no-op unless the aux stream is enabled in settings } else { @@ -649,6 +662,7 @@ public partial class MainForm : Form } else { + StopAuxStream(); _client.SetPushToTalk(false); _client.StopStream(_micStreamId); _micStreamId = 0; @@ -743,6 +757,82 @@ public partial class MainForm : Form AddActivity("Stopped sharing screen audio"); } + // ── Aux input stream (second hardware input device) ───────────────────────── + // A second outgoing stream (kind = AUX_DEVICE, external_feed). The core can't open a second + // capture device, so we capture the chosen device here and feed PCM in — the same external- + // feed pipeline as per-app screen audio. Tied to the voice session: started on Join Voice + // (when enabled) and stopped on Leave Voice. The aux is always-on (the core never gates + // AUX_DEVICE on VAD/PTT); volume is applied client-side before feeding. + + private void StartAuxStream() + { + if (_auxStreamId != 0 || !_voiceSettings.AuxEnabled) return; + + var (result, streamId) = _client.StartStreamExternalFeed(VcStreamKind.AuxDevice, "Aux device"); + if (result != VcResult.Ok) + { + AddActivity($"Failed to start aux stream: {result}"); + return; + } + _auxStreamId = streamId; + + _auxCapture = new InputDeviceCapture(_voiceSettings.AuxDeviceId); + _auxCapture.PcmFrameReady += OnAuxPcmFrame; + if (!_auxCapture.Start()) + { + AddActivity("Failed to open aux input device"); + StopAuxStream(); + return; + } + AddActivity("Aux input stream active"); + } + + private void StopAuxStream() + { + DisposeAuxCapture(); + if (_auxStreamId != 0) + { + _client.StopStream(_auxStreamId); + _auxStreamId = 0; + } + } + + // Re-open the capture on a different device while the aux stream stays up (the core stream id + // is unchanged — only the client-side capture source changes). + private void RestartAuxCapture() + { + if (_auxStreamId == 0) return; + DisposeAuxCapture(); + _auxCapture = new InputDeviceCapture(_voiceSettings.AuxDeviceId); + _auxCapture.PcmFrameReady += OnAuxPcmFrame; + if (!_auxCapture.Start()) + AddActivity("Failed to open aux input device"); + } + + private void DisposeAuxCapture() + { + if (_auxCapture == null) return; + _auxCapture.PcmFrameReady -= OnAuxPcmFrame; + _auxCapture.Stop(); + _auxCapture.Dispose(); + _auxCapture = null; + } + + // Fired on the capture thread. vc_stream_feed_pcm is thread-safe, so feed directly. Gain is + // read live from settings each frame (so the volume slider takes effect immediately). + private void OnAuxPcmFrame(short[] pcm, int samplesPerChannel, int channels) + { + if (_auxStreamId == 0) return; + float gain = _voiceSettings.AuxGain / 100f; + if (gain != 1f) + { + for (int i = 0; i < pcm.Length; i++) + pcm[i] = (short)Math.Clamp((int)MathF.Round(pcm[i] * gain), + short.MinValue, short.MaxValue); + } + _client.StreamFeedPcm(_auxStreamId, pcm, samplesPerChannel, (uint)channels); + } + private void TrkOutputVolume_Scroll(object? sender, EventArgs e) => _client.SetOutputVolume(trkOutputVolume.Value / 100f); @@ -1057,6 +1147,7 @@ public partial class MainForm : Form foreach (var win in _pmWindows.Values.ToList()) win.Close(); _pmWindows.Clear(); if (_screenStreamId != 0) { _screenMixer?.Stop(); _screenMixer?.Dispose(); _screenMixer = null; _client.StopStream(_screenStreamId); } + if (_auxStreamId != 0) StopAuxStream(); if (_micStreamId != 0) _client.StopStream(_micStreamId); _client.Disconnect(); _client.Dispose(); diff --git a/clients/windows/VoiceCat.App/Models/VoiceSettings.cs b/clients/windows/VoiceCat.App/Models/VoiceSettings.cs index e95a0e8..9b09795 100644 --- a/clients/windows/VoiceCat.App/Models/VoiceSettings.cs +++ b/clients/windows/VoiceCat.App/Models/VoiceSettings.cs @@ -27,6 +27,21 @@ public sealed class VoiceSettings /// Saved device ID from the last session; null means use the system default. public string? InputDeviceId { get; set; } = null; + /// Whether the secondary "aux" outgoing stream is enabled. The aux stream is a + /// second hardware input device the client captures itself and feeds to the core via + /// vc_stream_feed_pcm (kind = AUX_DEVICE, external_feed). Lets a user transmit e.g. mic + + /// a line-in at once. + public bool AuxEnabled { get; set; } = false; + + /// WASAPI endpoint id of the aux capture device; null = system default capture + /// device. NOTE: this is a WASAPI device id (from ), + /// NOT a core/miniaudio id — the aux device is opened client-side, so the two id spaces differ. + public string? AuxDeviceId { get; set; } = null; + + /// Aux input volume slider position, 0–300 percent (100 = unity). Applied client-side + /// to the captured PCM before feeding (the core's input gain is mic-only and global). + public int AuxGain { get; set; } = 100; + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true }; private static string AppDataDir => Path.Combine(